Introduce remote store path type in customData in IndexMetadata

ashking94 · ashking94 · commit d256b66e3dd8 · 2024-03-14T15:28:12.000+05:30
Signed-off-by: Ashish Singh &lt;ssashish@amazon.com&gt;
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java
@@ -635,6 +635,7 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
     static final String KEY_ROLLOVER_INFOS = "rollover_info";
     static final String KEY_SYSTEM = "system";
     public static final String KEY_PRIMARY_TERMS = "primary_terms";
+    public static final String REMOTE_STORE_CUSTOM_KEY = "remote_store";
 
     public static final String INDEX_STATE_FILE_PREFIX = "state-";
 
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -88,6 +88,7 @@
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.MapperService.MergeReason;
 import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.remote.RemoteStorePathType;
 import org.opensearch.index.shard.IndexSettingProvider;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.IndexCreationException;
@@ -498,7 +499,8 @@ private ClusterState applyCreateIndexWithTemporaryService(
                     temporaryIndexMeta.getSettings(),
                     temporaryIndexMeta.getRoutingNumShards(),
                     sourceMetadata,
-                    temporaryIndexMeta.isSystem()
+                    temporaryIndexMeta.isSystem(),
+                    temporaryIndexMeta.getCustomData()
                 );
             } catch (Exception e) {
                 logger.info("failed to build index metadata [{}]", request.index());
@@ -522,10 +524,11 @@ private ClusterState applyCreateIndexWithTemporaryService(
 
     /**
      * Given a state and index settings calculated after applying templates, validate metadata for
-     * the new index, returning an {@link IndexMetadata} for the new index
+     * the new index, returning an {@link IndexMetadata} for the new index.
+     * <p>
+     * The access level of the method changed to default level for visibility to test.
      */
-    private IndexMetadata buildAndValidateTemporaryIndexMetadata(
-        final ClusterState currentState,
+    IndexMetadata buildAndValidateTemporaryIndexMetadata(
         final Settings aggregatedIndexSettings,
         final CreateIndexClusterStateUpdateRequest request,
         final int routingNumShards
@@ -544,6 +547,16 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata(
         tmpImdBuilder.settings(indexSettings);
         tmpImdBuilder.system(isSystem);
 
+        if (isRemoteStoreAttributePresent(settings)) {
+            String pathType;
+            if (clusterService.getClusterSettings().get(IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_OPTIMISED_SETTING)) {
+                pathType = RemoteStorePathType.HASHED_PREFIX.toString();
+            } else {
+                pathType = RemoteStorePathType.FIXED.toString();
+            }
+            tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, Map.of(RemoteStorePathType.NAME, pathType));
+        }
+
         // Set up everything, now locally create the index to see that things are ok, and apply
         IndexMetadata tempMetadata = tmpImdBuilder.build();
         validateActiveShardCount(request.waitForActiveShards(), tempMetadata);
@@ -582,7 +595,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(
             clusterService.getClusterSettings()
         );
         int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
-        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards);
+        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
 
         return applyCreateIndexWithTemporaryService(
             currentState,
@@ -647,7 +660,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(
             clusterService.getClusterSettings()
         );
         int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null);
-        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards);
+        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
 
         return applyCreateIndexWithTemporaryService(
             currentState,
@@ -728,7 +741,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(
             clusterService.getClusterSettings()
         );
         final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata);
-        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards);
+        IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(aggregatedIndexSettings, request, routingNumShards);
 
         return applyCreateIndexWithTemporaryService(
             currentState,
@@ -1147,7 +1160,8 @@ static IndexMetadata buildIndexMetadata(
         Settings indexSettings,
         int routingNumShards,
         @Nullable IndexMetadata sourceMetadata,
-        boolean isSystem
+        boolean isSystem,
+        Map<String, DiffableStringMap> customData
     ) {
         IndexMetadata.Builder indexMetadataBuilder = createIndexMetadataBuilder(indexName, sourceMetadata, indexSettings, routingNumShards);
         indexMetadataBuilder.system(isSystem);
@@ -1168,6 +1182,10 @@ static IndexMetadata buildIndexMetadata(
             indexMetadataBuilder.putAlias(aliases.get(i));
         }
 
+        for (Map.Entry<String, DiffableStringMap> entry : customData.entrySet()) {
+            indexMetadataBuilder.putCustom(entry.getKey(), entry.getValue());
+        }
+
         indexMetadataBuilder.state(IndexMetadata.State.OPEN);
         return indexMetadataBuilder.build();
     }
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -709,6 +709,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 CpuBasedAdmissionControllerSettings.INDEXING_CPU_USAGE_LIMIT,
                 CpuBasedAdmissionControllerSettings.SEARCH_CPU_USAGE_LIMIT,
                 IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING,
+                IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_OPTIMISED_SETTING,
 
                 // Concurrent segment search settings
                 SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING,
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathType.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathType.java
@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+/**
+ * Enumerates the types of remote store paths resolution techniques supported by OpenSearch.
+ * For more information, see <a href="https://github.com/opensearch-project/OpenSearch/issues/12567">Github issue #12567</a>.
+ *
+ * @opensearch.internal
+ */
+public enum RemoteStorePathType {
+
+    FIXED,
+    HASHED_PREFIX;
+
+    public static RemoteStorePathType parseString(String remoteStorePathType) {
+        try {
+            return RemoteStorePathType.valueOf(remoteStorePathType);
+        } catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Could not parse RemoteStorePathType for [" + remoteStorePathType + "]");
+        } catch (NullPointerException npe) {
+            // return a default value for null input
+            return FIXED;
+        }
+    }
+
+    /**
+     * This string is used as key for storing information in the custom data in index settings.
+     */
+    public static final String NAME = "path_type";
+}
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -314,6 +314,17 @@ public class IndicesService extends AbstractLifecycleComponent
         Property.Final
     );
 
+    /**
+     * This setting is used to enable the optimisation in prefix path which helps in achieving higher throughput and lesser
+     * rate limiting by remote store providers. This setting is effective only for remote store enabled cluster.
+     */
+    public static final Setting<Boolean> CLUSTER_REMOTE_STORE_PATH_PREFIX_OPTIMISED_SETTING = Setting.boolSetting(
+        "cluster.remote_store.index.path.prefix.optimised",
+        false,
+        Property.NodeScope,
+        Property.Dynamic
+    );
+
     /**
      * The node's settings.
      */
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -71,6 +71,7 @@
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.remote.RemoteStorePathType;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.IndexCreationException;
 import org.opensearch.indices.IndicesService;
@@ -1563,13 +1564,104 @@ public void testBuildIndexMetadata() {
             .put(SETTING_NUMBER_OF_SHARDS, 1)
             .build();
         List<AliasMetadata> aliases = singletonList(AliasMetadata.builder("alias1").build());
-        IndexMetadata indexMetadata = buildIndexMetadata("test", aliases, () -> null, indexSettings, 4, sourceIndexMetadata, false);
+        IndexMetadata indexMetadata = buildIndexMetadata(
+            "test",
+            aliases,
+            () -> null,
+            indexSettings,
+            4,
+            sourceIndexMetadata,
+            false,
+            new HashMap<>()
+        );
 
         assertThat(indexMetadata.getAliases().size(), is(1));
         assertThat(indexMetadata.getAliases().keySet().iterator().next(), is("alias1"));
         assertThat("The source index primary term must be used", indexMetadata.primaryTerm(0), is(3L));
     }
 
+    /**
+     * This test checks if the cluster is a remote store cluster then we populate custom data for remote settings in
+     * index metadata of the underlying index. This captures information around the resolution pattern of the path for
+     * remote segments and translog.
+     */
+    public void testRemoteCustomData() {
+        // Case 1 - Remote store is not enabled
+        IndexMetadata indexMetadata = testRemoteCustomData(false, randomBoolean());
+        assertNull(indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY));
+
+        // Case 2 - cluster.remote_store.index.path.prefix.optimised=false (default value)
+        indexMetadata = testRemoteCustomData(true, false);
+        validateRemoteCustomData(
+            indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY),
+            RemoteStorePathType.NAME,
+            RemoteStorePathType.FIXED.toString()
+        );
+
+        // Case 3 - cluster.remote_store.index.path.prefix.optimised=true
+        indexMetadata = testRemoteCustomData(true, true);
+        validateRemoteCustomData(
+            indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY),
+            RemoteStorePathType.NAME,
+            RemoteStorePathType.HASHED_PREFIX.toString()
+        );
+    }
+
+    private IndexMetadata testRemoteCustomData(boolean remoteStoreEnabled, boolean optimisedPrefix) {
+        Settings.Builder settingsBuilder = Settings.builder();
+        if (remoteStoreEnabled) {
+            settingsBuilder.put(NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "test");
+        }
+        if (optimisedPrefix) {
+            settingsBuilder.put(IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_OPTIMISED_SETTING.getKey(), true);
+        }
+        Settings settings = settingsBuilder.build();
+
+        ClusterService clusterService = mock(ClusterService.class);
+        Metadata metadata = Metadata.builder()
+            .transientSettings(Settings.builder().put(Metadata.DEFAULT_REPLICA_COUNT_SETTING.getKey(), 1).build())
+            .build();
+        ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metadata(metadata)
+            .build();
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        when(clusterService.getSettings()).thenReturn(settings);
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+        when(clusterService.state()).thenReturn(clusterState);
+
+        ThreadPool threadPool = new TestThreadPool(getTestName());
+        MetadataCreateIndexService metadataCreateIndexService = new MetadataCreateIndexService(
+            settings,
+            clusterService,
+            null,
+            null,
+            null,
+            createTestShardLimitService(randomIntBetween(1, 1000), false, clusterService),
+            new Environment(Settings.builder().put("path.home", "dummy").build(), null),
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            threadPool,
+            null,
+            new SystemIndices(Collections.emptyMap()),
+            true,
+            new AwarenessReplicaBalance(settings, clusterService.getClusterSettings())
+        );
+        CreateIndexClusterStateUpdateRequest request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+        Settings indexSettings = Settings.builder()
+            .put("index.version.created", Version.CURRENT)
+            .put(INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            .build();
+
+        IndexMetadata indexMetadata = metadataCreateIndexService.buildAndValidateTemporaryIndexMetadata(indexSettings, request, 0);
+        threadPool.shutdown();
+        return indexMetadata;
+    }
+
+    private void validateRemoteCustomData(Map<String, String> customData, String expectedKey, String expectedValue) {
+        assertTrue(customData.containsKey(expectedKey));
+        assertEquals(expectedValue, customData.get(expectedKey));
+    }
+
     public void testGetIndexNumberOfRoutingShardsWithNullSourceIndex() {
         Settings indexSettings = Settings.builder()
             .put("index.version.created", Version.CURRENT)
