Reject Resize index requests. (i.e, split, shrink and…

CHANGELOG.md

@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Tracing for deep search path ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12103))
 - Add explicit dependency to validatePom and generatePom tasks ([#12807](https://github.com/opensearch-project/OpenSearch/pull/12807))
 - Replace configureEach with all for publication iteration ([#12876](https://github.com/opensearch-project/OpenSearch/pull/12876))
+- Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0

server/src/internalClusterTest/java/org/opensearch/remotemigration/ResizeIndexMigrationTestCase.java

@@ -0,0 +1,218 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotemigration;
+
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.indices.shrink.ResizeType;
+import org.opensearch.action.support.ActiveShardCount;
+import org.opensearch.client.Client;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.List;
+
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+public class ResizeIndexMigrationTestCase extends MigrationBaseTestCase {
+    private static final String TEST_INDEX = "test_index";
+    private final static String REMOTE_STORE_DIRECTION = "remote_store";
+    private final static String DOC_REP_DIRECTION = "docrep";
+    private final static String NONE_DIRECTION = "none";
+    private final static String STRICT_MODE = "strict";
+    private final static String MIXED_MODE = "mixed";
+
+    /*
+    * This test will verify the resize request failure, when cluster mode is mixed
+    * and index is on DocRep node, and migration to remote store is in progress.
+    * */
+    public void testFailResizeIndexWhileDocRepToRemoteStoreMigration() throws Exception {
+
+        internalCluster().setBootstrapClusterManagerNodeIndex(0);
+        List<String> cmNodes = internalCluster().startNodes(1);
+        Client client = internalCluster().client(cmNodes.get(0));
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), MIXED_MODE));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        // Adding a non remote and a remote node
+        addRemote = false;
+        String nonRemoteNodeName = internalCluster().startNode();
+
+        addRemote = true;
+        String remoteNodeName = internalCluster().startNode();
+
+        logger.info("-->Create index on non-remote node and SETTING_REMOTE_STORE_ENABLED is false. Resize should not happen");
+        Settings.Builder builder = Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT);
+        client.admin()
+            .indices()
+            .prepareCreate(TEST_INDEX)
+            .setSettings(
+                builder.put("index.number_of_shards", 10)
+                    .put("index.number_of_replicas", 0)
+                    .put("index.routing.allocation.include._name", nonRemoteNodeName)
+                    .put("index.routing.allocation.exclude._name", remoteNodeName)
+            )
+            .setWaitForActiveShards(ActiveShardCount.ALL)
+            .execute()
+            .actionGet();
+
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), REMOTE_STORE_DIRECTION));
+        assertAcked(client.admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        ResizeType resizeType;
+        int resizeShardsNum;
+        String cause;
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                resizeType = ResizeType.SHRINK;
+                resizeShardsNum = 5;
+                cause = "shrink_index";
+                break;
+            case 1:
+                resizeType = ResizeType.SPLIT;
+                resizeShardsNum = 20;
+                cause = "split_index";
+                break;
+            default:
+                resizeType = ResizeType.CLONE;
+                resizeShardsNum = 10;
+                cause = "clone_index";
+        }
+
+        client.admin()
+            .indices()
+            .prepareUpdateSettings(TEST_INDEX)
+            .setSettings(Settings.builder().put("index.blocks.write", true))
+            .execute()
+            .actionGet();
+
+        ensureGreen(TEST_INDEX);
+
+        Settings.Builder resizeSettingsBuilder = Settings.builder()
+            .put("index.number_of_replicas", 0)
+            .put("index.number_of_shards", resizeShardsNum)
+            .putNull("index.blocks.write");
+
+        IllegalStateException ex = expectThrows(
+            IllegalStateException.class,
+            () -> client().admin()
+                .indices()
+                .prepareResizeIndex(TEST_INDEX, "first_split")
+                .setResizeType(resizeType)
+                .setSettings(resizeSettingsBuilder.build())
+                .get()
+        );
+        assertEquals(
+            ex.getMessage(),
+            "index Resizing for type ["
+                + resizeType
+                + "] is not allowed as Cluster mode is [Mixed]"
+                + " and migration direction is [REMOTE_STORE]"
+                + " and index's SETTING_REMOTE_STORE_ENABLED = "
+                + "false"
+        );
+    }
+
+    /*
+     * This test will verify the resize request failure, when cluster mode is mixed
+     * and index is on Remote Store node, and migration to DocRep node is in progress.
+     * */
+    public void testFailResizeIndexWhileRemoteStoreToDocRepMigration() throws Exception {
+
+        addRemote = true;
+        internalCluster().setBootstrapClusterManagerNodeIndex(0);
+        List<String> cmNodes = internalCluster().startNodes(1);
+        Client client = internalCluster().client(cmNodes.get(0));
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), MIXED_MODE));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        // Adding a non remote and a remote node
+        String remoteNodeName = internalCluster().startNode();
+
+        addRemote = false;
+        String nonRemoteNodeName = internalCluster().startNode();
+
+        logger.info("-->Create index on remote node and SETTING_REMOTE_STORE_ENABLED is true. Resize should not happen");
+        Settings.Builder builder = Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT);
+        client.admin()
+            .indices()
+            .prepareCreate(TEST_INDEX)
+            .setSettings(
+                builder.put("index.number_of_shards", 10)
+                    .put("index.number_of_replicas", 0)
+                    .put("index.routing.allocation.include._name", remoteNodeName)
+                    .put("index.routing.allocation.exclude._name", nonRemoteNodeName)
+            )
+            .setWaitForActiveShards(ActiveShardCount.ALL)
+            .execute()
+            .actionGet();
+
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), DOC_REP_DIRECTION));
+        assertAcked(client.admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        ResizeType resizeType;
+        int resizeShardsNum;
+        String cause;
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                resizeType = ResizeType.SHRINK;
+                resizeShardsNum = 5;
+                cause = "shrink_index";
+                break;
+            case 1:
+                resizeType = ResizeType.SPLIT;
+                resizeShardsNum = 20;
+                cause = "split_index";
+                break;
+            default:
+                resizeType = ResizeType.CLONE;
+                resizeShardsNum = 10;
+                cause = "clone_index";
+        }
+
+        client.admin()
+            .indices()
+            .prepareUpdateSettings(TEST_INDEX)
+            .setSettings(Settings.builder().put("index.blocks.write", true))
+            .execute()
+            .actionGet();
+
+        ensureGreen(TEST_INDEX);
+
+        Settings.Builder resizeSettingsBuilder = Settings.builder()
+            .put("index.number_of_replicas", 0)
+            .put("index.number_of_shards", resizeShardsNum)
+            .putNull("index.blocks.write");
+
+        IllegalStateException ex = expectThrows(
+            IllegalStateException.class,
+            () -> client().admin()
+                .indices()
+                .prepareResizeIndex(TEST_INDEX, "first_split")
+                .setResizeType(resizeType)
+                .setSettings(resizeSettingsBuilder.build())
+                .get()
+        );
+        assertEquals(
+            ex.getMessage(),
+            "index Resizing for type ["
+                + resizeType
+                + "] is not allowed as Cluster mode is [Mixed]"
+                + " and migration direction is [DOCREP]"
+                + " and index's SETTING_REMOTE_STORE_ENABLED = "
+                + "true"
+        );
+    }
+}

server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java

@@ -48,6 +48,7 @@
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.Inject;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -57,6 +58,7 @@
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.shard.DocsStats;
 import org.opensearch.index.store.StoreStats;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
@@ -67,6 +69,7 @@
 import java.util.function.IntFunction;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
 
 /**
  * Main class to initiate resizing (shrink / split) an index into a new index
@@ -140,8 +143,8 @@
         // there is no need to fetch docs stats for split but we keep it simple and do it anyway for simplicity of the code
         final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex());
         final String targetIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getTargetIndexRequest().index());
-
         IndexMetadata indexMetadata = state.metadata().index(sourceIndex);
+        ClusterSettings clusterSettings = clusterService.getClusterSettings();
         if (resizeRequest.getResizeType().equals(ResizeType.SHRINK)
             && state.metadata().isSegmentReplicationEnabled(sourceIndex)
             && indexMetadata != null
@@ -161,7 +164,7 @@
                             CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(resizeRequest, state, i -> {
                                 IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i);
                                 return shard == null ? null : shard.getPrimary().getDocs();
-                            }, indicesStatsResponse.getPrimaries().store, sourceIndex, targetIndex);
+                            }, indicesStatsResponse.getPrimaries().store, clusterSettings, sourceIndex, targetIndex);
 
                             if (indicesStatsResponse.getIndex(sourceIndex)
                                 .getTotal()
@@ -200,7 +203,7 @@
                     CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(resizeRequest, state, i -> {
                         IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i);
                         return shard == null ? null : shard.getPrimary().getDocs();
-                    }, indicesStatsResponse.getPrimaries().store, sourceIndex, targetIndex);
+                    }, indicesStatsResponse.getPrimaries().store, clusterSettings, sourceIndex, targetIndex);
                     createIndexService.createIndex(
                         updateRequest,
                         ActionListener.map(
@@ -223,6 +226,7 @@
         final ClusterState state,
         final IntFunction<DocsStats> perShardDocStats,
         final StoreStats primaryShardsStoreStats,
+        final ClusterSettings clusterSettings,
         String sourceIndexName,
         String targetIndexName
     ) {
@@ -231,6 +235,7 @@
         if (metadata == null) {
             throw new IndexNotFoundException(sourceIndexName);
         }
+        validateClusterModeSettings(resizeRequest.getResizeType(), metadata, clusterSettings);
         final Settings.Builder targetIndexSettingsBuilder = Settings.builder()
             .put(targetIndex.settings())
             .normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
@@ -368,4 +373,43 @@
     protected String getClusterManagerActionName(DiscoveryNode node) {
         return super.getClusterManagerActionName(node);
     }
+
+    /**
+     * Reject resize request if cluster mode is [Mixed] and migration direction is [RemoteStore] and index is not on
+     * REMOTE_STORE_ENABLED node or [DocRep] and index is on REMOTE_STORE_ENABLED node.
+     * @param type resize type
+     * @param sourceIndexMetadata source index's metadata
+     * @param clusterSettings cluster settings
+     * @throws IllegalStateException if cluster mode is [Mixed] and migration direction is [RemoteStore] or [DocRep] and
+     *                               index's SETTING_REMOTE_STORE_ENABLED is not equal to the migration direction's value.
+     *                               For example, if migration direction is [RemoteStore] and index's SETTING_REMOTE_STORE_ENABLED
+     *                               is false, then throw IllegalStateException. If migration direction is [DocRep] and
+     *                               index's SETTING_REMOTE_STORE_ENABLED is true, then throw IllegalStateException.
+     */
+    private static void validateClusterModeSettings(
+        final ResizeType type,
+        IndexMetadata sourceIndexMetadata,
+        ClusterSettings clusterSettings
+    ) {
+        if (clusterSettings.get(RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING)
+            .equals(RemoteStoreNodeService.CompatibilityMode.MIXED)) {
+            boolean isRemoteStoreEnabled = sourceIndexMetadata.getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false);
+            if ((clusterSettings.get(RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING)
+                .equals(RemoteStoreNodeService.Direction.REMOTE_STORE)
+                && isRemoteStoreEnabled == false)
+                || (clusterSettings.get(RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING).equals(RemoteStoreNodeService.Direction.DOCREP)
+                    && isRemoteStoreEnabled == true)) {
+                throw new IllegalStateException(
+                    "index Resizing for type ["
+                        + type
+                        + "] is not allowed as Cluster mode is [Mixed]"
+                        + " and migration direction is ["
+                        + clusterSettings.get(RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING)
+                        + "]"
+                        + " and index's SETTING_REMOTE_STORE_ENABLED = "
+                        + isRemoteStoreEnabled
+                );
+            }
+        }
+    }
 }