Skip to content

Commit 9ab31f5

Browse files
astute-decipherastute-decipher
authored
Reject Resize index requests during remote store migration (opensearch-project#12686)
Signed-off-by: Shubh Sahu <shubhvs@amazon.com>
1 parent 17641f6 commit 9ab31f5

File tree

4 files changed

+418
-5
lines changed

4 files changed

+418
-5
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
113113
- Detect breaking changes on pull requests ([#9044](https://github.com/opensearch-project/OpenSearch/pull/9044))
114114
- Add cluster primary balance contraint for rebalancing with buffer ([#12656](https://github.com/opensearch-project/OpenSearch/pull/12656))
115115
- [Remote Store] Make translog transfer timeout configurable ([#12704](https://github.com/opensearch-project/OpenSearch/pull/12704))
116+
- Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686))
116117

117118
### Dependencies
118119
- Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))

‎server/src/internalClusterTest/java/org/opensearch/remotemigration/ResizeIndexMigrationTestCase.java

+208
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.remotemigration;
10+
11+
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
12+
import org.opensearch.action.admin.indices.shrink.ResizeType;
13+
import org.opensearch.action.support.ActiveShardCount;
14+
import org.opensearch.client.Client;
15+
import org.opensearch.common.settings.Settings;
16+
import org.opensearch.indices.replication.common.ReplicationType;
17+
import org.opensearch.test.OpenSearchIntegTestCase;
18+
19+
import java.util.List;
20+
21+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
22+
import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
23+
import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
24+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
25+
26+
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
27+
public class ResizeIndexMigrationTestCase extends MigrationBaseTestCase {
28+
private static final String TEST_INDEX = "test_index";
29+
private final static String REMOTE_STORE_DIRECTION = "remote_store";
30+
private final static String DOC_REP_DIRECTION = "docrep";
31+
private final static String NONE_DIRECTION = "none";
32+
private final static String STRICT_MODE = "strict";
33+
private final static String MIXED_MODE = "mixed";
34+
35+
/*
36+
* This test will verify the resize request failure, when cluster mode is mixed
37+
* and index is on DocRep node, and migration to remote store is in progress.
38+
* */
39+
public void testFailResizeIndexWhileDocRepToRemoteStoreMigration() throws Exception {
40+
41+
internalCluster().setBootstrapClusterManagerNodeIndex(0);
42+
List<String> cmNodes = internalCluster().startNodes(1);
43+
Client client = internalCluster().client(cmNodes.get(0));
44+
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
45+
updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), MIXED_MODE));
46+
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
47+
48+
// Adding a non remote and a remote node
49+
addRemote = false;
50+
String nonRemoteNodeName = internalCluster().startNode();
51+
52+
addRemote = true;
53+
String remoteNodeName = internalCluster().startNode();
54+
55+
logger.info("-->Create index on non-remote node and SETTING_REMOTE_STORE_ENABLED is false. Resize should not happen");
56+
Settings.Builder builder = Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT);
57+
client.admin()
58+
.indices()
59+
.prepareCreate(TEST_INDEX)
60+
.setSettings(
61+
builder.put("index.number_of_shards", 10)
62+
.put("index.number_of_replicas", 0)
63+
.put("index.routing.allocation.include._name", nonRemoteNodeName)
64+
.put("index.routing.allocation.exclude._name", remoteNodeName)
65+
)
66+
.setWaitForActiveShards(ActiveShardCount.ALL)
67+
.execute()
68+
.actionGet();
69+
70+
updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), REMOTE_STORE_DIRECTION));
71+
assertAcked(client.admin().cluster().updateSettings(updateSettingsRequest).actionGet());
72+
73+
ResizeType resizeType;
74+
int resizeShardsNum;
75+
String cause;
76+
switch (randomIntBetween(0, 2)) {
77+
case 0:
78+
resizeType = ResizeType.SHRINK;
79+
resizeShardsNum = 5;
80+
cause = "shrink_index";
81+
break;
82+
case 1:
83+
resizeType = ResizeType.SPLIT;
84+
resizeShardsNum = 20;
85+
cause = "split_index";
86+
break;
87+
default:
88+
resizeType = ResizeType.CLONE;
89+
resizeShardsNum = 10;
90+
cause = "clone_index";
91+
}
92+
93+
client.admin()
94+
.indices()
95+
.prepareUpdateSettings(TEST_INDEX)
96+
.setSettings(Settings.builder().put("index.blocks.write", true))
97+
.execute()
98+
.actionGet();
99+
100+
ensureGreen(TEST_INDEX);
101+
102+
Settings.Builder resizeSettingsBuilder = Settings.builder()
103+
.put("index.number_of_replicas", 0)
104+
.put("index.number_of_shards", resizeShardsNum)
105+
.putNull("index.blocks.write");
106+
107+
IllegalStateException ex = expectThrows(
108+
IllegalStateException.class,
109+
() -> client().admin()
110+
.indices()
111+
.prepareResizeIndex(TEST_INDEX, "first_split")
112+
.setResizeType(resizeType)
113+
.setSettings(resizeSettingsBuilder.build())
114+
.get()
115+
);
116+
assertEquals(
117+
ex.getMessage(),
118+
"Index " + resizeType + " is not allowed as remote migration mode is mixed" + " and index is remote store disabled"
119+
);
120+
}
121+
122+
/*
123+
* This test will verify the resize request failure, when cluster mode is mixed
124+
* and index is on Remote Store node, and migration to DocRep node is in progress.
125+
* */
126+
public void testFailResizeIndexWhileRemoteStoreToDocRepMigration() throws Exception {
127+
128+
addRemote = true;
129+
internalCluster().setBootstrapClusterManagerNodeIndex(0);
130+
List<String> cmNodes = internalCluster().startNodes(1);
131+
Client client = internalCluster().client(cmNodes.get(0));
132+
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
133+
updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), MIXED_MODE));
134+
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
135+
136+
// Adding a non remote and a remote node
137+
String remoteNodeName = internalCluster().startNode();
138+
139+
addRemote = false;
140+
String nonRemoteNodeName = internalCluster().startNode();
141+
142+
logger.info("-->Create index on remote node and SETTING_REMOTE_STORE_ENABLED is true. Resize should not happen");
143+
Settings.Builder builder = Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT);
144+
client.admin()
145+
.indices()
146+
.prepareCreate(TEST_INDEX)
147+
.setSettings(
148+
builder.put("index.number_of_shards", 10)
149+
.put("index.number_of_replicas", 0)
150+
.put("index.routing.allocation.include._name", remoteNodeName)
151+
.put("index.routing.allocation.exclude._name", nonRemoteNodeName)
152+
)
153+
.setWaitForActiveShards(ActiveShardCount.ALL)
154+
.execute()
155+
.actionGet();
156+
157+
updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), DOC_REP_DIRECTION));
158+
assertAcked(client.admin().cluster().updateSettings(updateSettingsRequest).actionGet());
159+
160+
ResizeType resizeType;
161+
int resizeShardsNum;
162+
String cause;
163+
switch (randomIntBetween(0, 2)) {
164+
case 0:
165+
resizeType = ResizeType.SHRINK;
166+
resizeShardsNum = 5;
167+
cause = "shrink_index";
168+
break;
169+
case 1:
170+
resizeType = ResizeType.SPLIT;
171+
resizeShardsNum = 20;
172+
cause = "split_index";
173+
break;
174+
default:
175+
resizeType = ResizeType.CLONE;
176+
resizeShardsNum = 10;
177+
cause = "clone_index";
178+
}
179+
180+
client.admin()
181+
.indices()
182+
.prepareUpdateSettings(TEST_INDEX)
183+
.setSettings(Settings.builder().put("index.blocks.write", true))
184+
.execute()
185+
.actionGet();
186+
187+
ensureGreen(TEST_INDEX);
188+
189+
Settings.Builder resizeSettingsBuilder = Settings.builder()
190+
.put("index.number_of_replicas", 0)
191+
.put("index.number_of_shards", resizeShardsNum)
192+
.putNull("index.blocks.write");
193+
194+
IllegalStateException ex = expectThrows(
195+
IllegalStateException.class,
196+
() -> client().admin()
197+
.indices()
198+
.prepareResizeIndex(TEST_INDEX, "first_split")
199+
.setResizeType(resizeType)
200+
.setSettings(resizeSettingsBuilder.build())
201+
.get()
202+
);
203+
assertEquals(
204+
ex.getMessage(),
205+
"Index " + resizeType + " is not allowed as remote migration mode is mixed" + " and index is remote store enabled"
206+
);
207+
}
208+
}

‎server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java

+45-3
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.opensearch.cluster.node.DiscoveryNode;
4949
import org.opensearch.cluster.service.ClusterService;
5050
import org.opensearch.common.inject.Inject;
51+
import org.opensearch.common.settings.ClusterSettings;
5152
import org.opensearch.common.settings.Settings;
5253
import org.opensearch.core.action.ActionListener;
5354
import org.opensearch.core.common.io.stream.StreamInput;
@@ -57,6 +58,9 @@
5758
import org.opensearch.index.IndexSettings;
5859
import org.opensearch.index.shard.DocsStats;
5960
import org.opensearch.index.store.StoreStats;
61+
import org.opensearch.node.remotestore.RemoteStoreNodeService;
62+
import org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
63+
import org.opensearch.node.remotestore.RemoteStoreNodeService.Direction;
6064
import org.opensearch.threadpool.ThreadPool;
6165
import org.opensearch.transport.TransportService;
6266

@@ -67,6 +71,7 @@
6771
import java.util.function.IntFunction;
6872

6973
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
74+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
7075

7176
/**
7277
* Main class to initiate resizing (shrink / split) an index into a new index
@@ -140,8 +145,8 @@ protected void clusterManagerOperation(
140145
// there is no need to fetch docs stats for split but we keep it simple and do it anyway for simplicity of the code
141146
final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex());
142147
final String targetIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getTargetIndexRequest().index());
143-
144148
IndexMetadata indexMetadata = state.metadata().index(sourceIndex);
149+
ClusterSettings clusterSettings = clusterService.getClusterSettings();
145150
if (resizeRequest.getResizeType().equals(ResizeType.SHRINK)
146151
&& state.metadata().isSegmentReplicationEnabled(sourceIndex)
147152
&& indexMetadata != null
@@ -161,7 +166,7 @@ protected void clusterManagerOperation(
161166
CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(resizeRequest, state, i -> {
162167
IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i);
163168
return shard == null ? null : shard.getPrimary().getDocs();
164-
}, indicesStatsResponse.getPrimaries().store, sourceIndex, targetIndex);
169+
}, indicesStatsResponse.getPrimaries().store, clusterSettings, sourceIndex, targetIndex);
165170

166171
if (indicesStatsResponse.getIndex(sourceIndex)
167172
.getTotal()
@@ -200,7 +205,7 @@ protected void clusterManagerOperation(
200205
CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(resizeRequest, state, i -> {
201206
IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i);
202207
return shard == null ? null : shard.getPrimary().getDocs();
203-
}, indicesStatsResponse.getPrimaries().store, sourceIndex, targetIndex);
208+
}, indicesStatsResponse.getPrimaries().store, clusterSettings, sourceIndex, targetIndex);
204209
createIndexService.createIndex(
205210
updateRequest,
206211
ActionListener.map(
@@ -223,6 +228,7 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(
223228
final ClusterState state,
224229
final IntFunction<DocsStats> perShardDocStats,
225230
final StoreStats primaryShardsStoreStats,
231+
final ClusterSettings clusterSettings,
226232
String sourceIndexName,
227233
String targetIndexName
228234
) {
@@ -231,6 +237,7 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(
231237
if (metadata == null) {
232238
throw new IndexNotFoundException(sourceIndexName);
233239
}
240+
validateRemoteMigrationModeSettings(resizeRequest.getResizeType(), metadata, clusterSettings);
234241
final Settings.Builder targetIndexSettingsBuilder = Settings.builder()
235242
.put(targetIndex.settings())
236243
.normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
@@ -368,4 +375,39 @@ protected static int calculateTargetIndexShardsNum(
368375
protected String getClusterManagerActionName(DiscoveryNode node) {
369376
return super.getClusterManagerActionName(node);
370377
}
378+
379+
/**
380+
* Reject resize request if cluster mode is [Mixed] and migration direction is [RemoteStore] and index is not on
381+
* REMOTE_STORE_ENABLED node or [DocRep] and index is on REMOTE_STORE_ENABLED node.
382+
* @param type resize type
383+
* @param sourceIndexMetadata source index's metadata
384+
* @param clusterSettings cluster settings
385+
* @throws IllegalStateException if cluster mode is [Mixed] and migration direction is [RemoteStore] or [DocRep] and
386+
* index's SETTING_REMOTE_STORE_ENABLED is not equal to the migration direction's value.
387+
* For example, if migration direction is [RemoteStore] and index's SETTING_REMOTE_STORE_ENABLED
388+
* is false, then throw IllegalStateException. If migration direction is [DocRep] and
389+
* index's SETTING_REMOTE_STORE_ENABLED is true, then throw IllegalStateException.
390+
*/
391+
private static void validateRemoteMigrationModeSettings(
392+
final ResizeType type,
393+
IndexMetadata sourceIndexMetadata,
394+
ClusterSettings clusterSettings
395+
) {
396+
CompatibilityMode compatibilityMode = clusterSettings.get(RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING);
397+
if (compatibilityMode == CompatibilityMode.MIXED) {
398+
boolean isRemoteStoreEnabled = sourceIndexMetadata.getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false);
399+
Direction migrationDirection = clusterSettings.get(RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING);
400+
boolean invalidConfiguration = (migrationDirection == Direction.REMOTE_STORE && isRemoteStoreEnabled == false)
401+
|| (migrationDirection == Direction.DOCREP && isRemoteStoreEnabled);
402+
if (invalidConfiguration) {
403+
throw new IllegalStateException(
404+
"Index "
405+
+ type
406+
+ " is not allowed as remote migration mode is mixed"
407+
+ " and index is remote store "
408+
+ (isRemoteStoreEnabled ? "enabled" : "disabled")
409+
);
410+
}
411+
}
412+
}
371413
}

0 commit comments

Comments
 (0)