Skip to content

Commit 5bf522c

Browse files
ltaragiltaragi
authored
Take shallow copy snapshot when remote store migration is in progress (opensearch-project#13309)
Signed-off-by: Lakshya Taragi <lakshya.taragi@gmail.com>
1 parent 42d46a9 commit 5bf522c

File tree

4 files changed

+165
-72
lines changed

4 files changed

+165
-72
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java

+9-72
Original file line numberDiff line numberDiff line change
@@ -8,27 +8,15 @@
88

99
package org.opensearch.remotemigration;
1010

11-
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
1211
import org.opensearch.client.Client;
13-
import org.opensearch.cluster.metadata.IndexMetadata;
1412
import org.opensearch.common.settings.Settings;
1513
import org.opensearch.common.settings.SettingsException;
16-
import org.opensearch.core.rest.RestStatus;
17-
import org.opensearch.index.IndexSettings;
18-
import org.opensearch.indices.replication.common.ReplicationType;
19-
import org.opensearch.snapshots.SnapshotInfo;
20-
import org.opensearch.snapshots.SnapshotState;
2114
import org.opensearch.test.InternalTestCluster;
2215
import org.opensearch.test.OpenSearchIntegTestCase;
2316

2417
import java.nio.file.Path;
2518
import java.util.Optional;
2619

27-
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
28-
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
29-
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
30-
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
31-
import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
3220
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED;
3321
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT;
3422
import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.REMOTE_STORE;
@@ -92,13 +80,7 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix
9280
assertNodeInCluster(remoteNodeName);
9381

9482
logger.info("Create a non remote-backed index");
95-
client.admin()
96-
.indices()
97-
.prepareCreate(TEST_INDEX)
98-
.setSettings(
99-
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
100-
)
101-
.get();
83+
createIndex(TEST_INDEX, 0);
10284

10385
logger.info("Verify that non remote stored backed index is created");
10486
assertNonRemoteStoreBackedIndex(TEST_INDEX);
@@ -115,21 +97,12 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix
11597

11698
logger.info("Create snapshot of non remote stored backed index");
11799

118-
SnapshotInfo snapshotInfo = client().admin()
119-
.cluster()
120-
.prepareCreateSnapshot(snapshotRepoName, snapshotName)
121-
.setIndices(TEST_INDEX)
122-
.setWaitForCompletion(true)
123-
.get()
124-
.getSnapshotInfo();
125-
126-
assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
127-
assertTrue(snapshotInfo.successfulShards() > 0);
128-
assertEquals(0, snapshotInfo.failedShards());
100+
createSnapshot(snapshotRepoName, snapshotName, TEST_INDEX);
129101

130102
logger.info("Restore index from snapshot under NONE direction");
131103
String restoredIndexName1 = TEST_INDEX + "-restored1";
132104
restoreSnapshot(snapshotRepoName, snapshotName, restoredIndexName1);
105+
ensureGreen(restoredIndexName1);
133106

134107
logger.info("Verify that restored index is non remote-backed");
135108
assertNonRemoteStoreBackedIndex(restoredIndexName1);
@@ -138,6 +111,7 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix
138111
setDirection(REMOTE_STORE.direction);
139112
String restoredIndexName2 = TEST_INDEX + "-restored2";
140113
restoreSnapshot(snapshotRepoName, snapshotName, restoredIndexName2);
114+
ensureGreen(restoredIndexName2);
141115

142116
logger.info("Verify that restored index is non remote-backed");
143117
assertRemoteStoreBackedIndex(restoredIndexName2);
@@ -146,10 +120,10 @@ public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMix
146120
// compatibility mode setting test
147121

148122
public void testSwitchToStrictMode() throws Exception {
149-
logger.info(" --> initialize cluster");
123+
logger.info("Initialize cluster");
150124
initializeCluster(false);
151125

152-
logger.info(" --> create a mixed mode cluster");
126+
logger.info("Create a mixed mode cluster");
153127
setClusterMode(MIXED.mode);
154128
addRemote = true;
155129
String remoteNodeName = internalCluster().startNode();
@@ -159,58 +133,21 @@ public void testSwitchToStrictMode() throws Exception {
159133
assertNodeInCluster(remoteNodeName);
160134
assertNodeInCluster(nonRemoteNodeName);
161135

162-
logger.info(" --> attempt switching to strict mode");
136+
logger.info("Attempt switching to strict mode");
163137
SettingsException exception = assertThrows(SettingsException.class, () -> setClusterMode(STRICT.mode));
164138
assertEquals(
165139
"can not switch to STRICT compatibility mode when the cluster contains both remote and non-remote nodes",
166140
exception.getMessage()
167141
);
168142

169-
logger.info(" --> stop remote node so that cluster had only non-remote nodes");
143+
logger.info("Stop remote node so that cluster had only non-remote nodes");
170144
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(remoteNodeName));
171145
ensureStableCluster(2);
172146

173-
logger.info(" --> attempt switching to strict mode");
147+
logger.info("Attempt switching to strict mode");
174148
setClusterMode(STRICT.mode);
175149
}
176150

177-
// restore indices from a snapshot
178-
private void restoreSnapshot(String snapshotRepoName, String snapshotName, String restoredIndexName) {
179-
RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
180-
.cluster()
181-
.prepareRestoreSnapshot(snapshotRepoName, snapshotName)
182-
.setWaitForCompletion(false)
183-
.setIndices(TEST_INDEX)
184-
.setRenamePattern(TEST_INDEX)
185-
.setRenameReplacement(restoredIndexName)
186-
.get();
187-
188-
assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
189-
ensureGreen(restoredIndexName);
190-
}
191-
192-
// verify that the created index is not remote store backed
193-
private void assertNonRemoteStoreBackedIndex(String indexName) {
194-
Settings indexSettings = client.admin().indices().prepareGetIndex().execute().actionGet().getSettings().get(indexName);
195-
assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
196-
assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
197-
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
198-
assertNull(indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
199-
}
200-
201-
// verify that the created index is remote store backed
202-
private void assertRemoteStoreBackedIndex(String indexName) {
203-
Settings indexSettings = client.admin().indices().prepareGetIndex().execute().actionGet().getSettings().get(indexName);
204-
assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
205-
assertEquals("true", indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
206-
assertEquals(REPOSITORY_NAME, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
207-
assertEquals(REPOSITORY_2_NAME, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
208-
assertEquals(
209-
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
210-
INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)
211-
);
212-
}
213-
214151
// bootstrap a cluster
215152
private void initializeCluster(boolean remoteClusterManager) {
216153
addRemote = remoteClusterManager;

‎server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java

+96
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,27 @@
99
package org.opensearch.remotemigration;
1010

1111
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
12+
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
13+
import org.opensearch.cluster.metadata.IndexMetadata;
1214
import org.opensearch.cluster.node.DiscoveryNode;
1315
import org.opensearch.cluster.node.DiscoveryNodes;
1416
import org.opensearch.cluster.routing.IndexShardRoutingTable;
1517
import org.opensearch.cluster.routing.ShardRouting;
1618
import org.opensearch.common.settings.Settings;
19+
import org.opensearch.core.rest.RestStatus;
20+
import org.opensearch.index.IndexSettings;
21+
import org.opensearch.indices.replication.common.ReplicationType;
22+
import org.opensearch.snapshots.SnapshotInfo;
23+
import org.opensearch.snapshots.SnapshotState;
1724

1825
import java.util.Map;
1926
import java.util.Optional;
2027

28+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
29+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
30+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
31+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
32+
import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
2133
import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
2234
import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
2335
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -98,4 +110,88 @@ protected ShardRouting getShardRouting(boolean isPrimary) {
98110
return (isPrimary ? table.primaryShard() : table.replicaShards().get(0));
99111
}
100112

113+
// create a snapshot
114+
public static SnapshotInfo createSnapshot(String snapshotRepoName, String snapshotName, String... indices) {
115+
SnapshotInfo snapshotInfo = internalCluster().client()
116+
.admin()
117+
.cluster()
118+
.prepareCreateSnapshot(snapshotRepoName, snapshotName)
119+
.setIndices(indices)
120+
.setWaitForCompletion(true)
121+
.get()
122+
.getSnapshotInfo();
123+
124+
assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
125+
assertTrue(snapshotInfo.successfulShards() > 0);
126+
assertEquals(0, snapshotInfo.failedShards());
127+
return snapshotInfo;
128+
}
129+
130+
// create new index
131+
public static void createIndex(String indexName, int replicaCount) {
132+
assertAcked(
133+
internalCluster().client()
134+
.admin()
135+
.indices()
136+
.prepareCreate(indexName)
137+
.setSettings(
138+
Settings.builder()
139+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
140+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicaCount)
141+
.build()
142+
)
143+
.get()
144+
);
145+
}
146+
147+
// restore indices from a snapshot
148+
public static RestoreSnapshotResponse restoreSnapshot(String snapshotRepoName, String snapshotName, String restoredIndexName) {
149+
RestoreSnapshotResponse restoreSnapshotResponse = internalCluster().client()
150+
.admin()
151+
.cluster()
152+
.prepareRestoreSnapshot(snapshotRepoName, snapshotName)
153+
.setWaitForCompletion(false)
154+
.setIndices(TEST_INDEX)
155+
.setRenamePattern(TEST_INDEX)
156+
.setRenameReplacement(restoredIndexName)
157+
.get();
158+
assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
159+
return restoreSnapshotResponse;
160+
}
161+
162+
// verify that the created index is not remote store backed
163+
public static void assertNonRemoteStoreBackedIndex(String indexName) {
164+
Settings indexSettings = internalCluster().client()
165+
.admin()
166+
.indices()
167+
.prepareGetIndex()
168+
.execute()
169+
.actionGet()
170+
.getSettings()
171+
.get(indexName);
172+
assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
173+
assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
174+
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
175+
assertNull(indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
176+
}
177+
178+
// verify that the created index is remote store backed
179+
public static void assertRemoteStoreBackedIndex(String indexName) {
180+
Settings indexSettings = internalCluster().client()
181+
.admin()
182+
.indices()
183+
.prepareGetIndex()
184+
.execute()
185+
.actionGet()
186+
.getSettings()
187+
.get(indexName);
188+
assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
189+
assertEquals("true", indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
190+
assertEquals(REPOSITORY_NAME, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
191+
assertEquals(REPOSITORY_2_NAME, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
192+
assertEquals(
193+
IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
194+
INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)
195+
);
196+
}
101197
}

‎server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java

+51
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,11 @@
1313
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
1414
import org.opensearch.client.Client;
1515
import org.opensearch.common.settings.Settings;
16+
import org.opensearch.repositories.blobstore.BlobStoreRepository;
17+
import org.opensearch.snapshots.SnapshotInfo;
1618
import org.opensearch.test.OpenSearchIntegTestCase;
1719

20+
import java.nio.file.Path;
1821
import java.util.List;
1922

2023
import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
@@ -70,4 +73,52 @@ public void testMigrationDirections() {
7073
updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "random"));
7174
assertThrows(IllegalArgumentException.class, () -> client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
7275
}
76+
77+
public void testNoShallowSnapshotInMixedMode() throws Exception {
78+
logger.info("Initialize remote cluster");
79+
addRemote = true;
80+
internalCluster().setBootstrapClusterManagerNodeIndex(0);
81+
List<String> cmNodes = internalCluster().startNodes(1);
82+
Client client = internalCluster().client(cmNodes.get(0));
83+
84+
logger.info("Add remote node");
85+
internalCluster().startNode();
86+
internalCluster().validateClusterFormed();
87+
88+
logger.info("Create remote backed index");
89+
RemoteStoreMigrationShardAllocationBaseTestCase.createIndex("test", 0);
90+
RemoteStoreMigrationShardAllocationBaseTestCase.assertRemoteStoreBackedIndex("test");
91+
92+
logger.info("Create shallow snapshot setting enabled repo");
93+
String shallowSnapshotRepoName = "shallow-snapshot-repo-name";
94+
Path shallowSnapshotRepoPath = randomRepoPath();
95+
assertAcked(
96+
clusterAdmin().preparePutRepository(shallowSnapshotRepoName)
97+
.setType("fs")
98+
.setSettings(
99+
Settings.builder()
100+
.put("location", shallowSnapshotRepoPath)
101+
.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE)
102+
)
103+
);
104+
105+
logger.info("Verify shallow snapshot creation");
106+
final String snapshot1 = "snapshot1";
107+
SnapshotInfo snapshotInfo1 = RemoteStoreMigrationShardAllocationBaseTestCase.createSnapshot(
108+
shallowSnapshotRepoName,
109+
snapshot1,
110+
"test"
111+
);
112+
assertEquals(snapshotInfo1.isRemoteStoreIndexShallowCopyEnabled(), true);
113+
114+
logger.info("Set MIXED compatibility mode");
115+
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
116+
updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed"));
117+
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
118+
119+
logger.info("Verify that new snapshot is not shallow");
120+
final String snapshot2 = "snapshot2";
121+
SnapshotInfo snapshotInfo2 = RemoteStoreMigrationShardAllocationBaseTestCase.createSnapshot(shallowSnapshotRepoName, snapshot2);
122+
assertEquals(snapshotInfo2.isRemoteStoreIndexShallowCopyEnabled(), false);
123+
}
73124
}

‎server/src/main/java/org/opensearch/snapshots/SnapshotsService.java

+9
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,8 @@
131131
import static java.util.Collections.emptySet;
132132
import static java.util.Collections.unmodifiableList;
133133
import static org.opensearch.cluster.SnapshotsInProgress.completed;
134+
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
135+
import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
134136
import static org.opensearch.repositories.blobstore.BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY;
135137
import static org.opensearch.snapshots.SnapshotUtils.validateSnapshotsBackingAnyIndex;
136138

@@ -343,6 +345,13 @@ public ClusterState execute(ClusterState currentState) {
343345
}
344346

345347
boolean remoteStoreIndexShallowCopy = REMOTE_STORE_INDEX_SHALLOW_COPY.get(repository.getMetadata().settings());
348+
logger.debug("remote_store_index_shallow_copy setting is set as [{}]", remoteStoreIndexShallowCopy);
349+
if (remoteStoreIndexShallowCopy
350+
&& clusterService.getClusterSettings().get(REMOTE_STORE_COMPATIBILITY_MODE_SETTING).equals(CompatibilityMode.MIXED)) {
351+
// don't allow shallow snapshots if compatibility mode is not strict
352+
logger.warn("Shallow snapshots are not supported during migration. Falling back to full snapshot.");
353+
remoteStoreIndexShallowCopy = false;
354+
}
346355
newEntry = SnapshotsInProgress.startedEntry(
347356
new Snapshot(repositoryName, snapshotId),
348357
request.includeGlobalState(),

0 commit comments

Comments
 (0)