Skip to content

Commit 4fc3a02

Browse files
harishbhakuniharishbhakuni
authored
[Snapshot Interop] Add changes for overriding remote store and replic… (opensearch-project#11868)
Signed-off-by: Harish Bhakuni <hbhakuni@amazon.com>
1 parent 74232c7 commit 4fc3a02

File tree

5 files changed

+229
-164
lines changed

5 files changed

+229
-164
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
1717
- [Remote Store] Make translog transfer timeout configurable ([#12704](https://github.com/opensearch-project/OpenSearch/pull/12704))
1818
- Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686))
1919
- Add support for more than one protocol for transport ([#12967](https://github.com/opensearch-project/OpenSearch/pull/12967))
20+
- Add changes for overriding remote store and replication settings during snapshot restore. ([#11868](https://github.com/opensearch-project/OpenSearch/pull/11868))
2021

2122
### Dependencies
2223
- Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

+96-137
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,6 @@
1212
import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
1313
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
1414
import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
15-
import org.opensearch.action.admin.indices.get.GetIndexRequest;
16-
import org.opensearch.action.admin.indices.get.GetIndexResponse;
1715
import org.opensearch.action.delete.DeleteResponse;
1816
import org.opensearch.action.support.PlainActionFuture;
1917
import org.opensearch.client.Client;
@@ -36,6 +34,7 @@
3634
import org.opensearch.indices.replication.common.ReplicationType;
3735
import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
3836
import org.opensearch.snapshots.SnapshotInfo;
37+
import org.opensearch.snapshots.SnapshotRestoreException;
3938
import org.opensearch.snapshots.SnapshotState;
4039
import org.opensearch.test.InternalTestCluster;
4140
import org.opensearch.test.OpenSearchIntegTestCase;
@@ -55,7 +54,6 @@
5554
import java.util.stream.Collectors;
5655
import java.util.stream.Stream;
5756

58-
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
5957
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
6058
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
6159
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
@@ -118,7 +116,7 @@ private void assertDocsPresentInIndex(Client client, String indexName, int numOf
118116
}
119117
}
120118

121-
public void testRestoreOperationsShallowCopyEnabled() throws IOException, ExecutionException, InterruptedException {
119+
public void testRestoreOperationsShallowCopyEnabled() throws Exception {
122120
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
123121
String primary = internalCluster().startDataOnlyNode();
124122
String indexName1 = "testindex1";
@@ -129,8 +127,6 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut
129127
Path absolutePath1 = randomRepoPath().toAbsolutePath();
130128
logger.info("Snapshot Path [{}]", absolutePath1);
131129
String restoredIndexName1 = indexName1 + "-restored";
132-
String restoredIndexName1Seg = indexName1 + "-restored-seg";
133-
String restoredIndexName1Doc = indexName1 + "-restored-doc";
134130
String restoredIndexName2 = indexName2 + "-restored";
135131

136132
createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));
@@ -212,60 +208,6 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut
212208
indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2);
213209
ensureGreen(restoredIndexName1);
214210
assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2);
215-
216-
// restore index as seg rep enabled with remote store and remote translog disabled
217-
RestoreSnapshotResponse restoreSnapshotResponse3 = client.admin()
218-
.cluster()
219-
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
220-
.setWaitForCompletion(false)
221-
.setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED)
222-
.setIndices(indexName1)
223-
.setRenamePattern(indexName1)
224-
.setRenameReplacement(restoredIndexName1Seg)
225-
.get();
226-
assertEquals(restoreSnapshotResponse3.status(), RestStatus.ACCEPTED);
227-
ensureGreen(restoredIndexName1Seg);
228-
229-
GetIndexResponse getIndexResponse = client.admin()
230-
.indices()
231-
.getIndex(new GetIndexRequest().indices(restoredIndexName1Seg).includeDefaults(true))
232-
.get();
233-
indexSettings = getIndexResponse.settings().get(restoredIndexName1Seg);
234-
assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
235-
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null));
236-
assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE));
237-
assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1);
238-
// indexing some new docs and validating
239-
indexDocuments(client, restoredIndexName1Seg, numDocsInIndex1, numDocsInIndex1 + 2);
240-
ensureGreen(restoredIndexName1Seg);
241-
assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1 + 2);
242-
243-
// restore index as doc rep based from shallow copy snapshot
244-
RestoreSnapshotResponse restoreSnapshotResponse4 = client.admin()
245-
.cluster()
246-
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
247-
.setWaitForCompletion(false)
248-
.setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, IndexMetadata.SETTING_REPLICATION_TYPE)
249-
.setIndices(indexName1)
250-
.setRenamePattern(indexName1)
251-
.setRenameReplacement(restoredIndexName1Doc)
252-
.get();
253-
assertEquals(restoreSnapshotResponse4.status(), RestStatus.ACCEPTED);
254-
ensureGreen(restoredIndexName1Doc);
255-
256-
getIndexResponse = client.admin()
257-
.indices()
258-
.getIndex(new GetIndexRequest().indices(restoredIndexName1Doc).includeDefaults(true))
259-
.get();
260-
indexSettings = getIndexResponse.settings().get(restoredIndexName1Doc);
261-
assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
262-
assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null));
263-
assertNull(indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE));
264-
assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1);
265-
// indexing some new docs and validating
266-
indexDocuments(client, restoredIndexName1Doc, numDocsInIndex1, numDocsInIndex1 + 2);
267-
ensureGreen(restoredIndexName1Doc);
268-
assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1 + 2);
269211
}
270212

271213
/**
@@ -579,83 +521,6 @@ protected IndexShard getIndexShard(String node, String indexName) {
579521
return shardId.map(indexService::getShard).orElse(null);
580522
}
581523

582-
public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException {
583-
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
584-
String primary = internalCluster().startDataOnlyNode();
585-
String indexName1 = "testindex1";
586-
String indexName2 = "testindex2";
587-
String snapshotRepoName = "test-restore-snapshot-repo";
588-
String remoteStoreRepo2Name = "test-rs-repo-2" + TEST_REMOTE_STORE_REPO_SUFFIX;
589-
String snapshotName1 = "test-restore-snapshot1";
590-
Path absolutePath1 = randomRepoPath().toAbsolutePath();
591-
Path absolutePath3 = randomRepoPath().toAbsolutePath();
592-
String restoredIndexName1 = indexName1 + "-restored";
593-
594-
createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false));
595-
createRepository(remoteStoreRepo2Name, "fs", absolutePath3);
596-
597-
Client client = client();
598-
Settings indexSettings = getIndexSettings(1, 0).build();
599-
createIndex(indexName1, indexSettings);
600-
601-
Settings indexSettings2 = getIndexSettings(1, 0).build();
602-
createIndex(indexName2, indexSettings2);
603-
604-
final int numDocsInIndex1 = 5;
605-
final int numDocsInIndex2 = 6;
606-
indexDocuments(client, indexName1, numDocsInIndex1);
607-
indexDocuments(client, indexName2, numDocsInIndex2);
608-
ensureGreen(indexName1, indexName2);
609-
610-
internalCluster().startDataOnlyNode();
611-
612-
logger.info("--> snapshot");
613-
SnapshotInfo snapshotInfo1 = createSnapshot(
614-
snapshotRepoName,
615-
snapshotName1,
616-
new ArrayList<>(Arrays.asList(indexName1, indexName2))
617-
);
618-
assertThat(snapshotInfo1.successfulShards(), greaterThan(0));
619-
assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards()));
620-
assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS));
621-
622-
Settings remoteStoreIndexSettings = Settings.builder()
623-
.put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo2Name)
624-
.build();
625-
// restore index as a remote store index with different remote store repo
626-
RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
627-
.cluster()
628-
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
629-
.setWaitForCompletion(false)
630-
.setIndexSettings(remoteStoreIndexSettings)
631-
.setIndices(indexName1)
632-
.setRenamePattern(indexName1)
633-
.setRenameReplacement(restoredIndexName1)
634-
.get();
635-
assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
636-
ensureGreen(restoredIndexName1);
637-
assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1);
638-
639-
// deleting data for restoredIndexName1 and restoring from remote store.
640-
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary));
641-
// Re-initialize client to make sure we are not using client from stopped node.
642-
client = client(clusterManagerNode);
643-
assertAcked(client.admin().indices().prepareClose(restoredIndexName1));
644-
client.admin()
645-
.cluster()
646-
.restoreRemoteStore(
647-
new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true),
648-
PlainActionFuture.newFuture()
649-
);
650-
ensureYellowAndNoInitializingShards(restoredIndexName1);
651-
ensureGreen(restoredIndexName1);
652-
// indexing some new docs and validating
653-
assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1);
654-
indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2);
655-
ensureGreen(restoredIndexName1);
656-
assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2);
657-
}
658-
659524
public void testRestoreShallowSnapshotRepository() throws ExecutionException, InterruptedException {
660525
String indexName1 = "testindex1";
661526
String snapshotRepoName = "test-restore-snapshot-repo";
@@ -787,4 +652,98 @@ public void testRestoreShallowSnapshotIndexAfterSnapshot() throws ExecutionExcep
787652
assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2);
788653
}
789654

655+
public void testInvalidRestoreRequestScenarios() throws Exception {
656+
internalCluster().startClusterManagerOnlyNode();
657+
internalCluster().startDataOnlyNode();
658+
String index = "test-index";
659+
String snapshotRepo = "test-restore-snapshot-repo";
660+
String newRemoteStoreRepo = "test-new-rs-repo";
661+
String snapshotName1 = "test-restore-snapshot1";
662+
String snapshotName2 = "test-restore-snapshot2";
663+
Path absolutePath1 = randomRepoPath().toAbsolutePath();
664+
logger.info("Snapshot Path [{}]", absolutePath1);
665+
String restoredIndex = index + "-restored";
666+
667+
createRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, true));
668+
669+
Client client = client();
670+
Settings indexSettings = getIndexSettings(1, 0).build();
671+
createIndex(index, indexSettings);
672+
673+
final int numDocsInIndex = 5;
674+
indexDocuments(client, index, numDocsInIndex);
675+
ensureGreen(index);
676+
677+
internalCluster().startDataOnlyNode();
678+
logger.info("--> snapshot");
679+
680+
SnapshotInfo snapshotInfo = createSnapshot(snapshotRepo, snapshotName1, new ArrayList<>(List.of(index)));
681+
assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
682+
assertThat(snapshotInfo.successfulShards(), greaterThan(0));
683+
assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
684+
685+
updateRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, false));
686+
SnapshotInfo snapshotInfo2 = createSnapshot(snapshotRepo, snapshotName2, new ArrayList<>(List.of(index)));
687+
assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS));
688+
assertThat(snapshotInfo2.successfulShards(), greaterThan(0));
689+
assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards()));
690+
691+
DeleteResponse deleteResponse = client().prepareDelete(index, "0").execute().actionGet();
692+
assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED);
693+
indexDocuments(client, index, numDocsInIndex, numDocsInIndex + randomIntBetween(2, 5));
694+
ensureGreen(index);
695+
696+
// try index restore with remote store disabled
697+
SnapshotRestoreException exception = expectThrows(
698+
SnapshotRestoreException.class,
699+
() -> client().admin()
700+
.cluster()
701+
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
702+
.setWaitForCompletion(false)
703+
.setIgnoreIndexSettings(SETTING_REMOTE_STORE_ENABLED)
704+
.setIndices(index)
705+
.setRenamePattern(index)
706+
.setRenameReplacement(restoredIndex)
707+
.get()
708+
);
709+
assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.enabled] on restore"));
710+
711+
// try index restore with remote store repository modified
712+
Settings remoteStoreIndexSettings = Settings.builder()
713+
.put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo)
714+
.build();
715+
716+
exception = expectThrows(
717+
SnapshotRestoreException.class,
718+
() -> client().admin()
719+
.cluster()
720+
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
721+
.setWaitForCompletion(false)
722+
.setIndexSettings(remoteStoreIndexSettings)
723+
.setIndices(index)
724+
.setRenamePattern(index)
725+
.setRenameReplacement(restoredIndex)
726+
.get()
727+
);
728+
assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore"));
729+
730+
// try index restore with remote store repository and translog store repository disabled
731+
exception = expectThrows(
732+
SnapshotRestoreException.class,
733+
() -> client().admin()
734+
.cluster()
735+
.prepareRestoreSnapshot(snapshotRepo, snapshotName1)
736+
.setWaitForCompletion(false)
737+
.setIgnoreIndexSettings(
738+
IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
739+
IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY
740+
)
741+
.setIndices(index)
742+
.setRenamePattern(index)
743+
.setRenameReplacement(restoredIndex)
744+
.get()
745+
);
746+
assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore"));
747+
}
748+
790749
}

‎server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java

+38-4
Original file line numberDiff line numberDiff line change
@@ -112,12 +112,16 @@ public void createSnapshot() {
112112
}
113113

114114
public RestoreSnapshotResponse restoreSnapshotWithSettings(Settings indexSettings) {
115+
return restoreSnapshotWithSettings(indexSettings, RESTORED_INDEX_NAME);
116+
}
117+
118+
public RestoreSnapshotResponse restoreSnapshotWithSettings(Settings indexSettings, String restoredIndexName) {
115119
RestoreSnapshotRequestBuilder builder = client().admin()
116120
.cluster()
117121
.prepareRestoreSnapshot(REPOSITORY_NAME, SNAPSHOT_NAME)
118122
.setWaitForCompletion(false)
119123
.setRenamePattern(INDEX_NAME)
120-
.setRenameReplacement(RESTORED_INDEX_NAME);
124+
.setRenameReplacement(restoredIndexName);
121125
if (indexSettings != null) {
122126
builder.setIndexSettings(indexSettings);
123127
}
@@ -311,7 +315,8 @@ public void testSnapshotRestoreOnIndexWithSegRepClusterSetting() throws Exceptio
311315
* 2. Snapshot index
312316
* 3. Add new set of nodes with `cluster.indices.replication.strategy` set to SEGMENT and `cluster.index.restrict.replication.type`
313317
* set to true.
314-
* 4. Perform restore on new set of nodes to validate restored index has `DOCUMENT` replication.
318+
* 4. Perform restore on new set of nodes to validate restored index has `SEGMENT` replication.
319+
* 5. Validate that if replication type is passed as DOCUMENT as request parameter, restore operation fails
315320
*/
316321
public void testSnapshotRestoreOnRestrictReplicationSetting() throws Exception {
317322
final int documentCount = scaledRandomIntBetween(1, 10);
@@ -337,9 +342,20 @@ public void testSnapshotRestoreOnRestrictReplicationSetting() throws Exception {
337342

338343
createSnapshot();
339344

340-
// Delete index
345+
RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(restoreIndexSegRepSettings(), RESTORED_INDEX_NAME);
346+
assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
347+
ensureGreen(RESTORED_INDEX_NAME);
348+
GetSettingsResponse settingsResponse = client().admin()
349+
.indices()
350+
.getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME).includeDefaults(true))
351+
.get();
352+
assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.SEGMENT.toString());
353+
354+
// Delete indices
341355
assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get());
342356
assertFalse("index [" + INDEX_NAME + "] should have been deleted", indexExists(INDEX_NAME));
357+
assertAcked(client().admin().indices().delete(new DeleteIndexRequest(RESTORED_INDEX_NAME)).get());
358+
assertFalse("index [" + RESTORED_INDEX_NAME + "] should have been deleted", indexExists(RESTORED_INDEX_NAME));
343359

344360
// Start new set of nodes with cluster level replication type setting and restrict replication type setting.
345361
Settings settings = Settings.builder()
@@ -361,7 +377,25 @@ public void testSnapshotRestoreOnRestrictReplicationSetting() throws Exception {
361377
// Perform snapshot restore
362378
logger.info("--> Performing snapshot restore to target index");
363379

364-
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> restoreSnapshotWithSettings(null));
380+
restoreSnapshotResponse = restoreSnapshotWithSettings(null);
381+
382+
// Assertions
383+
assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
384+
ensureGreen(RESTORED_INDEX_NAME);
385+
settingsResponse = client().admin()
386+
.indices()
387+
.getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME).includeDefaults(true))
388+
.get();
389+
assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.SEGMENT.toString());
390+
391+
// restore index with cluster default setting
392+
restoreSnapshotWithSettings(restoreIndexSegRepSettings(), RESTORED_INDEX_NAME + "1");
393+
394+
// Perform Snapshot Restore with different index name
395+
IllegalArgumentException exception = expectThrows(
396+
IllegalArgumentException.class,
397+
() -> restoreSnapshotWithSettings(restoreIndexDocRepSettings(), RESTORED_INDEX_NAME + "2")
398+
);
365399
assertEquals(REPLICATION_MISMATCH_VALIDATION_ERROR, exception.getMessage());
366400
}
367401
}

0 commit comments

Comments
 (0)