Skip to content

Commit 45aaf5b

Browse files
ashking94ashking94
committed
Add integ tests
Signed-off-by: Ashish Singh <ssashish@amazon.com>
1 parent 08cb85c commit 45aaf5b

File tree

2 files changed

+140
-1
lines changed

2 files changed

+140
-1
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

+83-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.opensearch.action.support.PlainActionFuture;
1919
import org.opensearch.client.Client;
2020
import org.opensearch.client.Requests;
21+
import org.opensearch.cluster.ClusterState;
2122
import org.opensearch.cluster.metadata.IndexMetadata;
2223
import org.opensearch.common.io.PathUtils;
2324
import org.opensearch.common.settings.Settings;
@@ -26,6 +27,7 @@
2627
import org.opensearch.core.rest.RestStatus;
2728
import org.opensearch.index.IndexService;
2829
import org.opensearch.index.IndexSettings;
30+
import org.opensearch.index.remote.RemoteStorePathType;
2931
import org.opensearch.index.shard.IndexShard;
3032
import org.opensearch.indices.IndicesService;
3133
import org.opensearch.indices.replication.common.ReplicationType;
@@ -43,14 +45,15 @@
4345
import java.util.ArrayList;
4446
import java.util.Arrays;
4547
import java.util.List;
48+
import java.util.Map;
4649
import java.util.Optional;
4750
import java.util.concurrent.ExecutionException;
4851
import java.util.stream.Collectors;
4952
import java.util.stream.Stream;
5053

5154
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
5255
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
53-
import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
56+
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING;
5457
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
5558
import static org.hamcrest.Matchers.equalTo;
5659
import static org.hamcrest.Matchers.greaterThan;
@@ -257,6 +260,85 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut
257260
assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1 + 2);
258261
}
259262

263+
/**
264+
* In this test, we validate presence of remote_store custom data in index metadata for standard index creation and
265+
* on snapshot restore.
266+
*/
267+
public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
268+
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
269+
internalCluster().startDataOnlyNode();
270+
String indexName1 = "testindex1";
271+
String indexName2 = "testindex2";
272+
String snapshotRepoName = "test-restore-snapshot-repo";
273+
String snapshotName1 = "test-restore-snapshot1";
274+
Path absolutePath1 = randomRepoPath().toAbsolutePath();
275+
logger.info("Snapshot Path [{}]", absolutePath1);
276+
String restoredIndexName1version1 = indexName1 + "-restored-1";
277+
String restoredIndexName1version2 = indexName1 + "-restored-2";
278+
279+
createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));
280+
Client client = client();
281+
Settings indexSettings = getIndexSettings(1, 0).build();
282+
createIndex(indexName1, indexSettings);
283+
284+
indexDocuments(client, indexName1, randomIntBetween(5, 10));
285+
ensureGreen(indexName1);
286+
validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
287+
288+
logger.info("--> snapshot");
289+
SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1)));
290+
assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
291+
assertTrue(snapshotInfo.successfulShards() > 0);
292+
assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards());
293+
294+
RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
295+
.cluster()
296+
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
297+
.setWaitForCompletion(false)
298+
.setRenamePattern(indexName1)
299+
.setRenameReplacement(restoredIndexName1version1)
300+
.get();
301+
assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
302+
ensureGreen(restoredIndexName1version1);
303+
validateRemoteStorePathType(restoredIndexName1version1, RemoteStorePathType.FIXED);
304+
305+
client(clusterManagerNode).admin()
306+
.cluster()
307+
.prepareUpdateSettings()
308+
.setTransientSettings(
309+
Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), RemoteStorePathType.HASHED_PREFIX)
310+
)
311+
.get();
312+
313+
restoreSnapshotResponse = client.admin()
314+
.cluster()
315+
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
316+
.setWaitForCompletion(false)
317+
.setRenamePattern(indexName1)
318+
.setRenameReplacement(restoredIndexName1version2)
319+
.get();
320+
assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
321+
ensureGreen(restoredIndexName1version2);
322+
validateRemoteStorePathType(restoredIndexName1version2, RemoteStorePathType.HASHED_PREFIX);
323+
324+
// Create index with cluster setting cluster.remote_store.index.path.prefix.type as hashed_prefix.
325+
indexSettings = getIndexSettings(1, 0).build();
326+
createIndex(indexName2, indexSettings);
327+
ensureGreen(indexName2);
328+
validateRemoteStorePathType(indexName2, RemoteStorePathType.HASHED_PREFIX);
329+
330+
// Validating that custom data has not changed for indexes which were created before the cluster setting got updated
331+
validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
332+
}
333+
334+
private void validateRemoteStorePathType(String index, RemoteStorePathType pathType) {
335+
ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
336+
// Validate that the remote_store custom data is present in index metadata for the created index.
337+
Map<String, String> remoteCustomData = state.metadata().index(index).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
338+
assertNotNull(remoteCustomData);
339+
assertEquals(pathType.toString(), remoteCustomData.get(RemoteStorePathType.NAME));
340+
}
341+
260342
public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
261343
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
262344
String primary = internalCluster().startDataOnlyNode();

‎server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java

+57
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.opensearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
4040
import org.opensearch.action.index.IndexRequestBuilder;
4141
import org.opensearch.client.Client;
42+
import org.opensearch.cluster.ClusterState;
4243
import org.opensearch.cluster.block.ClusterBlocks;
4344
import org.opensearch.cluster.metadata.IndexMetadata;
4445
import org.opensearch.cluster.metadata.MappingMetadata;
@@ -151,6 +152,62 @@ public void testParallelRestoreOperations() {
151152
assertThat(client.prepareGet(restoredIndexName2, docId2).get().isExists(), equalTo(true));
152153
}
153154

155+
/**
156+
* In this test, we test that an index created does not have any remote_store custom data in index metadata at the
157+
* time of index creation and after snapshot restore.
158+
*/
159+
public void testNoRemoteStoreCustomDataOnIndexCreationAndRestore() {
160+
String indexName1 = "testindex1";
161+
String repoName = "test-restore-snapshot-repo";
162+
String snapshotName1 = "test-restore-snapshot1";
163+
Path absolutePath = randomRepoPath().toAbsolutePath();
164+
logger.info("Path [{}]", absolutePath);
165+
String restoredIndexName1 = indexName1 + "-restored";
166+
String expectedValue = "expected";
167+
168+
Client client = client();
169+
// Write a document
170+
String docId = Integer.toString(randomInt());
171+
index(indexName1, "_doc", docId, "value", expectedValue);
172+
173+
createRepository(repoName, "fs", absolutePath);
174+
175+
logger.info("--> snapshot");
176+
CreateSnapshotResponse createSnapshotResponse = client.admin()
177+
.cluster()
178+
.prepareCreateSnapshot(repoName, snapshotName1)
179+
.setWaitForCompletion(true)
180+
.setIndices(indexName1)
181+
.get();
182+
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
183+
assertThat(
184+
createSnapshotResponse.getSnapshotInfo().successfulShards(),
185+
equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())
186+
);
187+
assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));
188+
189+
ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
190+
191+
// Validate that the remote_store custom data is not present in index metadata for the created index.
192+
assertNull(state.metadata().index(indexName1).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY));
193+
194+
RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin()
195+
.cluster()
196+
.prepareRestoreSnapshot(repoName, snapshotName1)
197+
.setWaitForCompletion(false)
198+
.setRenamePattern(indexName1)
199+
.setRenameReplacement(restoredIndexName1)
200+
.get();
201+
assertThat(restoreSnapshotResponse1.status(), equalTo(RestStatus.ACCEPTED));
202+
ensureGreen(restoredIndexName1);
203+
assertThat(client.prepareGet(restoredIndexName1, docId).get().isExists(), equalTo(true));
204+
205+
state = client().admin().cluster().prepareState().execute().actionGet().getState();
206+
207+
// Validate that the remote_store custom data is not present in index metadata for the restored index.
208+
assertNull(state.metadata().index(restoredIndexName1).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY));
209+
}
210+
154211
public void testParallelRestoreOperationsFromSingleSnapshot() throws Exception {
155212
String indexName1 = "testindex1";
156213
String indexName2 = "testindex2";

0 commit comments

Comments
 (0)