Skip to content

Commit bdb725d

Browse files
vinaykpudvinaykpud
authored
Add support for restoring from snapshot with search replicas (#16111) (#16478)
Signed-off-by: Vinay Krishna Pudyodu <vinkrish.neo@gmail.com>
1 parent 050421a commit bdb725d

File tree

7 files changed

+392
-0
lines changed

7 files changed

+392
-0
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
55

66
## [Unreleased 2.x]
77
### Added
8+
- Add support for restoring from snapshot with search replicas ([#16111](https://github.com/opensearch-project/OpenSearch/pull/16111))
89
- Switch from `buildSrc/version.properties` to Gradle version catalog (`gradle/libs.versions.toml`) to enable dependabot to perform automated upgrades on common libs ([#16284](https://github.com/opensearch-project/OpenSearch/pull/16284))
910

1011
### Dependencies

‎server/src/internalClusterTest/java/org/opensearch/indices/replication/SearchReplicaRestoreIT.java

+226
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.indices.replication;
10+
11+
import org.opensearch.action.search.SearchResponse;
12+
import org.opensearch.cluster.metadata.IndexMetadata;
13+
import org.opensearch.cluster.metadata.Metadata;
14+
import org.opensearch.common.settings.Settings;
15+
import org.opensearch.common.util.FeatureFlags;
16+
import org.opensearch.index.query.QueryBuilders;
17+
import org.opensearch.indices.replication.common.ReplicationType;
18+
import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
19+
import org.opensearch.snapshots.SnapshotRestoreException;
20+
import org.opensearch.test.OpenSearchIntegTestCase;
21+
22+
import java.util.List;
23+
24+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS;
25+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
26+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
27+
28+
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
29+
public class SearchReplicaRestoreIT extends AbstractSnapshotIntegTestCase {
30+
31+
private static final String INDEX_NAME = "test-idx-1";
32+
private static final String RESTORED_INDEX_NAME = INDEX_NAME + "-restored";
33+
private static final String REPOSITORY_NAME = "test-repo";
34+
private static final String SNAPSHOT_NAME = "test-snapshot";
35+
private static final String FS_REPOSITORY_TYPE = "fs";
36+
private static final int DOC_COUNT = 10;
37+
38+
@Override
39+
protected Settings featureFlagSettings() {
40+
return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL, true).build();
41+
}
42+
43+
public void testSearchReplicaRestore_WhenSnapshotOnDocRep_RestoreOnDocRepWithSearchReplica() throws Exception {
44+
bootstrapIndexWithOutSearchReplicas(ReplicationType.DOCUMENT);
45+
createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME);
46+
47+
SnapshotRestoreException exception = expectThrows(
48+
SnapshotRestoreException.class,
49+
() -> restoreSnapshot(
50+
REPOSITORY_NAME,
51+
SNAPSHOT_NAME,
52+
INDEX_NAME,
53+
RESTORED_INDEX_NAME,
54+
Settings.builder()
55+
.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT)
56+
.put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1)
57+
.build()
58+
)
59+
);
60+
assertTrue(exception.getMessage().contains(getSnapshotExceptionMessage(ReplicationType.DOCUMENT, ReplicationType.DOCUMENT)));
61+
}
62+
63+
public void testSearchReplicaRestore_WhenSnapshotOnDocRep_RestoreOnSegRepWithSearchReplica() throws Exception {
64+
bootstrapIndexWithOutSearchReplicas(ReplicationType.DOCUMENT);
65+
createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME);
66+
67+
restoreSnapshot(
68+
REPOSITORY_NAME,
69+
SNAPSHOT_NAME,
70+
INDEX_NAME,
71+
RESTORED_INDEX_NAME,
72+
Settings.builder()
73+
.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
74+
.put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1)
75+
.build()
76+
);
77+
ensureYellowAndNoInitializingShards(RESTORED_INDEX_NAME);
78+
internalCluster().startDataOnlyNode();
79+
ensureGreen(RESTORED_INDEX_NAME);
80+
assertEquals(1, getNumberOfSearchReplicas(RESTORED_INDEX_NAME));
81+
82+
SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get();
83+
assertHitCount(resp, DOC_COUNT);
84+
}
85+
86+
public void testSearchReplicaRestore_WhenSnapshotOnSegRep_RestoreOnDocRepWithSearchReplica() throws Exception {
87+
bootstrapIndexWithOutSearchReplicas(ReplicationType.SEGMENT);
88+
createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME);
89+
90+
SnapshotRestoreException exception = expectThrows(
91+
SnapshotRestoreException.class,
92+
() -> restoreSnapshot(
93+
REPOSITORY_NAME,
94+
SNAPSHOT_NAME,
95+
INDEX_NAME,
96+
RESTORED_INDEX_NAME,
97+
Settings.builder()
98+
.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT)
99+
.put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1)
100+
.build()
101+
)
102+
);
103+
assertTrue(exception.getMessage().contains(getSnapshotExceptionMessage(ReplicationType.SEGMENT, ReplicationType.DOCUMENT)));
104+
}
105+
106+
public void testSearchReplicaRestore_WhenSnapshotOnSegRep_RestoreOnSegRepWithSearchReplica() throws Exception {
107+
bootstrapIndexWithOutSearchReplicas(ReplicationType.SEGMENT);
108+
createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME);
109+
110+
restoreSnapshot(
111+
REPOSITORY_NAME,
112+
SNAPSHOT_NAME,
113+
INDEX_NAME,
114+
RESTORED_INDEX_NAME,
115+
Settings.builder().put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1).build()
116+
);
117+
ensureYellowAndNoInitializingShards(RESTORED_INDEX_NAME);
118+
internalCluster().startDataOnlyNode();
119+
ensureGreen(RESTORED_INDEX_NAME);
120+
assertEquals(1, getNumberOfSearchReplicas(RESTORED_INDEX_NAME));
121+
122+
SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get();
123+
assertHitCount(resp, DOC_COUNT);
124+
}
125+
126+
public void testSearchReplicaRestore_WhenSnapshotOnSegRepWithSearchReplica_RestoreOnDocRep() throws Exception {
127+
bootstrapIndexWithSearchReplicas();
128+
createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME);
129+
130+
SnapshotRestoreException exception = expectThrows(
131+
SnapshotRestoreException.class,
132+
() -> restoreSnapshot(
133+
REPOSITORY_NAME,
134+
SNAPSHOT_NAME,
135+
INDEX_NAME,
136+
RESTORED_INDEX_NAME,
137+
Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT).build()
138+
)
139+
);
140+
assertTrue(exception.getMessage().contains(getSnapshotExceptionMessage(ReplicationType.SEGMENT, ReplicationType.DOCUMENT)));
141+
}
142+
143+
public void testSearchReplicaRestore_WhenSnapshotOnSegRepWithSearchReplica_RestoreOnDocRepWithNoSearchReplica() throws Exception {
144+
bootstrapIndexWithSearchReplicas();
145+
createRepoAndSnapshot(REPOSITORY_NAME, FS_REPOSITORY_TYPE, SNAPSHOT_NAME, INDEX_NAME);
146+
147+
restoreSnapshot(
148+
REPOSITORY_NAME,
149+
SNAPSHOT_NAME,
150+
INDEX_NAME,
151+
RESTORED_INDEX_NAME,
152+
Settings.builder()
153+
.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT)
154+
.put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 0)
155+
.build()
156+
);
157+
ensureGreen(RESTORED_INDEX_NAME);
158+
assertEquals(0, getNumberOfSearchReplicas(RESTORED_INDEX_NAME));
159+
160+
SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get();
161+
assertHitCount(resp, DOC_COUNT);
162+
}
163+
164+
private void bootstrapIndexWithOutSearchReplicas(ReplicationType replicationType) throws InterruptedException {
165+
startCluster(2);
166+
167+
Settings settings = Settings.builder()
168+
.put(super.indexSettings())
169+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
170+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
171+
.put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 0)
172+
.put(IndexMetadata.SETTING_REPLICATION_TYPE, replicationType)
173+
.build();
174+
175+
createIndex(INDEX_NAME, settings);
176+
indexRandomDocs(INDEX_NAME, DOC_COUNT);
177+
refresh(INDEX_NAME);
178+
ensureGreen(INDEX_NAME);
179+
}
180+
181+
private void bootstrapIndexWithSearchReplicas() throws InterruptedException {
182+
startCluster(3);
183+
184+
Settings settings = Settings.builder()
185+
.put(super.indexSettings())
186+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
187+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
188+
.put(SETTING_NUMBER_OF_SEARCH_REPLICAS, 1)
189+
.put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
190+
.build();
191+
192+
createIndex(INDEX_NAME, settings);
193+
ensureGreen(INDEX_NAME);
194+
for (int i = 0; i < DOC_COUNT; i++) {
195+
client().prepareIndex(INDEX_NAME).setId(String.valueOf(i)).setSource("foo", "bar").get();
196+
}
197+
flushAndRefresh(INDEX_NAME);
198+
}
199+
200+
private void startCluster(int numOfNodes) {
201+
internalCluster().startClusterManagerOnlyNode();
202+
internalCluster().startDataOnlyNodes(numOfNodes);
203+
}
204+
205+
private void createRepoAndSnapshot(String repositoryName, String repositoryType, String snapshotName, String indexName) {
206+
createRepository(repositoryName, repositoryType, randomRepoPath().toAbsolutePath());
207+
createSnapshot(repositoryName, snapshotName, List.of(indexName));
208+
assertAcked(client().admin().indices().prepareDelete(INDEX_NAME));
209+
assertFalse("index [" + INDEX_NAME + "] should have been deleted", indexExists(INDEX_NAME));
210+
}
211+
212+
private String getSnapshotExceptionMessage(ReplicationType snapshotReplicationType, ReplicationType restoreReplicationType) {
213+
return "snapshot was created with [index.replication.type] as ["
214+
+ snapshotReplicationType
215+
+ "]. "
216+
+ "To restore with [index.replication.type] as ["
217+
+ restoreReplicationType
218+
+ "], "
219+
+ "[index.number_of_search_only_replicas] must be set to [0]";
220+
}
221+
222+
private int getNumberOfSearchReplicas(String index) {
223+
Metadata metadata = client().admin().cluster().prepareState().get().getState().metadata();
224+
return Integer.valueOf(metadata.index(index).getSettings().get(SETTING_NUMBER_OF_SEARCH_REPLICAS));
225+
}
226+
}

‎server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java

+11
Original file line numberDiff line numberDiff line change
@@ -572,6 +572,17 @@ private Builder initializeAsRestore(
572572
);
573573
}
574574
}
575+
for (int i = 0; i < indexMetadata.getNumberOfSearchOnlyReplicas(); i++) {
576+
indexShardRoutingBuilder.addShard(
577+
ShardRouting.newUnassigned(
578+
shardId,
579+
false,
580+
true,
581+
PeerRecoverySource.INSTANCE, // TODO: Update to remote store if enabled
582+
unassignedInfo
583+
)
584+
);
585+
}
575586
shards.put(shardNumber, indexShardRoutingBuilder.build());
576587
}
577588
return this;

‎server/src/main/java/org/opensearch/snapshots/RestoreService.java

+36
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@
9393
import org.opensearch.index.store.remote.filecache.FileCacheStats;
9494
import org.opensearch.indices.IndicesService;
9595
import org.opensearch.indices.ShardLimitValidator;
96+
import org.opensearch.indices.replication.common.ReplicationType;
9697
import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
9798
import org.opensearch.node.remotestore.RemoteStoreNodeService;
9899
import org.opensearch.repositories.IndexId;
@@ -121,10 +122,12 @@
121122
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_HISTORY_UUID;
122123
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID;
123124
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
125+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS;
124126
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
125127
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
126128
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
127129
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
130+
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
128131
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
129132
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED;
130133
import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY;
@@ -413,6 +416,13 @@ public ClusterState execute(ClusterState currentState) {
413416
overrideSettingsInternal,
414417
ignoreSettingsInternal
415418
);
419+
420+
validateReplicationTypeRestoreSettings(
421+
snapshot,
422+
metadata.index(index).getSettings().get(SETTING_REPLICATION_TYPE),
423+
snapshotIndexMetadata
424+
);
425+
416426
if (isRemoteSnapshot) {
417427
snapshotIndexMetadata = addSnapshotToIndexSettings(snapshotIndexMetadata, snapshot, snapshotIndexId);
418428
}
@@ -1320,6 +1330,32 @@ private static void validateSnapshotRestorable(final String repository, final Sn
13201330
}
13211331
}
13221332

1333+
// Visible for testing
1334+
static void validateReplicationTypeRestoreSettings(Snapshot snapshot, String snapshotReplicationType, IndexMetadata updatedMetadata) {
1335+
int restoreNumberOfSearchReplicas = updatedMetadata.getSettings().getAsInt(SETTING_NUMBER_OF_SEARCH_REPLICAS, 0);
1336+
1337+
if (restoreNumberOfSearchReplicas > 0
1338+
&& ReplicationType.DOCUMENT.toString().equals(updatedMetadata.getSettings().get(SETTING_REPLICATION_TYPE))) {
1339+
throw new SnapshotRestoreException(
1340+
snapshot,
1341+
"snapshot was created with ["
1342+
+ SETTING_REPLICATION_TYPE
1343+
+ "]"
1344+
+ " as ["
1345+
+ snapshotReplicationType
1346+
+ "]."
1347+
+ " To restore with ["
1348+
+ SETTING_REPLICATION_TYPE
1349+
+ "]"
1350+
+ " as ["
1351+
+ ReplicationType.DOCUMENT
1352+
+ "], ["
1353+
+ SETTING_NUMBER_OF_SEARCH_REPLICAS
1354+
+ "] must be set to [0]"
1355+
);
1356+
}
1357+
}
1358+
13231359
public static boolean failed(SnapshotInfo snapshot, String index) {
13241360
for (SnapshotShardFailure failure : snapshot.shardFailures()) {
13251361
if (index.equals(failure.index())) {

‎server/src/test/java/org/opensearch/cluster/routing/SearchOnlyReplicaRestoreTests.java

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.cluster.routing;
10+
11+
import org.opensearch.Version;
12+
import org.opensearch.cluster.ClusterName;
13+
import org.opensearch.cluster.ClusterState;
14+
import org.opensearch.cluster.metadata.IndexMetadata;
15+
import org.opensearch.cluster.metadata.Metadata;
16+
import org.opensearch.common.UUIDs;
17+
import org.opensearch.common.settings.Settings;
18+
import org.opensearch.repositories.IndexId;
19+
import org.opensearch.snapshots.Snapshot;
20+
import org.opensearch.snapshots.SnapshotId;
21+
import org.opensearch.test.OpenSearchTestCase;
22+
23+
import java.util.HashSet;
24+
25+
public class SearchOnlyReplicaRestoreTests extends OpenSearchTestCase {
26+
27+
public void testSearchOnlyReplicasRestored() {
28+
Metadata metadata = Metadata.builder()
29+
.put(
30+
IndexMetadata.builder("test")
31+
.settings(settings(Version.CURRENT))
32+
.numberOfShards(1)
33+
.numberOfReplicas(1)
34+
.numberOfSearchReplicas(1)
35+
)
36+
.build();
37+
38+
IndexMetadata indexMetadata = metadata.index("test");
39+
RecoverySource.SnapshotRecoverySource snapshotRecoverySource = new RecoverySource.SnapshotRecoverySource(
40+
UUIDs.randomBase64UUID(),
41+
new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())),
42+
Version.CURRENT,
43+
new IndexId("test", UUIDs.randomBase64UUID(random()))
44+
);
45+
46+
RoutingTable routingTable = RoutingTable.builder().addAsNewRestore(indexMetadata, snapshotRecoverySource, new HashSet<>()).build();
47+
48+
ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
49+
.metadata(metadata)
50+
.routingTable(routingTable)
51+
.build();
52+
53+
IndexShardRoutingTable indexShardRoutingTable = clusterState.routingTable().index("test").shard(0);
54+
55+
assertEquals(1, clusterState.routingTable().index("test").shards().size());
56+
assertEquals(3, indexShardRoutingTable.getShards().size());
57+
assertEquals(1, indexShardRoutingTable.searchOnlyReplicas().size());
58+
}
59+
}

0 commit comments

Comments
 (0)