Skip to content

Commit 245fa47

Browse files
authored
Add path prefix support to hashed prefix snapshots (opensearch-project#15664)
* Add path prefix support to hashed prefix snapshots Signed-off-by: Ashish Singh <ssashish@amazon.com> * Fix test Signed-off-by: Ashish Singh <ssashish@amazon.com> --------- Signed-off-by: Ashish Singh <ssashish@amazon.com>
1 parent e5e8504 commit 245fa47

File tree

4 files changed

+21
-0
lines changed

4 files changed

+21
-0
lines changed

CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4747
- [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291)))
4848
- Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409))
4949
- Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557))
50+
- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664))
5051
- Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568))
5152
- [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218))
5253
- Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010))

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

+2
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@
152152
import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlSettings;
153153
import org.opensearch.ratelimitting.admissioncontrol.settings.CpuBasedAdmissionControllerSettings;
154154
import org.opensearch.ratelimitting.admissioncontrol.settings.IoBasedAdmissionControllerSettings;
155+
import org.opensearch.repositories.blobstore.BlobStoreRepository;
155156
import org.opensearch.repositories.fs.FsRepository;
156157
import org.opensearch.rest.BaseRestHandler;
157158
import org.opensearch.script.ScriptService;
@@ -777,6 +778,7 @@ public void apply(Settings value, Settings current, Settings previous) {
777778
RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED,
778779
RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX,
779780
RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX,
781+
BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING,
780782

781783
SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING,
782784

server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java

+14
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,16 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
343343
Setting.Property.NodeScope
344344
);
345345

346+
/**
347+
* Controls the fixed prefix for the snapshot shard blob path.
348+
*/
349+
public static final Setting<String> SNAPSHOT_SHARD_PATH_PREFIX_SETTING = Setting.simpleString(
350+
"cluster.snapshot.shard.path.prefix",
351+
"",
352+
Setting.Property.NodeScope,
353+
Setting.Property.Final
354+
);
355+
346356
protected volatile boolean supportURLRepo;
347357

348358
private volatile int maxShardBlobDeleteBatch;
@@ -434,6 +444,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
434444

435445
private final NamedXContentRegistry namedXContentRegistry;
436446

447+
private final String snapshotShardPathPrefix;
448+
437449
/**
438450
* Flag that is set to {@code true} if this instance is started with {@link #metadata} that has a higher value for
439451
* {@link RepositoryMetadata#pendingGeneration()} than for {@link RepositoryMetadata#generation()} indicating a full cluster restart
@@ -485,6 +497,7 @@ protected BlobStoreRepository(
485497
this.clusterService = clusterService;
486498
this.recoverySettings = recoverySettings;
487499
this.remoteStoreSettings = new RemoteStoreSettings(clusterService.getSettings(), clusterService.getClusterSettings());
500+
this.snapshotShardPathPrefix = SNAPSHOT_SHARD_PATH_PREFIX_SETTING.get(clusterService.getSettings());
488501
}
489502

490503
@Override
@@ -2729,6 +2742,7 @@ private BlobPath shardPath(IndexId indexId, int shardId) {
27292742
SnapshotShardPathInput shardPathInput = new SnapshotShardPathInput.Builder().basePath(basePath())
27302743
.indexUUID(indexId.getId())
27312744
.shardId(String.valueOf(shardId))
2745+
.fixedPrefix(snapshotShardPathPrefix)
27322746
.build();
27332747
PathHashAlgorithm pathHashAlgorithm = pathType != PathType.FIXED ? FNV_1A_COMPOSITE_1 : null;
27342748
return pathType.path(shardPathInput, pathHashAlgorithm);

test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java

+4
Original file line numberDiff line numberDiff line change
@@ -403,6 +403,8 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase {
403403

404404
private static Boolean segmentsPathFixedPrefix;
405405

406+
private static Boolean snapshotShardPathFixedPrefix;
407+
406408
private Path remoteStoreRepositoryPath;
407409

408410
private ReplicationType randomReplicationType;
@@ -414,6 +416,7 @@ public static void beforeClass() throws Exception {
414416
prefixModeVerificationEnable = randomBoolean();
415417
translogPathFixedPrefix = randomBoolean();
416418
segmentsPathFixedPrefix = randomBoolean();
419+
snapshotShardPathFixedPrefix = randomBoolean();
417420
testClusterRule.beforeClass();
418421
}
419422

@@ -2901,6 +2904,7 @@ private static Settings buildRemoteStoreNodeAttributes(
29012904
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean());
29022905
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : "");
29032906
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : "");
2907+
settings.put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), segmentsPathFixedPrefix ? "c" : "");
29042908
return settings.build();
29052909
}
29062910

0 commit comments

Comments
 (0)