Skip to content

Commit c348c7d

Browse files
ashking94ashking94
committed
Introduce remote store hash algo in customData in IndexMetadata (opensearch-project#12986)
Signed-off-by: Ashish Singh <ssashish@amazon.com>
1 parent 151a671 commit c348c7d

26 files changed

+685
-270
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

+11-11
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@
2525
import org.opensearch.core.rest.RestStatus;
2626
import org.opensearch.index.IndexService;
2727
import org.opensearch.index.IndexSettings;
28-
import org.opensearch.index.remote.RemoteStorePathType;
28+
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
29+
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
2930
import org.opensearch.index.shard.IndexShard;
3031
import org.opensearch.indices.IndicesService;
3132
import org.opensearch.indices.replication.common.ReplicationType;
@@ -225,7 +226,7 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
225226

226227
indexDocuments(client, indexName1, randomIntBetween(5, 10));
227228
ensureGreen(indexName1);
228-
validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
229+
validatePathType(indexName1, PathType.FIXED, PathHashAlgorithm.FNV_1A);
229230

230231
logger.info("--> snapshot");
231232
SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1)));
@@ -242,14 +243,12 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
242243
.get();
243244
assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
244245
ensureGreen(restoredIndexName1version1);
245-
validateRemoteStorePathType(restoredIndexName1version1, RemoteStorePathType.FIXED);
246+
validatePathType(restoredIndexName1version1, PathType.FIXED, PathHashAlgorithm.FNV_1A);
246247

247248
client(clusterManagerNode).admin()
248249
.cluster()
249250
.prepareUpdateSettings()
250-
.setTransientSettings(
251-
Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), RemoteStorePathType.HASHED_PREFIX)
252-
)
251+
.setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX))
253252
.get();
254253

255254
restoreSnapshotResponse = client.admin()
@@ -261,24 +260,25 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
261260
.get();
262261
assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
263262
ensureGreen(restoredIndexName1version2);
264-
validateRemoteStorePathType(restoredIndexName1version2, RemoteStorePathType.HASHED_PREFIX);
263+
validatePathType(restoredIndexName1version2, PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A);
265264

266265
// Create index with cluster setting cluster.remote_store.index.path.prefix.type as hashed_prefix.
267266
indexSettings = getIndexSettings(1, 0).build();
268267
createIndex(indexName2, indexSettings);
269268
ensureGreen(indexName2);
270-
validateRemoteStorePathType(indexName2, RemoteStorePathType.HASHED_PREFIX);
269+
validatePathType(indexName2, PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A);
271270

272271
// Validating that custom data has not changed for indexes which were created before the cluster setting got updated
273-
validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
272+
validatePathType(indexName1, PathType.FIXED, PathHashAlgorithm.FNV_1A);
274273
}
275274

276-
private void validateRemoteStorePathType(String index, RemoteStorePathType pathType) {
275+
private void validatePathType(String index, PathType pathType, PathHashAlgorithm pathHashAlgorithm) {
277276
ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
278277
// Validate that the remote_store custom data is present in index metadata for the created index.
279278
Map<String, String> remoteCustomData = state.metadata().index(index).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
280279
assertNotNull(remoteCustomData);
281-
assertEquals(pathType.toString(), remoteCustomData.get(RemoteStorePathType.NAME));
280+
assertEquals(pathType.name(), remoteCustomData.get(PathType.NAME));
281+
assertEquals(pathHashAlgorithm.name(), remoteCustomData.get(PathHashAlgorithm.NAME));
282282
}
283283

284284
public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

+22-12
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,10 @@
9090
import org.opensearch.index.mapper.MapperService;
9191
import org.opensearch.index.mapper.MapperService.MergeReason;
9292
import org.opensearch.index.query.QueryShardContext;
93-
import org.opensearch.index.remote.RemoteStorePathType;
94-
import org.opensearch.index.remote.RemoteStorePathTypeResolver;
93+
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
94+
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
95+
import org.opensearch.index.remote.RemoteStorePathStrategy;
96+
import org.opensearch.index.remote.RemoteStorePathStrategyResolver;
9597
import org.opensearch.index.shard.IndexSettingProvider;
9698
import org.opensearch.index.translog.Translog;
9799
import org.opensearch.indices.IndexCreationException;
@@ -175,7 +177,7 @@ public class MetadataCreateIndexService {
175177
private AwarenessReplicaBalance awarenessReplicaBalance;
176178

177179
@Nullable
178-
private final RemoteStorePathTypeResolver remoteStorePathTypeResolver;
180+
private final RemoteStorePathStrategyResolver remoteStorePathStrategyResolver;
179181

180182
public MetadataCreateIndexService(
181183
final Settings settings,
@@ -208,8 +210,8 @@ public MetadataCreateIndexService(
208210

209211
// Task is onboarded for throttling, it will get retried from associated TransportClusterManagerNodeAction.
210212
createIndexTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.CREATE_INDEX_KEY, true);
211-
remoteStorePathTypeResolver = isRemoteDataAttributePresent(settings)
212-
? new RemoteStorePathTypeResolver(clusterService.getClusterSettings())
213+
remoteStorePathStrategyResolver = isRemoteDataAttributePresent(settings)
214+
? new RemoteStorePathStrategyResolver(clusterService.getClusterSettings())
213215
: null;
214216
}
215217

@@ -558,7 +560,7 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
558560
tmpImdBuilder.setRoutingNumShards(routingNumShards);
559561
tmpImdBuilder.settings(indexSettings);
560562
tmpImdBuilder.system(isSystem);
561-
addRemoteStorePathTypeInCustomData(tmpImdBuilder, true);
563+
addRemoteStorePathStrategyInCustomData(tmpImdBuilder, true);
562564

563565
// Set up everything, now locally create the index to see that things are ok, and apply
564566
IndexMetadata tempMetadata = tmpImdBuilder.build();
@@ -573,19 +575,27 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
573575
* @param tmpImdBuilder index metadata builder.
574576
* @param assertNullOldType flag to verify that the old remote store path type is null
575577
*/
576-
public void addRemoteStorePathTypeInCustomData(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType) {
577-
if (remoteStorePathTypeResolver != null) {
578+
public void addRemoteStorePathStrategyInCustomData(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType) {
579+
if (remoteStorePathStrategyResolver != null) {
578580
// It is possible that remote custom data exists already. In such cases, we need to only update the path type
579581
// in the remote store custom data map.
580582
Map<String, String> existingRemoteCustomData = tmpImdBuilder.removeCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
581583
Map<String, String> remoteCustomData = existingRemoteCustomData == null
582584
? new HashMap<>()
583585
: new HashMap<>(existingRemoteCustomData);
584586
// Determine the path type for use using the remoteStorePathResolver.
585-
String newPathType = remoteStorePathTypeResolver.getType().toString();
586-
String oldPathType = remoteCustomData.put(RemoteStorePathType.NAME, newPathType);
587-
assert !assertNullOldType || Objects.isNull(oldPathType);
588-
logger.trace(() -> new ParameterizedMessage("Added new path type {}, replaced old path type {}", newPathType, oldPathType));
587+
RemoteStorePathStrategy newPathStrategy = remoteStorePathStrategyResolver.get();
588+
String oldPathType = remoteCustomData.put(PathType.NAME, newPathStrategy.getType().name());
589+
String oldHashAlgorithm = remoteCustomData.put(PathHashAlgorithm.NAME, newPathStrategy.getHashAlgorithm().name());
590+
assert !assertNullOldType || (Objects.isNull(oldPathType) && Objects.isNull(oldHashAlgorithm));
591+
logger.trace(
592+
() -> new ParameterizedMessage(
593+
"Added newPathStrategy={}, replaced oldPathType={} oldHashAlgorithm={}",
594+
newPathStrategy,
595+
oldPathType,
596+
oldHashAlgorithm
597+
)
598+
);
589599
tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, remoteCustomData);
590600
}
591601
}

‎server/src/main/java/org/opensearch/common/hash/FNV1a.java

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.common.hash;
10+
11+
import java.nio.charset.StandardCharsets;
12+
13+
/**
14+
* Provides hashing function using FNV1a hash function. @see <a href="http://www.isthe.com/chongo/tech/comp/fnv/#FNV-1">FNV author's website</a>.
15+
* 32 bit Java port of http://www.isthe.com/chongo/src/fnv/hash_32a.c
16+
* 64 bit Java port of http://www.isthe.com/chongo/src/fnv/hash_64a.c
17+
*
18+
* @opensearch.internal
19+
*/
20+
public class FNV1a {
21+
private static final long FNV_OFFSET_BASIS_32 = 0x811c9dc5L;
22+
private static final long FNV_PRIME_32 = 0x01000193L;
23+
24+
private static final long FNV_OFFSET_BASIS_64 = 0xcbf29ce484222325L;
25+
private static final long FNV_PRIME_64 = 0x100000001b3L;
26+
27+
// FNV-1a hash computation for 32-bit hash
28+
public static long hash32(String input) {
29+
long hash = FNV_OFFSET_BASIS_32;
30+
byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
31+
for (byte b : bytes) {
32+
hash ^= (b & 0xFF);
33+
hash *= FNV_PRIME_32;
34+
}
35+
return hash;
36+
}
37+
38+
// FNV-1a hash computation for 64-bit hash
39+
public static long hash64(String input) {
40+
long hash = FNV_OFFSET_BASIS_64;
41+
byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
42+
for (byte b : bytes) {
43+
hash ^= (b & 0xFF);
44+
hash *= FNV_PRIME_64;
45+
}
46+
return hash;
47+
}
48+
}

‎server/src/main/java/org/opensearch/index/IndexService.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,7 @@ public synchronized IndexShard createShard(
511511
RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(this.indexSettings.getNodeSettings()),
512512
this.indexSettings.getUUID(),
513513
shardId,
514-
this.indexSettings.getRemoteStorePathType()
514+
this.indexSettings.getRemoteStorePathStrategy()
515515
);
516516
}
517517
remoteStore = new Store(shardId, this.indexSettings, remoteDirectory, lock, Store.OnClose.EMPTY, path);

‎server/src/main/java/org/opensearch/index/IndexSettings.java

+12-5
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,9 @@
4949
import org.opensearch.core.common.unit.ByteSizeUnit;
5050
import org.opensearch.core.common.unit.ByteSizeValue;
5151
import org.opensearch.core.index.Index;
52-
import org.opensearch.index.remote.RemoteStorePathType;
52+
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
53+
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
54+
import org.opensearch.index.remote.RemoteStorePathStrategy;
5355
import org.opensearch.index.translog.Translog;
5456
import org.opensearch.indices.replication.common.ReplicationType;
5557
import org.opensearch.ingest.IngestService;
@@ -1906,10 +1908,15 @@ public void setDocIdFuzzySetFalsePositiveProbability(double docIdFuzzySetFalsePo
19061908
this.docIdFuzzySetFalsePositiveProbability = docIdFuzzySetFalsePositiveProbability;
19071909
}
19081910

1909-
public RemoteStorePathType getRemoteStorePathType() {
1911+
public RemoteStorePathStrategy getRemoteStorePathStrategy() {
19101912
Map<String, String> remoteCustomData = indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
1911-
return remoteCustomData != null && remoteCustomData.containsKey(RemoteStorePathType.NAME)
1912-
? RemoteStorePathType.parseString(remoteCustomData.get(RemoteStorePathType.NAME))
1913-
: RemoteStorePathType.FIXED;
1913+
if (remoteCustomData != null
1914+
&& remoteCustomData.containsKey(PathType.NAME)
1915+
&& remoteCustomData.containsKey(PathHashAlgorithm.NAME)) {
1916+
PathType pathType = PathType.parseString(remoteCustomData.get(PathType.NAME));
1917+
PathHashAlgorithm pathHashAlgorithm = PathHashAlgorithm.parseString(remoteCustomData.get(PathHashAlgorithm.NAME));
1918+
return new RemoteStorePathStrategy(pathType, pathHashAlgorithm);
1919+
}
1920+
return new RemoteStorePathStrategy(PathType.FIXED);
19141921
}
19151922
}

‎server/src/main/java/org/opensearch/index/remote/RemoteStoreDataEnums.java

-69
This file was deleted.

0 commit comments

Comments
 (0)