Skip to content

Commit 170ea7a

Browse files
ashking94ashking94
authored
Introduce remote store hash algo in customData in IndexMetadata (#12986)
Signed-off-by: Ashish Singh <ssashish@amazon.com>
1 parent 3491bcb commit 170ea7a

26 files changed

+685
-270
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

+11-11
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@
2727
import org.opensearch.core.rest.RestStatus;
2828
import org.opensearch.index.IndexService;
2929
import org.opensearch.index.IndexSettings;
30-
import org.opensearch.index.remote.RemoteStorePathType;
30+
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
31+
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
3132
import org.opensearch.index.shard.IndexShard;
3233
import org.opensearch.indices.IndicesService;
3334
import org.opensearch.indices.replication.common.ReplicationType;
@@ -283,7 +284,7 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
283284

284285
indexDocuments(client, indexName1, randomIntBetween(5, 10));
285286
ensureGreen(indexName1);
286-
validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
287+
validatePathType(indexName1, PathType.FIXED, PathHashAlgorithm.FNV_1A);
287288

288289
logger.info("--> snapshot");
289290
SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1)));
@@ -300,14 +301,12 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
300301
.get();
301302
assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
302303
ensureGreen(restoredIndexName1version1);
303-
validateRemoteStorePathType(restoredIndexName1version1, RemoteStorePathType.FIXED);
304+
validatePathType(restoredIndexName1version1, PathType.FIXED, PathHashAlgorithm.FNV_1A);
304305

305306
client(clusterManagerNode).admin()
306307
.cluster()
307308
.prepareUpdateSettings()
308-
.setTransientSettings(
309-
Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), RemoteStorePathType.HASHED_PREFIX)
310-
)
309+
.setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX))
311310
.get();
312311

313312
restoreSnapshotResponse = client.admin()
@@ -319,24 +318,25 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
319318
.get();
320319
assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
321320
ensureGreen(restoredIndexName1version2);
322-
validateRemoteStorePathType(restoredIndexName1version2, RemoteStorePathType.HASHED_PREFIX);
321+
validatePathType(restoredIndexName1version2, PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A);
323322

324323
// Create index with cluster setting cluster.remote_store.index.path.prefix.type as hashed_prefix.
325324
indexSettings = getIndexSettings(1, 0).build();
326325
createIndex(indexName2, indexSettings);
327326
ensureGreen(indexName2);
328-
validateRemoteStorePathType(indexName2, RemoteStorePathType.HASHED_PREFIX);
327+
validatePathType(indexName2, PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A);
329328

330329
// Validating that custom data has not changed for indexes which were created before the cluster setting got updated
331-
validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
330+
validatePathType(indexName1, PathType.FIXED, PathHashAlgorithm.FNV_1A);
332331
}
333332

334-
private void validateRemoteStorePathType(String index, RemoteStorePathType pathType) {
333+
private void validatePathType(String index, PathType pathType, PathHashAlgorithm pathHashAlgorithm) {
335334
ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
336335
// Validate that the remote_store custom data is present in index metadata for the created index.
337336
Map<String, String> remoteCustomData = state.metadata().index(index).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
338337
assertNotNull(remoteCustomData);
339-
assertEquals(pathType.toString(), remoteCustomData.get(RemoteStorePathType.NAME));
338+
assertEquals(pathType.name(), remoteCustomData.get(PathType.NAME));
339+
assertEquals(pathHashAlgorithm.name(), remoteCustomData.get(PathHashAlgorithm.NAME));
340340
}
341341

342342
public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

+22-12
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,10 @@
8888
import org.opensearch.index.mapper.MapperService;
8989
import org.opensearch.index.mapper.MapperService.MergeReason;
9090
import org.opensearch.index.query.QueryShardContext;
91-
import org.opensearch.index.remote.RemoteStorePathType;
92-
import org.opensearch.index.remote.RemoteStorePathTypeResolver;
91+
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
92+
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
93+
import org.opensearch.index.remote.RemoteStorePathStrategy;
94+
import org.opensearch.index.remote.RemoteStorePathStrategyResolver;
9395
import org.opensearch.index.shard.IndexSettingProvider;
9496
import org.opensearch.index.translog.Translog;
9597
import org.opensearch.indices.IndexCreationException;
@@ -171,7 +173,7 @@ public class MetadataCreateIndexService {
171173
private AwarenessReplicaBalance awarenessReplicaBalance;
172174

173175
@Nullable
174-
private final RemoteStorePathTypeResolver remoteStorePathTypeResolver;
176+
private final RemoteStorePathStrategyResolver remoteStorePathStrategyResolver;
175177

176178
public MetadataCreateIndexService(
177179
final Settings settings,
@@ -204,8 +206,8 @@ public MetadataCreateIndexService(
204206

205207
// Task is onboarded for throttling, it will get retried from associated TransportClusterManagerNodeAction.
206208
createIndexTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.CREATE_INDEX_KEY, true);
207-
remoteStorePathTypeResolver = isRemoteDataAttributePresent(settings)
208-
? new RemoteStorePathTypeResolver(clusterService.getClusterSettings())
209+
remoteStorePathStrategyResolver = isRemoteDataAttributePresent(settings)
210+
? new RemoteStorePathStrategyResolver(clusterService.getClusterSettings())
209211
: null;
210212
}
211213

@@ -554,7 +556,7 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
554556
tmpImdBuilder.setRoutingNumShards(routingNumShards);
555557
tmpImdBuilder.settings(indexSettings);
556558
tmpImdBuilder.system(isSystem);
557-
addRemoteStorePathTypeInCustomData(tmpImdBuilder, true);
559+
addRemoteStorePathStrategyInCustomData(tmpImdBuilder, true);
558560

559561
// Set up everything, now locally create the index to see that things are ok, and apply
560562
IndexMetadata tempMetadata = tmpImdBuilder.build();
@@ -569,19 +571,27 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
569571
* @param tmpImdBuilder index metadata builder.
570572
* @param assertNullOldType flag to verify that the old remote store path type is null
571573
*/
572-
public void addRemoteStorePathTypeInCustomData(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType) {
573-
if (remoteStorePathTypeResolver != null) {
574+
public void addRemoteStorePathStrategyInCustomData(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType) {
575+
if (remoteStorePathStrategyResolver != null) {
574576
// It is possible that remote custom data exists already. In such cases, we need to only update the path type
575577
// in the remote store custom data map.
576578
Map<String, String> existingRemoteCustomData = tmpImdBuilder.removeCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
577579
Map<String, String> remoteCustomData = existingRemoteCustomData == null
578580
? new HashMap<>()
579581
: new HashMap<>(existingRemoteCustomData);
580582
// Determine the path type for use using the remoteStorePathResolver.
581-
String newPathType = remoteStorePathTypeResolver.getType().toString();
582-
String oldPathType = remoteCustomData.put(RemoteStorePathType.NAME, newPathType);
583-
assert !assertNullOldType || Objects.isNull(oldPathType);
584-
logger.trace(() -> new ParameterizedMessage("Added new path type {}, replaced old path type {}", newPathType, oldPathType));
583+
RemoteStorePathStrategy newPathStrategy = remoteStorePathStrategyResolver.get();
584+
String oldPathType = remoteCustomData.put(PathType.NAME, newPathStrategy.getType().name());
585+
String oldHashAlgorithm = remoteCustomData.put(PathHashAlgorithm.NAME, newPathStrategy.getHashAlgorithm().name());
586+
assert !assertNullOldType || (Objects.isNull(oldPathType) && Objects.isNull(oldHashAlgorithm));
587+
logger.trace(
588+
() -> new ParameterizedMessage(
589+
"Added newPathStrategy={}, replaced oldPathType={} oldHashAlgorithm={}",
590+
newPathStrategy,
591+
oldPathType,
592+
oldHashAlgorithm
593+
)
594+
);
585595
tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, remoteCustomData);
586596
}
587597
}

‎server/src/main/java/org/opensearch/common/hash/FNV1a.java

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.common.hash;
10+
11+
import java.nio.charset.StandardCharsets;
12+
13+
/**
14+
* Provides hashing function using FNV1a hash function. @see <a href="http://www.isthe.com/chongo/tech/comp/fnv/#FNV-1">FNV author's website</a>.
15+
* 32 bit Java port of http://www.isthe.com/chongo/src/fnv/hash_32a.c
16+
* 64 bit Java port of http://www.isthe.com/chongo/src/fnv/hash_64a.c
17+
*
18+
* @opensearch.internal
19+
*/
20+
public class FNV1a {
21+
private static final long FNV_OFFSET_BASIS_32 = 0x811c9dc5L;
22+
private static final long FNV_PRIME_32 = 0x01000193L;
23+
24+
private static final long FNV_OFFSET_BASIS_64 = 0xcbf29ce484222325L;
25+
private static final long FNV_PRIME_64 = 0x100000001b3L;
26+
27+
// FNV-1a hash computation for 32-bit hash
28+
public static long hash32(String input) {
29+
long hash = FNV_OFFSET_BASIS_32;
30+
byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
31+
for (byte b : bytes) {
32+
hash ^= (b & 0xFF);
33+
hash *= FNV_PRIME_32;
34+
}
35+
return hash;
36+
}
37+
38+
// FNV-1a hash computation for 64-bit hash
39+
public static long hash64(String input) {
40+
long hash = FNV_OFFSET_BASIS_64;
41+
byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
42+
for (byte b : bytes) {
43+
hash ^= (b & 0xFF);
44+
hash *= FNV_PRIME_64;
45+
}
46+
return hash;
47+
}
48+
}

‎server/src/main/java/org/opensearch/index/IndexService.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -509,7 +509,7 @@ public synchronized IndexShard createShard(
509509
RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(this.indexSettings.getNodeSettings()),
510510
this.indexSettings.getUUID(),
511511
shardId,
512-
this.indexSettings.getRemoteStorePathType()
512+
this.indexSettings.getRemoteStorePathStrategy()
513513
);
514514
}
515515
remoteStore = new Store(shardId, this.indexSettings, remoteDirectory, lock, Store.OnClose.EMPTY, path);

‎server/src/main/java/org/opensearch/index/IndexSettings.java

+12-5
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,9 @@
4848
import org.opensearch.core.common.unit.ByteSizeUnit;
4949
import org.opensearch.core.common.unit.ByteSizeValue;
5050
import org.opensearch.core.index.Index;
51-
import org.opensearch.index.remote.RemoteStorePathType;
51+
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
52+
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
53+
import org.opensearch.index.remote.RemoteStorePathStrategy;
5254
import org.opensearch.index.translog.Translog;
5355
import org.opensearch.indices.replication.common.ReplicationType;
5456
import org.opensearch.ingest.IngestService;
@@ -1908,10 +1910,15 @@ public void setDocIdFuzzySetFalsePositiveProbability(double docIdFuzzySetFalsePo
19081910
this.docIdFuzzySetFalsePositiveProbability = docIdFuzzySetFalsePositiveProbability;
19091911
}
19101912

1911-
public RemoteStorePathType getRemoteStorePathType() {
1913+
public RemoteStorePathStrategy getRemoteStorePathStrategy() {
19121914
Map<String, String> remoteCustomData = indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
1913-
return remoteCustomData != null && remoteCustomData.containsKey(RemoteStorePathType.NAME)
1914-
? RemoteStorePathType.parseString(remoteCustomData.get(RemoteStorePathType.NAME))
1915-
: RemoteStorePathType.FIXED;
1915+
if (remoteCustomData != null
1916+
&& remoteCustomData.containsKey(PathType.NAME)
1917+
&& remoteCustomData.containsKey(PathHashAlgorithm.NAME)) {
1918+
PathType pathType = PathType.parseString(remoteCustomData.get(PathType.NAME));
1919+
PathHashAlgorithm pathHashAlgorithm = PathHashAlgorithm.parseString(remoteCustomData.get(PathHashAlgorithm.NAME));
1920+
return new RemoteStorePathStrategy(pathType, pathHashAlgorithm);
1921+
}
1922+
return new RemoteStorePathStrategy(PathType.FIXED);
19161923
}
19171924
}

‎server/src/main/java/org/opensearch/index/remote/RemoteStoreDataEnums.java

-69
This file was deleted.

0 commit comments

Comments
 (0)