Skip to content

Commit 61a0532

Browse files
ashking94ashking94
committed
Add remote path settings to RemoteStoreSettings
Signed-off-by: Ashish Singh <ssashish@amazon.com>
1 parent 1c208d5 commit 61a0532

File tree

14 files changed

+133
-89
lines changed

14 files changed

+133
-89
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
6060
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
6161
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
62-
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
62+
import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
6363
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
6464
import static org.hamcrest.Matchers.equalTo;
6565
import static org.hamcrest.Matchers.greaterThan;

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
import org.opensearch.indices.IndexCreationException;
9999
import org.opensearch.indices.IndicesService;
100100
import org.opensearch.indices.InvalidIndexNameException;
101+
import org.opensearch.indices.RemoteStoreSettings;
101102
import org.opensearch.indices.ShardLimitValidator;
102103
import org.opensearch.indices.SystemIndices;
103104
import org.opensearch.indices.replication.common.ReplicationType;
@@ -191,7 +192,8 @@ public MetadataCreateIndexService(
191192
final NamedXContentRegistry xContentRegistry,
192193
final SystemIndices systemIndices,
193194
final boolean forbidPrivateIndexSettings,
194-
final AwarenessReplicaBalance awarenessReplicaBalance
195+
final AwarenessReplicaBalance awarenessReplicaBalance,
196+
final RemoteStoreSettings remoteStoreSettings
195197
) {
196198
this.settings = settings;
197199
this.clusterService = clusterService;
@@ -211,7 +213,7 @@ public MetadataCreateIndexService(
211213
createIndexTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.CREATE_INDEX_KEY, true);
212214
Supplier<Version> minNodeVersionSupplier = () -> clusterService.state().nodes().getMinNodeVersion();
213215
remoteStorePathStrategyResolver = isRemoteDataAttributePresent(settings)
214-
? new RemoteStorePathStrategyResolver(clusterService.getClusterSettings(), minNodeVersionSupplier)
216+
? new RemoteStorePathStrategyResolver(remoteStoreSettings, minNodeVersionSupplier)
215217
: null;
216218
}
217219

‎server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -713,8 +713,6 @@ public void apply(Settings value, Settings current, Settings previous) {
713713
RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING,
714714
IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING,
715715
IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING,
716-
IndicesService.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING,
717-
IndicesService.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING,
718716

719717
// Admission Control Settings
720718
AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
@@ -732,7 +730,9 @@ public void apply(Settings value, Settings current, Settings previous) {
732730

733731
RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING,
734732
RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
735-
RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING
733+
RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING,
734+
RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING,
735+
RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING
736736
)
737737
)
738738
);

‎server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategyResolver.java

+6-21
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,9 @@
99
package org.opensearch.index.remote;
1010

1111
import org.opensearch.Version;
12-
import org.opensearch.common.settings.ClusterSettings;
1312
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
1413
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
15-
import org.opensearch.indices.IndicesService;
14+
import org.opensearch.indices.RemoteStoreSettings;
1615

1716
import java.util.function.Supplier;
1817

@@ -23,35 +22,21 @@
2322
*/
2423
public class RemoteStorePathStrategyResolver {
2524

26-
private volatile PathType type;
27-
28-
private volatile PathHashAlgorithm hashAlgorithm;
29-
25+
private final RemoteStoreSettings remoteStoreSettings;
3026
private final Supplier<Version> minNodeVersionSupplier;
3127

32-
public RemoteStorePathStrategyResolver(ClusterSettings clusterSettings, Supplier<Version> minNodeVersionSupplier) {
28+
public RemoteStorePathStrategyResolver(RemoteStoreSettings remoteStoreSettings, Supplier<Version> minNodeVersionSupplier) {
29+
this.remoteStoreSettings = remoteStoreSettings;
3330
this.minNodeVersionSupplier = minNodeVersionSupplier;
34-
type = clusterSettings.get(IndicesService.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING);
35-
hashAlgorithm = clusterSettings.get(IndicesService.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING);
36-
clusterSettings.addSettingsUpdateConsumer(IndicesService.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, this::setType);
37-
clusterSettings.addSettingsUpdateConsumer(IndicesService.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, this::setHashAlgorithm);
3831
}
3932

4033
public RemoteStorePathStrategy get() {
4134
PathType pathType;
4235
PathHashAlgorithm pathHashAlgorithm;
4336
// Min node version check ensures that we are enabling the new prefix type only when all the nodes understand it.
44-
pathType = Version.CURRENT.compareTo(minNodeVersionSupplier.get()) <= 0 ? type : PathType.FIXED;
37+
pathType = Version.CURRENT.compareTo(minNodeVersionSupplier.get()) <= 0 ? remoteStoreSettings.getPathType() : PathType.FIXED;
4538
// If the path type is fixed, hash algorithm is not applicable.
46-
pathHashAlgorithm = pathType == PathType.FIXED ? null : hashAlgorithm;
39+
pathHashAlgorithm = pathType == PathType.FIXED ? null : remoteStoreSettings.getPathHashAlgorithm();
4740
return new RemoteStorePathStrategy(pathType, pathHashAlgorithm);
4841
}
49-
50-
private void setType(PathType type) {
51-
this.type = type;
52-
}
53-
54-
private void setHashAlgorithm(PathHashAlgorithm hashAlgorithm) {
55-
this.hashAlgorithm = hashAlgorithm;
56-
}
5742
}

‎server/src/main/java/org/opensearch/indices/IndicesService.java

-30
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@
6262
import org.opensearch.common.CheckedFunction;
6363
import org.opensearch.common.CheckedSupplier;
6464
import org.opensearch.common.Nullable;
65-
import org.opensearch.common.annotation.ExperimentalApi;
6665
import org.opensearch.common.annotation.PublicApi;
6766
import org.opensearch.common.cache.policy.CachedQueryResult;
6867
import org.opensearch.common.cache.service.CacheService;
@@ -125,8 +124,6 @@
125124
import org.opensearch.index.query.QueryRewriteContext;
126125
import org.opensearch.index.recovery.RecoveryStats;
127126
import org.opensearch.index.refresh.RefreshStats;
128-
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
129-
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
130127
import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory;
131128
import org.opensearch.index.search.stats.SearchStats;
132129
import org.opensearch.index.seqno.RetentionLeaseStats;
@@ -308,33 +305,6 @@ public class IndicesService extends AbstractLifecycleComponent
308305
Property.Final
309306
);
310307

311-
/**
312-
* This setting is used to set the remote store blob store path type strategy. This setting is effective only for
313-
* remote store enabled cluster.
314-
*/
315-
@ExperimentalApi
316-
public static final Setting<PathType> CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING = new Setting<>(
317-
"cluster.remote_store.index.path.type",
318-
PathType.FIXED.toString(),
319-
PathType::parseString,
320-
Property.NodeScope,
321-
Property.Dynamic
322-
);
323-
324-
/**
325-
* This setting is used to set the remote store blob store path hash algorithm strategy. This setting is effective only for
326-
* remote store enabled cluster. This setting will come to effect if the {@link #CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING}
327-
* is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}.
328-
*/
329-
@ExperimentalApi
330-
public static final Setting<PathHashAlgorithm> CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING = new Setting<>(
331-
"cluster.remote_store.index.path.hash_algorithm",
332-
PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString(),
333-
PathHashAlgorithm::parseString,
334-
Property.NodeScope,
335-
Property.Dynamic
336-
);
337-
338308
/**
339309
* The node's settings.
340310
*/

‎server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java

+57-2
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,15 @@
88

99
package org.opensearch.indices;
1010

11+
import org.opensearch.common.annotation.ExperimentalApi;
1112
import org.opensearch.common.annotation.PublicApi;
1213
import org.opensearch.common.settings.ClusterSettings;
1314
import org.opensearch.common.settings.Setting;
1415
import org.opensearch.common.settings.Setting.Property;
1516
import org.opensearch.common.settings.Settings;
1617
import org.opensearch.common.unit.TimeValue;
1718
import org.opensearch.index.IndexSettings;
19+
import org.opensearch.index.remote.RemoteStoreEnums;
1820

1921
/**
2022
* Settings for remote store
@@ -65,12 +67,41 @@ public class RemoteStoreSettings {
6567
Property.Dynamic
6668
);
6769

70+
/**
71+
* This setting is used to set the remote store blob store path type strategy. This setting is effective only for
72+
* remote store enabled cluster.
73+
*/
74+
@ExperimentalApi
75+
public static final Setting<RemoteStoreEnums.PathType> CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING = new Setting<>(
76+
"cluster.remote_store.index.path.type",
77+
RemoteStoreEnums.PathType.FIXED.toString(),
78+
RemoteStoreEnums.PathType::parseString,
79+
Property.NodeScope,
80+
Property.Dynamic
81+
);
82+
83+
/**
84+
* This setting is used to set the remote store blob store path hash algorithm strategy. This setting is effective only for
85+
* remote store enabled cluster. This setting will come to effect if the {@link #CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING}
86+
* is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}.
87+
*/
88+
@ExperimentalApi
89+
public static final Setting<RemoteStoreEnums.PathHashAlgorithm> CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING = new Setting<>(
90+
"cluster.remote_store.index.path.hash_algorithm",
91+
RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString(),
92+
RemoteStoreEnums.PathHashAlgorithm::parseString,
93+
Property.NodeScope,
94+
Property.Dynamic
95+
);
96+
6897
private volatile TimeValue clusterRemoteTranslogBufferInterval;
6998
private volatile int minRemoteSegmentMetadataFiles;
7099
private volatile TimeValue clusterRemoteTranslogTransferTimeout;
100+
private volatile RemoteStoreEnums.PathType pathType;
101+
private volatile RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm;
71102

72103
public RemoteStoreSettings(Settings settings, ClusterSettings clusterSettings) {
73-
this.clusterRemoteTranslogBufferInterval = CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
104+
clusterRemoteTranslogBufferInterval = CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
74105
clusterSettings.addSettingsUpdateConsumer(
75106
CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
76107
this::setClusterRemoteTranslogBufferInterval
@@ -82,11 +113,17 @@ public RemoteStoreSettings(Settings settings, ClusterSettings clusterSettings) {
82113
this::setMinRemoteSegmentMetadataFiles
83114
);
84115

85-
this.clusterRemoteTranslogTransferTimeout = CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING.get(settings);
116+
clusterRemoteTranslogTransferTimeout = CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING.get(settings);
86117
clusterSettings.addSettingsUpdateConsumer(
87118
CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING,
88119
this::setClusterRemoteTranslogTransferTimeout
89120
);
121+
122+
pathType = clusterSettings.get(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING);
123+
clusterSettings.addSettingsUpdateConsumer(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, this::setPathType);
124+
125+
pathHashAlgorithm = clusterSettings.get(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING);
126+
clusterSettings.addSettingsUpdateConsumer(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, this::setPathHashAlgorithm);
90127
}
91128

92129
public TimeValue getClusterRemoteTranslogBufferInterval() {
@@ -112,4 +149,22 @@ public TimeValue getClusterRemoteTranslogTransferTimeout() {
112149
private void setClusterRemoteTranslogTransferTimeout(TimeValue clusterRemoteTranslogTransferTimeout) {
113150
this.clusterRemoteTranslogTransferTimeout = clusterRemoteTranslogTransferTimeout;
114151
}
152+
153+
@ExperimentalApi
154+
public RemoteStoreEnums.PathType getPathType() {
155+
return pathType;
156+
}
157+
158+
@ExperimentalApi
159+
public RemoteStoreEnums.PathHashAlgorithm getPathHashAlgorithm() {
160+
return pathHashAlgorithm;
161+
}
162+
163+
private void setPathType(RemoteStoreEnums.PathType pathType) {
164+
this.pathType = pathType;
165+
}
166+
167+
private void setPathHashAlgorithm(RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm) {
168+
this.pathHashAlgorithm = pathHashAlgorithm;
169+
}
115170
}

‎server/src/main/java/org/opensearch/node/Node.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -863,7 +863,8 @@ protected Node(
863863
xContentRegistry,
864864
systemIndices,
865865
forbidPrivateIndexSettings,
866-
awarenessReplicaBalance
866+
awarenessReplicaBalance,
867+
remoteStoreSettings
867868
);
868869
pluginsService.filterPlugins(Plugin.class)
869870
.forEach(

‎server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

+7-3
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
import org.opensearch.index.mapper.MetadataFieldMapper;
8282
import org.opensearch.index.mapper.RoutingFieldMapper;
8383
import org.opensearch.index.shard.IndexEventListener;
84+
import org.opensearch.indices.DefaultRemoteStoreSettings;
8485
import org.opensearch.indices.IndicesService;
8586
import org.opensearch.indices.InvalidIndexNameException;
8687
import org.opensearch.indices.ShardLimitValidator;
@@ -738,7 +739,8 @@ public void testRolloverClusterState() throws Exception {
738739
null,
739740
systemIndices,
740741
false,
741-
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
742+
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
743+
DefaultRemoteStoreSettings.INSTANCE
742744
);
743745
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(
744746
clusterService,
@@ -876,7 +878,8 @@ public void testRolloverClusterStateForDataStream() throws Exception {
876878
null,
877879
systemIndices,
878880
false,
879-
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
881+
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
882+
DefaultRemoteStoreSettings.INSTANCE
880883
);
881884
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(
882885
clusterService,
@@ -1054,7 +1057,8 @@ public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception {
10541057
null,
10551058
new SystemIndices(emptyMap()),
10561059
false,
1057-
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
1060+
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
1061+
DefaultRemoteStoreSettings.INSTANCE
10581062
);
10591063
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(
10601064
clusterService,

0 commit comments

Comments
 (0)