Skip to content

Commit ed21977

Browse files
ashking94ashking94
committed
Add remote path settings to RemoteStoreSettings
Signed-off-by: Ashish Singh <ssashish@amazon.com>
1 parent 5e72e1d commit ed21977

File tree

14 files changed

+128
-86
lines changed

14 files changed

+128
-86
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
6060
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
6161
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
62-
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
62+
import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
6363
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
6464
import static org.hamcrest.Matchers.equalTo;
6565
import static org.hamcrest.Matchers.greaterThan;

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
import org.opensearch.indices.IndexCreationException;
9999
import org.opensearch.indices.IndicesService;
100100
import org.opensearch.indices.InvalidIndexNameException;
101+
import org.opensearch.indices.RemoteStoreSettings;
101102
import org.opensearch.indices.ShardLimitValidator;
102103
import org.opensearch.indices.SystemIndices;
103104
import org.opensearch.indices.replication.common.ReplicationType;
@@ -191,7 +192,8 @@ public MetadataCreateIndexService(
191192
final NamedXContentRegistry xContentRegistry,
192193
final SystemIndices systemIndices,
193194
final boolean forbidPrivateIndexSettings,
194-
final AwarenessReplicaBalance awarenessReplicaBalance
195+
final AwarenessReplicaBalance awarenessReplicaBalance,
196+
final RemoteStoreSettings remoteStoreSettings
195197
) {
196198
this.settings = settings;
197199
this.clusterService = clusterService;
@@ -211,7 +213,7 @@ public MetadataCreateIndexService(
211213
createIndexTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.CREATE_INDEX_KEY, true);
212214
Supplier<Version> minNodeVersionSupplier = () -> clusterService.state().nodes().getMinNodeVersion();
213215
remoteStorePathStrategyResolver = isRemoteDataAttributePresent(settings)
214-
? new RemoteStorePathStrategyResolver(clusterService.getClusterSettings(), minNodeVersionSupplier)
216+
? new RemoteStorePathStrategyResolver(remoteStoreSettings, minNodeVersionSupplier)
215217
: null;
216218
}
217219

‎server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -713,8 +713,6 @@ public void apply(Settings value, Settings current, Settings previous) {
713713
RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING,
714714
IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING,
715715
IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING,
716-
IndicesService.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING,
717-
IndicesService.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING,
718716

719717
// Admission Control Settings
720718
AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
@@ -732,7 +730,9 @@ public void apply(Settings value, Settings current, Settings previous) {
732730

733731
RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING,
734732
RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
735-
RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING
733+
RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING,
734+
RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING,
735+
RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING
736736
)
737737
)
738738
);

‎server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategyResolver.java

+6-21
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,9 @@
99
package org.opensearch.index.remote;
1010

1111
import org.opensearch.Version;
12-
import org.opensearch.common.settings.ClusterSettings;
1312
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
1413
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
15-
import org.opensearch.indices.IndicesService;
14+
import org.opensearch.indices.RemoteStoreSettings;
1615

1716
import java.util.function.Supplier;
1817

@@ -23,35 +22,21 @@
2322
*/
2423
public class RemoteStorePathStrategyResolver {
2524

26-
private volatile PathType type;
27-
28-
private volatile PathHashAlgorithm hashAlgorithm;
29-
25+
private final RemoteStoreSettings remoteStoreSettings;
3026
private final Supplier<Version> minNodeVersionSupplier;
3127

32-
public RemoteStorePathStrategyResolver(ClusterSettings clusterSettings, Supplier<Version> minNodeVersionSupplier) {
28+
public RemoteStorePathStrategyResolver(RemoteStoreSettings remoteStoreSettings, Supplier<Version> minNodeVersionSupplier) {
29+
this.remoteStoreSettings = remoteStoreSettings;
3330
this.minNodeVersionSupplier = minNodeVersionSupplier;
34-
type = clusterSettings.get(IndicesService.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING);
35-
hashAlgorithm = clusterSettings.get(IndicesService.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING);
36-
clusterSettings.addSettingsUpdateConsumer(IndicesService.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, this::setType);
37-
clusterSettings.addSettingsUpdateConsumer(IndicesService.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, this::setHashAlgorithm);
3831
}
3932

4033
public RemoteStorePathStrategy get() {
4134
PathType pathType;
4235
PathHashAlgorithm pathHashAlgorithm;
4336
// Min node version check ensures that we are enabling the new prefix type only when all the nodes understand it.
44-
pathType = Version.CURRENT.compareTo(minNodeVersionSupplier.get()) <= 0 ? type : PathType.FIXED;
37+
pathType = Version.CURRENT.compareTo(minNodeVersionSupplier.get()) <= 0 ? remoteStoreSettings.getPathType() : PathType.FIXED;
4538
// If the path type is fixed, hash algorithm is not applicable.
46-
pathHashAlgorithm = pathType == PathType.FIXED ? null : hashAlgorithm;
39+
pathHashAlgorithm = pathType == PathType.FIXED ? null : remoteStoreSettings.getPathHashAlgorithm();
4740
return new RemoteStorePathStrategy(pathType, pathHashAlgorithm);
4841
}
49-
50-
private void setType(PathType type) {
51-
this.type = type;
52-
}
53-
54-
private void setHashAlgorithm(PathHashAlgorithm hashAlgorithm) {
55-
this.hashAlgorithm = hashAlgorithm;
56-
}
5742
}

‎server/src/main/java/org/opensearch/indices/IndicesService.java

-27
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,6 @@
124124
import org.opensearch.index.query.QueryRewriteContext;
125125
import org.opensearch.index.recovery.RecoveryStats;
126126
import org.opensearch.index.refresh.RefreshStats;
127-
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
128-
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
129127
import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory;
130128
import org.opensearch.index.search.stats.SearchStats;
131129
import org.opensearch.index.seqno.RetentionLeaseStats;
@@ -307,31 +305,6 @@ public class IndicesService extends AbstractLifecycleComponent
307305
Property.Final
308306
);
309307

310-
/**
311-
* This setting is used to set the remote store blob store path type strategy. This setting is effective only for
312-
* remote store enabled cluster.
313-
*/
314-
public static final Setting<PathType> CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING = new Setting<>(
315-
"cluster.remote_store.index.path.type",
316-
PathType.FIXED.toString(),
317-
PathType::parseString,
318-
Property.NodeScope,
319-
Property.Dynamic
320-
);
321-
322-
/**
323-
* This setting is used to set the remote store blob store path hash algorithm strategy. This setting is effective only for
324-
* remote store enabled cluster. This setting will come to effect if the {@link #CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING}
325-
* is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}.
326-
*/
327-
public static final Setting<PathHashAlgorithm> CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING = new Setting<>(
328-
"cluster.remote_store.index.path.hash_algorithm",
329-
PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString(),
330-
PathHashAlgorithm::parseString,
331-
Property.NodeScope,
332-
Property.Dynamic
333-
);
334-
335308
/**
336309
* The node's settings.
337310
*/

‎server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java

+52-2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.opensearch.common.settings.Settings;
1616
import org.opensearch.common.unit.TimeValue;
1717
import org.opensearch.index.IndexSettings;
18+
import org.opensearch.index.remote.RemoteStoreEnums;
1819

1920
/**
2021
* Settings for remote store
@@ -65,12 +66,39 @@ public class RemoteStoreSettings {
6566
Property.Dynamic
6667
);
6768

69+
/**
70+
* This setting is used to set the remote store blob store path type strategy. This setting is effective only for
71+
* remote store enabled cluster.
72+
*/
73+
public static final Setting<RemoteStoreEnums.PathType> CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING = new Setting<>(
74+
"cluster.remote_store.index.path.type",
75+
RemoteStoreEnums.PathType.FIXED.toString(),
76+
RemoteStoreEnums.PathType::parseString,
77+
Property.NodeScope,
78+
Property.Dynamic
79+
);
80+
81+
/**
82+
* This setting is used to set the remote store blob store path hash algorithm strategy. This setting is effective only for
83+
* remote store enabled cluster. This setting will come to effect if the {@link #CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING}
84+
* is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}.
85+
*/
86+
public static final Setting<RemoteStoreEnums.PathHashAlgorithm> CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING = new Setting<>(
87+
"cluster.remote_store.index.path.hash_algorithm",
88+
RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString(),
89+
RemoteStoreEnums.PathHashAlgorithm::parseString,
90+
Property.NodeScope,
91+
Property.Dynamic
92+
);
93+
6894
private volatile TimeValue clusterRemoteTranslogBufferInterval;
6995
private volatile int minRemoteSegmentMetadataFiles;
7096
private volatile TimeValue clusterRemoteTranslogTransferTimeout;
97+
private volatile RemoteStoreEnums.PathType pathType;
98+
private volatile RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm;
7199

72100
public RemoteStoreSettings(Settings settings, ClusterSettings clusterSettings) {
73-
this.clusterRemoteTranslogBufferInterval = CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
101+
clusterRemoteTranslogBufferInterval = CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
74102
clusterSettings.addSettingsUpdateConsumer(
75103
CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
76104
this::setClusterRemoteTranslogBufferInterval
@@ -82,11 +110,17 @@ public RemoteStoreSettings(Settings settings, ClusterSettings clusterSettings) {
82110
this::setMinRemoteSegmentMetadataFiles
83111
);
84112

85-
this.clusterRemoteTranslogTransferTimeout = CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING.get(settings);
113+
clusterRemoteTranslogTransferTimeout = CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING.get(settings);
86114
clusterSettings.addSettingsUpdateConsumer(
87115
CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING,
88116
this::setClusterRemoteTranslogTransferTimeout
89117
);
118+
119+
pathType = clusterSettings.get(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING);
120+
clusterSettings.addSettingsUpdateConsumer(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, this::setPathType);
121+
122+
pathHashAlgorithm = clusterSettings.get(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING);
123+
clusterSettings.addSettingsUpdateConsumer(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, this::setPathHashAlgorithm);
90124
}
91125

92126
public TimeValue getClusterRemoteTranslogBufferInterval() {
@@ -112,4 +146,20 @@ public TimeValue getClusterRemoteTranslogTransferTimeout() {
112146
private void setClusterRemoteTranslogTransferTimeout(TimeValue clusterRemoteTranslogTransferTimeout) {
113147
this.clusterRemoteTranslogTransferTimeout = clusterRemoteTranslogTransferTimeout;
114148
}
149+
150+
public RemoteStoreEnums.PathType getPathType() {
151+
return pathType;
152+
}
153+
154+
public RemoteStoreEnums.PathHashAlgorithm getPathHashAlgorithm() {
155+
return pathHashAlgorithm;
156+
}
157+
158+
private void setPathType(RemoteStoreEnums.PathType pathType) {
159+
this.pathType = pathType;
160+
}
161+
162+
private void setPathHashAlgorithm(RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm) {
163+
this.pathHashAlgorithm = pathHashAlgorithm;
164+
}
115165
}

‎server/src/main/java/org/opensearch/node/Node.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -863,7 +863,8 @@ protected Node(
863863
xContentRegistry,
864864
systemIndices,
865865
forbidPrivateIndexSettings,
866-
awarenessReplicaBalance
866+
awarenessReplicaBalance,
867+
remoteStoreSettings
867868
);
868869
pluginsService.filterPlugins(Plugin.class)
869870
.forEach(

‎server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java

+7-3
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
import org.opensearch.index.mapper.MetadataFieldMapper;
8282
import org.opensearch.index.mapper.RoutingFieldMapper;
8383
import org.opensearch.index.shard.IndexEventListener;
84+
import org.opensearch.indices.DefaultRemoteStoreSettings;
8485
import org.opensearch.indices.IndicesService;
8586
import org.opensearch.indices.InvalidIndexNameException;
8687
import org.opensearch.indices.ShardLimitValidator;
@@ -738,7 +739,8 @@ public void testRolloverClusterState() throws Exception {
738739
null,
739740
systemIndices,
740741
false,
741-
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
742+
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
743+
DefaultRemoteStoreSettings.INSTANCE
742744
);
743745
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(
744746
clusterService,
@@ -876,7 +878,8 @@ public void testRolloverClusterStateForDataStream() throws Exception {
876878
null,
877879
systemIndices,
878880
false,
879-
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
881+
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
882+
DefaultRemoteStoreSettings.INSTANCE
880883
);
881884
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(
882885
clusterService,
@@ -1054,7 +1057,8 @@ public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception {
10541057
null,
10551058
new SystemIndices(emptyMap()),
10561059
false,
1057-
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
1060+
new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
1061+
DefaultRemoteStoreSettings.INSTANCE
10581062
);
10591063
MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(
10601064
clusterService,

0 commit comments

Comments
 (0)