Skip to content

Commit 6267e94

Browse files
Star tree mapping changes (#14605)
* Star tree mapping changes with feature flag --------- Signed-off-by: Bharathwaj G <bharath78910@gmail.com>
1 parent c71fd4a commit 6267e94

39 files changed

+2951
-15
lines changed

distribution/src/config/opensearch.yml

+4
Original file line numberDiff line numberDiff line change
@@ -125,3 +125,7 @@ ${path.logs}
125125
# Gates the functionality of enabling Opensearch to use pluggable caches with respective store names via setting.
126126
#
127127
#opensearch.experimental.feature.pluggable.caching.enabled: false
128+
#
129+
# Gates the functionality of star tree index, which improves the performance of search aggregations.
130+
#
131+
#opensearch.experimental.feature.composite_index.star_tree.enabled: true

server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

+440
Large diffs are not rendered by default.

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

+5
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
import org.opensearch.index.IndexNotFoundException;
8686
import org.opensearch.index.IndexService;
8787
import org.opensearch.index.IndexSettings;
88+
import org.opensearch.index.compositeindex.CompositeIndexValidator;
8889
import org.opensearch.index.mapper.DocumentMapper;
8990
import org.opensearch.index.mapper.MapperService;
9091
import org.opensearch.index.mapper.MapperService.MergeReason;
@@ -1318,6 +1319,10 @@ private static void updateIndexMappingsAndBuildSortOrder(
13181319
}
13191320
}
13201321

1322+
if (mapperService.isCompositeIndexPresent()) {
1323+
CompositeIndexValidator.validate(mapperService, indexService.getCompositeIndexSettings(), indexService.getIndexSettings());
1324+
}
1325+
13211326
if (sourceMetadata == null) {
13221327
// now that the mapping is merged we can validate the index sort.
13231328
// we cannot validate for index shrinking since the mapping is empty

server/src/main/java/org/opensearch/cluster/metadata/MetadataMappingService.java

+11-1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.opensearch.core.common.Strings;
5656
import org.opensearch.core.index.Index;
5757
import org.opensearch.index.IndexService;
58+
import org.opensearch.index.compositeindex.CompositeIndexValidator;
5859
import org.opensearch.index.mapper.DocumentMapper;
5960
import org.opensearch.index.mapper.MapperService;
6061
import org.opensearch.index.mapper.MapperService.MergeReason;
@@ -282,6 +283,7 @@ private ClusterState applyRequest(
282283
// first, simulate: just call merge and ignore the result
283284
existingMapper.merge(newMapper.mapping(), MergeReason.MAPPING_UPDATE);
284285
}
286+
285287
}
286288
Metadata.Builder builder = Metadata.builder(metadata);
287289
boolean updated = false;
@@ -291,7 +293,7 @@ private ClusterState applyRequest(
291293
// we use the exact same indexService and metadata we used to validate above here to actually apply the update
292294
final Index index = indexMetadata.getIndex();
293295
final MapperService mapperService = indexMapperServices.get(index);
294-
296+
boolean isCompositeFieldPresent = !mapperService.getCompositeFieldTypes().isEmpty();
295297
CompressedXContent existingSource = null;
296298
DocumentMapper existingMapper = mapperService.documentMapper();
297299
if (existingMapper != null) {
@@ -302,6 +304,14 @@ private ClusterState applyRequest(
302304
mappingUpdateSource,
303305
MergeReason.MAPPING_UPDATE
304306
);
307+
308+
CompositeIndexValidator.validate(
309+
mapperService,
310+
indicesService.getCompositeIndexSettings(),
311+
mapperService.getIndexSettings(),
312+
isCompositeFieldPresent
313+
);
314+
305315
CompressedXContent updatedSource = mergedMapper.mappingSource();
306316

307317
if (existingSource != null) {

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@
115115
import org.opensearch.index.ShardIndexingPressureMemoryManager;
116116
import org.opensearch.index.ShardIndexingPressureSettings;
117117
import org.opensearch.index.ShardIndexingPressureStore;
118+
import org.opensearch.index.compositeindex.CompositeIndexSettings;
118119
import org.opensearch.index.remote.RemoteStorePressureSettings;
119120
import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory;
120121
import org.opensearch.index.store.remote.filecache.FileCacheSettings;
@@ -754,7 +755,10 @@ public void apply(Settings value, Settings current, Settings previous) {
754755
RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING,
755756
RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS,
756757
RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA,
757-
SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING
758+
SearchService.CLUSTER_ALLOW_DERIVED_FIELD_SETTING,
759+
760+
// Composite index settings
761+
CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING
758762
)
759763
)
760764
);

server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ protected FeatureFlagSettings(
3737
FeatureFlags.TIERED_REMOTE_INDEX_SETTING,
3838
FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
3939
FeatureFlags.PLUGGABLE_CACHE_SETTING,
40-
FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING
40+
FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL_SETTING,
41+
FeatureFlags.STAR_TREE_INDEX_SETTING
4142
);
4243
}

server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

+10
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.opensearch.index.SearchSlowLog;
5353
import org.opensearch.index.TieredMergePolicyProvider;
5454
import org.opensearch.index.cache.bitset.BitsetFilterCache;
55+
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
5556
import org.opensearch.index.engine.EngineConfig;
5657
import org.opensearch.index.fielddata.IndexFieldDataService;
5758
import org.opensearch.index.mapper.FieldMapper;
@@ -239,6 +240,15 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
239240
// Settings for concurrent segment search
240241
IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING,
241242
IndexSettings.ALLOW_DERIVED_FIELDS,
243+
244+
// Settings for star tree index
245+
StarTreeIndexSettings.STAR_TREE_DEFAULT_MAX_LEAF_DOCS,
246+
StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING,
247+
StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING,
248+
StarTreeIndexSettings.DEFAULT_METRICS_LIST,
249+
StarTreeIndexSettings.DEFAULT_DATE_INTERVALS,
250+
StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING,
251+
242252
// validate that built-in similarities don't get redefined
243253
Setting.groupSetting("index.similarity.", (s) -> {
244254
Map<String, Settings> groups = s.getAsGroups();

server/src/main/java/org/opensearch/common/util/FeatureFlags.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,13 @@ public class FeatureFlags {
100100
Property.NodeScope
101101
);
102102

103+
/**
104+
* Gates the functionality of star tree index, which improves the performance of search
105+
* aggregations.
106+
*/
107+
public static final String STAR_TREE_INDEX = "opensearch.experimental.feature.composite_index.star_tree.enabled";
108+
public static final Setting<Boolean> STAR_TREE_INDEX_SETTING = Setting.boolSetting(STAR_TREE_INDEX, false, Property.NodeScope);
109+
103110
private static final List<Setting<Boolean>> ALL_FEATURE_FLAG_SETTINGS = List.of(
104111
REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
105112
EXTENSIONS_SETTING,
@@ -108,7 +115,8 @@ public class FeatureFlags {
108115
DATETIME_FORMATTER_CACHING_SETTING,
109116
TIERED_REMOTE_INDEX_SETTING,
110117
PLUGGABLE_CACHE_SETTING,
111-
REMOTE_PUBLICATION_EXPERIMENTAL_SETTING
118+
REMOTE_PUBLICATION_EXPERIMENTAL_SETTING,
119+
STAR_TREE_INDEX_SETTING
112120
);
113121
/**
114122
* Should store the settings from opensearch.yml.

server/src/main/java/org/opensearch/index/IndexModule.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@
6666
import org.opensearch.index.cache.query.DisabledQueryCache;
6767
import org.opensearch.index.cache.query.IndexQueryCache;
6868
import org.opensearch.index.cache.query.QueryCache;
69+
import org.opensearch.index.compositeindex.CompositeIndexSettings;
6970
import org.opensearch.index.engine.Engine;
7071
import org.opensearch.index.engine.EngineConfigFactory;
7172
import org.opensearch.index.engine.EngineFactory;
@@ -311,6 +312,7 @@ public Iterator<Setting<?>> settings() {
311312
private final BooleanSupplier allowExpensiveQueries;
312313
private final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories;
313314
private final FileCache fileCache;
315+
private final CompositeIndexSettings compositeIndexSettings;
314316

315317
/**
316318
* Construct the index module for the index with the specified index settings. The index module contains extension points for plugins
@@ -330,7 +332,8 @@ public IndexModule(
330332
final BooleanSupplier allowExpensiveQueries,
331333
final IndexNameExpressionResolver expressionResolver,
332334
final Map<String, IndexStorePlugin.RecoveryStateFactory> recoveryStateFactories,
333-
final FileCache fileCache
335+
final FileCache fileCache,
336+
final CompositeIndexSettings compositeIndexSettings
334337
) {
335338
this.indexSettings = indexSettings;
336339
this.analysisRegistry = analysisRegistry;
@@ -343,6 +346,7 @@ public IndexModule(
343346
this.expressionResolver = expressionResolver;
344347
this.recoveryStateFactories = recoveryStateFactories;
345348
this.fileCache = fileCache;
349+
this.compositeIndexSettings = compositeIndexSettings;
346350
}
347351

348352
public IndexModule(
@@ -364,6 +368,7 @@ public IndexModule(
364368
allowExpensiveQueries,
365369
expressionResolver,
366370
recoveryStateFactories,
371+
null,
367372
null
368373
);
369374
}
@@ -739,7 +744,8 @@ public IndexService newIndexService(
739744
clusterDefaultRefreshIntervalSupplier,
740745
recoverySettings,
741746
remoteStoreSettings,
742-
fileCache
747+
fileCache,
748+
compositeIndexSettings
743749
);
744750
success = true;
745751
return indexService;

server/src/main/java/org/opensearch/index/IndexService.java

+10-1
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
import org.opensearch.index.cache.IndexCache;
7474
import org.opensearch.index.cache.bitset.BitsetFilterCache;
7575
import org.opensearch.index.cache.query.QueryCache;
76+
import org.opensearch.index.compositeindex.CompositeIndexSettings;
7677
import org.opensearch.index.engine.Engine;
7778
import org.opensearch.index.engine.EngineConfigFactory;
7879
import org.opensearch.index.engine.EngineFactory;
@@ -192,6 +193,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
192193
private final RecoverySettings recoverySettings;
193194
private final RemoteStoreSettings remoteStoreSettings;
194195
private final FileCache fileCache;
196+
private final CompositeIndexSettings compositeIndexSettings;
195197

196198
public IndexService(
197199
IndexSettings indexSettings,
@@ -228,7 +230,8 @@ public IndexService(
228230
Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier,
229231
RecoverySettings recoverySettings,
230232
RemoteStoreSettings remoteStoreSettings,
231-
FileCache fileCache
233+
FileCache fileCache,
234+
CompositeIndexSettings compositeIndexSettings
232235
) {
233236
super(indexSettings);
234237
this.allowExpensiveQueries = allowExpensiveQueries;
@@ -306,6 +309,7 @@ public IndexService(
306309
this.translogFactorySupplier = translogFactorySupplier;
307310
this.recoverySettings = recoverySettings;
308311
this.remoteStoreSettings = remoteStoreSettings;
312+
this.compositeIndexSettings = compositeIndexSettings;
309313
this.fileCache = fileCache;
310314
updateFsyncTaskIfNecessary();
311315
}
@@ -381,6 +385,7 @@ public IndexService(
381385
clusterDefaultRefreshIntervalSupplier,
382386
recoverySettings,
383387
remoteStoreSettings,
388+
null,
384389
null
385390
);
386391
}
@@ -1110,6 +1115,10 @@ private void rescheduleRefreshTasks() {
11101115
}
11111116
}
11121117

1118+
public CompositeIndexSettings getCompositeIndexSettings() {
1119+
return compositeIndexSettings;
1120+
}
1121+
11131122
/**
11141123
* Shard Store Deleter Interface
11151124
*
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.index.compositeindex;
10+
11+
import org.opensearch.common.annotation.ExperimentalApi;
12+
import org.opensearch.common.settings.ClusterSettings;
13+
import org.opensearch.common.settings.Setting;
14+
import org.opensearch.common.settings.Settings;
15+
import org.opensearch.common.util.FeatureFlags;
16+
17+
/**
18+
* Cluster level settings for composite indices
19+
*
20+
* @opensearch.experimental
21+
*/
22+
@ExperimentalApi
23+
public class CompositeIndexSettings {
24+
public static final Setting<Boolean> STAR_TREE_INDEX_ENABLED_SETTING = Setting.boolSetting(
25+
"indices.composite_index.star_tree.enabled",
26+
false,
27+
value -> {
28+
if (FeatureFlags.isEnabled(FeatureFlags.STAR_TREE_INDEX_SETTING) == false && value == true) {
29+
throw new IllegalArgumentException(
30+
"star tree index is under an experimental feature and can be activated only by enabling "
31+
+ FeatureFlags.STAR_TREE_INDEX_SETTING.getKey()
32+
+ " feature flag in the JVM options"
33+
);
34+
}
35+
},
36+
Setting.Property.NodeScope,
37+
Setting.Property.Dynamic
38+
);
39+
40+
private volatile boolean starTreeIndexCreationEnabled;
41+
42+
public CompositeIndexSettings(Settings settings, ClusterSettings clusterSettings) {
43+
this.starTreeIndexCreationEnabled = STAR_TREE_INDEX_ENABLED_SETTING.get(settings);
44+
clusterSettings.addSettingsUpdateConsumer(STAR_TREE_INDEX_ENABLED_SETTING, this::starTreeIndexCreationEnabled);
45+
46+
}
47+
48+
private void starTreeIndexCreationEnabled(boolean value) {
49+
this.starTreeIndexCreationEnabled = value;
50+
}
51+
52+
public boolean isStarTreeIndexCreationEnabled() {
53+
return starTreeIndexCreationEnabled;
54+
}
55+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.index.compositeindex;
10+
11+
import org.opensearch.common.annotation.ExperimentalApi;
12+
import org.opensearch.index.IndexSettings;
13+
import org.opensearch.index.compositeindex.datacube.startree.StarTreeValidator;
14+
import org.opensearch.index.mapper.MapperService;
15+
16+
import java.util.Locale;
17+
18+
/**
19+
* Validation for composite indices as part of mappings
20+
*
21+
* @opensearch.experimental
22+
*/
23+
@ExperimentalApi
24+
public class CompositeIndexValidator {
25+
26+
public static void validate(MapperService mapperService, CompositeIndexSettings compositeIndexSettings, IndexSettings indexSettings) {
27+
StarTreeValidator.validate(mapperService, compositeIndexSettings, indexSettings);
28+
}
29+
30+
public static void validate(
31+
MapperService mapperService,
32+
CompositeIndexSettings compositeIndexSettings,
33+
IndexSettings indexSettings,
34+
boolean isCompositeFieldPresent
35+
) {
36+
if (!isCompositeFieldPresent && mapperService.isCompositeIndexPresent()) {
37+
throw new IllegalArgumentException(
38+
String.format(
39+
Locale.ROOT,
40+
"Composite fields must be specified during index creation, addition of new composite fields during update is not supported"
41+
)
42+
);
43+
}
44+
StarTreeValidator.validate(mapperService, compositeIndexSettings, indexSettings);
45+
}
46+
}

0 commit comments

Comments
 (0)