Skip to content

Commit 5875393

Browse files
bharath-techiebharath-techie
authored
[Backport 2.x] [Star tree] Star tree index validations (opensearch-project#15533) (opensearch-project#15610)
--------- Signed-off-by: Bharathwaj G <bharath78910@gmail.com>
1 parent 01b38be commit 5875393

18 files changed

+863
-26
lines changed

‎server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

+336-11
Large diffs are not rendered by default.

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

+70
Original file line numberDiff line numberDiff line change
@@ -82,14 +82,17 @@
8282
import org.opensearch.common.xcontent.XContentHelper;
8383
import org.opensearch.core.action.ActionListener;
8484
import org.opensearch.core.common.Strings;
85+
import org.opensearch.core.common.unit.ByteSizeValue;
8586
import org.opensearch.core.index.Index;
8687
import org.opensearch.core.xcontent.NamedXContentRegistry;
8788
import org.opensearch.env.Environment;
8889
import org.opensearch.index.IndexModule;
8990
import org.opensearch.index.IndexNotFoundException;
9091
import org.opensearch.index.IndexService;
9192
import org.opensearch.index.IndexSettings;
93+
import org.opensearch.index.compositeindex.CompositeIndexSettings;
9294
import org.opensearch.index.compositeindex.CompositeIndexValidator;
95+
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
9396
import org.opensearch.index.mapper.DocumentMapper;
9497
import org.opensearch.index.mapper.MapperService;
9598
import org.opensearch.index.mapper.MapperService.MergeReason;
@@ -154,6 +157,7 @@
154157
import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING;
155158
import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findContextTemplateName;
156159
import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
160+
import static org.opensearch.index.IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING;
157161
import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
158162
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent;
159163
import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
@@ -1079,6 +1083,7 @@ static Settings aggregateIndexSettings(
10791083
validateTranslogRetentionSettings(indexSettings);
10801084
validateStoreTypeSettings(indexSettings);
10811085
validateRefreshIntervalSettings(request.settings(), clusterSettings);
1086+
validateTranslogFlushIntervalSettingsForCompositeIndex(request.settings(), clusterSettings);
10821087
validateTranslogDurabilitySettings(request.settings(), clusterSettings, settings);
10831088
return indexSettings;
10841089
}
@@ -1766,6 +1771,71 @@ public static void validateTranslogRetentionSettings(Settings indexSettings) {
17661771
}
17671772
}
17681773

1774+
/**
1775+
* Validates {@code index.translog.flush_threshold_size} is equal or below the {@code indices.composite_index.translog.max_flush_threshold_size}
1776+
* for composite indices based on {{@code index.composite_index}}
1777+
*
1778+
* @param requestSettings settings passed in during index create/update request
1779+
* @param clusterSettings cluster setting
1780+
*/
1781+
public static void validateTranslogFlushIntervalSettingsForCompositeIndex(Settings requestSettings, ClusterSettings clusterSettings) {
1782+
if (StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.exists(requestSettings) == false
1783+
|| requestSettings.get(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey()) == null) {
1784+
return;
1785+
}
1786+
ByteSizeValue translogFlushSize = INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.get(requestSettings);
1787+
ByteSizeValue compositeIndexMaxFlushSize = clusterSettings.get(
1788+
CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING
1789+
);
1790+
if (translogFlushSize.compareTo(compositeIndexMaxFlushSize) > 0) {
1791+
throw new IllegalArgumentException(
1792+
String.format(
1793+
Locale.ROOT,
1794+
"You can configure '%s' with upto '%s' for composite index",
1795+
INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
1796+
compositeIndexMaxFlushSize
1797+
)
1798+
);
1799+
}
1800+
}
1801+
1802+
/**
1803+
* Validates {@code index.translog.flush_threshold_size} is equal or below the {@code indices.composite_index.translog.max_flush_threshold_size}
1804+
* for composite indices based on {{@code index.composite_index}}
1805+
* This is used during update index settings flow
1806+
*
1807+
* @param requestSettings settings passed in during index update request
1808+
* @param clusterSettings cluster setting
1809+
* @param indexSettings index settings
1810+
*/
1811+
public static Optional<String> validateTranslogFlushIntervalSettingsForCompositeIndex(
1812+
Settings requestSettings,
1813+
ClusterSettings clusterSettings,
1814+
Settings indexSettings
1815+
) {
1816+
if (INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.exists(requestSettings) == false
1817+
|| requestSettings.get(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey()) == null
1818+
|| StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.exists(indexSettings) == false
1819+
|| indexSettings.get(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey()) == null) {
1820+
return Optional.empty();
1821+
}
1822+
ByteSizeValue translogFlushSize = INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.get(requestSettings);
1823+
ByteSizeValue compositeIndexMaxFlushSize = clusterSettings.get(
1824+
CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING
1825+
);
1826+
if (translogFlushSize.compareTo(compositeIndexMaxFlushSize) > 0) {
1827+
return Optional.of(
1828+
String.format(
1829+
Locale.ROOT,
1830+
"You can configure '%s' with upto '%s' for composite index",
1831+
INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
1832+
compositeIndexMaxFlushSize
1833+
)
1834+
);
1835+
}
1836+
return Optional.empty();
1837+
}
1838+
17691839
/**
17701840
* Validates {@code index.refresh_interval} is equal or below the {@code cluster.minimum.index.refresh_interval}.
17711841
*

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java

+2
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@
102102

103103
import static org.opensearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping;
104104
import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateRefreshIntervalSettings;
105+
import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex;
105106
import static org.opensearch.common.util.concurrent.ThreadContext.ACTION_ORIGIN_TRANSIENT_NAME;
106107
import static org.opensearch.indices.cluster.IndicesClusterStateService.AllocatedIndices.IndexRemovalReason.NO_LONGER_ASSIGNED;
107108

@@ -1639,6 +1640,7 @@ private void validate(String name, @Nullable Settings settings, List<String> ind
16391640

16401641
// validate index refresh interval and translog durability settings
16411642
validateRefreshIntervalSettings(settings, clusterService.getClusterSettings());
1643+
validateTranslogFlushIntervalSettingsForCompositeIndex(settings, clusterService.getClusterSettings());
16421644
validateTranslogDurabilitySettingsInTemplate(settings, clusterService.getClusterSettings());
16431645
}
16441646

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java

+7
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateOverlap;
8383
import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateRefreshIntervalSettings;
8484
import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogDurabilitySettings;
85+
import static org.opensearch.cluster.metadata.MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex;
8586
import static org.opensearch.cluster.metadata.MetadataIndexTemplateService.findComponentTemplate;
8687
import static org.opensearch.common.settings.AbstractScopedSettings.ARCHIVED_SETTINGS_PREFIX;
8788
import static org.opensearch.index.IndexSettings.same;
@@ -221,6 +222,12 @@ public ClusterState execute(ClusterState currentState) {
221222
index.getName()
222223
).ifPresent(validationErrors::add);
223224
}
225+
validateTranslogFlushIntervalSettingsForCompositeIndex(
226+
normalizedSettings,
227+
clusterService.getClusterSettings(),
228+
metadata.getSettings()
229+
).ifPresent(validationErrors::add);
230+
224231
}
225232

226233
if (validationErrors.size() > 0) {

‎server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

+4
Original file line numberDiff line numberDiff line change
@@ -770,6 +770,10 @@ public void apply(Settings value, Settings current, Settings previous) {
770770
RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_LOOKBACK_INTERVAL,
771771
RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED,
772772

773+
// Composite index settings
774+
CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING,
775+
CompositeIndexSettings.COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING,
776+
773777
SystemTemplatesService.SETTING_APPLICATION_BASED_CONFIGURATION_TEMPLATES_ENABLED,
774778

775779
// WorkloadManagement settings

‎server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

+2
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
250250
StarTreeIndexSettings.DEFAULT_METRICS_LIST,
251251
StarTreeIndexSettings.DEFAULT_DATE_INTERVALS,
252252
StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING,
253+
StarTreeIndexSettings.STAR_TREE_MAX_BASE_METRICS_SETTING,
254+
StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING,
253255

254256
IndexSettings.INDEX_CONTEXT_CREATED_VERSION,
255257
IndexSettings.INDEX_CONTEXT_CURRENT_VERSION,

‎server/src/main/java/org/opensearch/index/IndexSettings.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.opensearch.core.common.unit.ByteSizeUnit;
5050
import org.opensearch.core.common.unit.ByteSizeValue;
5151
import org.opensearch.core.index.Index;
52+
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
5253
import org.opensearch.index.remote.RemoteStorePathStrategy;
5354
import org.opensearch.index.remote.RemoteStoreUtils;
5455
import org.opensearch.index.translog.Translog;
@@ -902,6 +903,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
902903
*/
903904
private volatile double docIdFuzzySetFalsePositiveProbability;
904905

906+
private final boolean isCompositeIndex;
907+
905908
/**
906909
* Returns the default search fields for this index.
907910
*/
@@ -1064,7 +1067,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
10641067

10651068
setEnableFuzzySetForDocId(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_ENABLED_SETTING));
10661069
setDocIdFuzzySetFalsePositiveProbability(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING));
1067-
1070+
isCompositeIndex = scopedSettings.get(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING);
10681071
scopedSettings.addSettingsUpdateConsumer(
10691072
TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING,
10701073
tieredMergePolicyProvider::setNoCFSRatio
@@ -1309,6 +1312,10 @@ public int getNumberOfReplicas() {
13091312
return settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, null);
13101313
}
13111314

1315+
public boolean isCompositeIndex() {
1316+
return isCompositeIndex;
1317+
}
1318+
13121319
/**
13131320
* Returns true if segment replication is enabled on the index.
13141321
*

‎server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java

+14-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
import org.opensearch.common.settings.Setting;
1414
import org.opensearch.common.settings.Settings;
1515
import org.opensearch.common.util.FeatureFlags;
16+
import org.opensearch.core.common.unit.ByteSizeUnit;
17+
import org.opensearch.core.common.unit.ByteSizeValue;
1618

1719
/**
1820
* Cluster level settings for composite indices
@@ -37,12 +39,23 @@ public class CompositeIndexSettings {
3739
Setting.Property.Dynamic
3840
);
3941

42+
/**
43+
* This sets the max flush threshold size for composite index
44+
*/
45+
public static final Setting<ByteSizeValue> COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING = Setting.byteSizeSetting(
46+
"indices.composite_index.translog.max_flush_threshold_size",
47+
new ByteSizeValue(512, ByteSizeUnit.MB),
48+
new ByteSizeValue(128, ByteSizeUnit.MB),
49+
new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES),
50+
Setting.Property.NodeScope,
51+
Setting.Property.Dynamic
52+
);
53+
4054
private volatile boolean starTreeIndexCreationEnabled;
4155

4256
public CompositeIndexSettings(Settings settings, ClusterSettings clusterSettings) {
4357
this.starTreeIndexCreationEnabled = STAR_TREE_INDEX_ENABLED_SETTING.get(settings);
4458
clusterSettings.addSettingsUpdateConsumer(STAR_TREE_INDEX_ENABLED_SETTING, this::starTreeIndexCreationEnabled);
45-
4659
}
4760

4861
private void starTreeIndexCreationEnabled(boolean value) {

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java

+21
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
public class StarTreeIndexSettings {
2727

2828
public static int STAR_TREE_MAX_DIMENSIONS_DEFAULT = 10;
29+
public static int STAR_TREE_MAX_BASE_METRICS_DEFAULT = 100;
2930
/**
3031
* This setting determines the max number of star tree fields that can be part of composite index mapping. For each
3132
* star tree field, we will generate associated star tree index.
@@ -52,6 +53,19 @@ public class StarTreeIndexSettings {
5253
Setting.Property.Final
5354
);
5455

56+
/**
57+
* This setting determines the max number of dimensions that can be part of star tree index field. Number of
58+
* dimensions and associated cardinality has direct effect of star tree index size and query performance.
59+
*/
60+
public static final Setting<Integer> STAR_TREE_MAX_BASE_METRICS_SETTING = Setting.intSetting(
61+
"index.composite_index.star_tree.field.max_base_metrics",
62+
STAR_TREE_MAX_BASE_METRICS_DEFAULT,
63+
4,
64+
100,
65+
Setting.Property.IndexScope,
66+
Setting.Property.Final
67+
);
68+
5569
/**
5670
* This setting determines the max number of date intervals that can be part of star tree date field.
5771
*/
@@ -108,4 +122,11 @@ public static Rounding.DateTimeUnit getTimeUnit(String expression) {
108122
}
109123
return DateHistogramAggregationBuilder.DATE_FIELD_UNITS.get(expression);
110124
}
125+
126+
public static final Setting<Boolean> IS_COMPOSITE_INDEX_SETTING = Setting.boolSetting(
127+
"index.composite_index",
128+
false,
129+
Setting.Property.IndexScope,
130+
Setting.Property.Final
131+
);
111132
}

‎server/src/main/java/org/opensearch/index/mapper/DocumentParser.java

+11
Original file line numberDiff line numberDiff line change
@@ -661,6 +661,17 @@ private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapp
661661
throws IOException {
662662
XContentParser parser = context.parser();
663663
XContentParser.Token token;
664+
// block array values for composite index fields
665+
if (context.indexSettings().isCompositeIndex() && context.mapperService().isFieldPartOfCompositeIndex(arrayFieldName)) {
666+
throw new MapperParsingException(
667+
String.format(
668+
Locale.ROOT,
669+
"object mapping for [%s] with array for [%s] cannot be accepted as field is also part of composite index mapping which does not accept arrays",
670+
mapper.name(),
671+
arrayFieldName
672+
)
673+
);
674+
}
664675
final String[] paths = splitAndValidatePath(lastFieldName);
665676
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
666677
if (token == XContentParser.Token.START_OBJECT) {

‎server/src/main/java/org/opensearch/index/mapper/MapperService.java

+14
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,7 @@ public enum MergeReason {
228228
private final BooleanSupplier idFieldDataEnabled;
229229

230230
private volatile Set<CompositeMappedFieldType> compositeMappedFieldTypes;
231+
private volatile Set<String> fieldsPartOfCompositeMappings;
231232

232233
public MapperService(
233234
IndexSettings indexSettings,
@@ -543,9 +544,18 @@ private synchronized Map<String, DocumentMapper> internalMerge(DocumentMapper ma
543544

544545
// initialize composite fields post merge
545546
this.compositeMappedFieldTypes = getCompositeFieldTypesFromMapper();
547+
buildCompositeFieldLookup();
546548
return results;
547549
}
548550

551+
private void buildCompositeFieldLookup() {
552+
Set<String> fieldsPartOfCompositeMappings = new HashSet<>();
553+
for (CompositeMappedFieldType fieldType : compositeMappedFieldTypes) {
554+
fieldsPartOfCompositeMappings.addAll(fieldType.fields());
555+
}
556+
this.fieldsPartOfCompositeMappings = fieldsPartOfCompositeMappings;
557+
}
558+
549559
private boolean assertSerialization(DocumentMapper mapper) {
550560
// capture the source now, it may change due to concurrent parsing
551561
final CompressedXContent mappingSource = mapper.mappingSource();
@@ -672,6 +682,10 @@ private Set<CompositeMappedFieldType> getCompositeFieldTypesFromMapper() {
672682
return compositeMappedFieldTypes;
673683
}
674684

685+
public boolean isFieldPartOfCompositeIndex(String field) {
686+
return fieldsPartOfCompositeMappings.contains(field);
687+
}
688+
675689
public ObjectMapper getObjectMapper(String name) {
676690
return this.mapper == null ? null : this.mapper.objectMappers().get(name);
677691
}

‎server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java

+9
Original file line numberDiff line numberDiff line change
@@ -440,6 +440,15 @@ protected static void parseCompositeField(
440440
+ " feature flag in the JVM options"
441441
);
442442
}
443+
if (StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.get(parserContext.getSettings()) == false) {
444+
throw new IllegalArgumentException(
445+
String.format(
446+
Locale.ROOT,
447+
"Set '%s' as true as part of index settings to use star tree index",
448+
StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey()
449+
)
450+
);
451+
}
443452
Iterator<Map.Entry<String, Object>> iterator = compositeNode.entrySet().iterator();
444453
if (compositeNode.size() > StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(parserContext.getSettings())) {
445454
throw new IllegalArgumentException(

0 commit comments

Comments
 (0)