Skip to content

Commit b2ee0e3

Browse files
bharath-techiebharath-techie
authored
Enforcing append only setting for star-tree index (#17180)
Signed-off-by: bharath-techie <bharath78910@gmail.com>
1 parent 7306905 commit b2ee0e3

File tree

8 files changed

+58
-3
lines changed

8 files changed

+58
-3
lines changed

‎modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
import org.apache.lucene.index.DocValuesType;
3636
import org.apache.lucene.index.IndexableField;
37+
import org.opensearch.cluster.metadata.IndexMetadata;
3738
import org.opensearch.common.settings.Settings;
3839
import org.opensearch.common.util.FeatureFlags;
3940
import org.opensearch.common.xcontent.XContentFactory;
@@ -132,6 +133,7 @@ public void testScaledFloatWithStarTree() throws Exception {
132133
protected Settings getIndexSettings() {
133134
return Settings.builder()
134135
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
136+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
135137
.put(super.getIndexSettings())
136138
.build();
137139
}

‎server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

+22
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.opensearch.action.index.IndexResponse;
1313
import org.opensearch.action.search.SearchResponse;
1414
import org.opensearch.action.support.clustermanager.AcknowledgedResponse;
15+
import org.opensearch.cluster.metadata.IndexMetadata;
1516
import org.opensearch.common.Rounding;
1617
import org.opensearch.common.settings.Settings;
1718
import org.opensearch.common.util.FeatureFlags;
@@ -56,6 +57,7 @@ public class StarTreeMapperIT extends OpenSearchIntegTestCase {
5657
private static final String TEST_INDEX = "test";
5758
Settings settings = Settings.builder()
5859
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
60+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
5961
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
6062
.build();
6163

@@ -756,6 +758,7 @@ public void testValidCompositeIndexWithDuplicateDates() {
756758
public void testCompositeIndexWithIndexNotSpecified() {
757759
Settings settings = Settings.builder()
758760
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
761+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
759762
.build();
760763
MapperParsingException ex = expectThrows(
761764
MapperParsingException.class,
@@ -767,9 +770,25 @@ public void testCompositeIndexWithIndexNotSpecified() {
767770
);
768771
}
769772

773+
public void testAppendOnlyInCompositeIndexNotSpecified() {
774+
Settings settings = Settings.builder()
775+
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
776+
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
777+
.build();
778+
MapperParsingException ex = expectThrows(
779+
MapperParsingException.class,
780+
() -> prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get()
781+
);
782+
assertEquals(
783+
"Failed to parse mapping [_doc]: Set 'index.append_only.enabled' as true as part of index settings to use star tree index",
784+
ex.getMessage()
785+
);
786+
}
787+
770788
public void testCompositeIndexWithHigherTranslogFlushSize() {
771789
Settings settings = Settings.builder()
772790
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
791+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
773792
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(513, ByteSizeUnit.MB))
774793
.build();
775794
IllegalArgumentException ex = expectThrows(
@@ -1082,6 +1101,7 @@ public void testMaxDimsCompositeIndex() {
10821101
Settings.builder()
10831102
.put(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 2)
10841103
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
1104+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
10851105
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
10861106
)
10871107
.get()
@@ -1101,6 +1121,7 @@ public void testMaxMetricsCompositeIndex() {
11011121
Settings.builder()
11021122
.put(StarTreeIndexSettings.STAR_TREE_MAX_BASE_METRICS_SETTING.getKey(), 4)
11031123
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
1124+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
11041125
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
11051126
)
11061127
.get()
@@ -1119,6 +1140,7 @@ public void testMaxCalendarIntervalsCompositeIndex() {
11191140
Settings.builder()
11201141
.put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1)
11211142
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
1143+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
11221144
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
11231145
)
11241146
.get()

‎server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java

+9
Original file line numberDiff line numberDiff line change
@@ -462,6 +462,15 @@ protected static void parseCompositeField(
462462
)
463463
);
464464
}
465+
if (IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.get(parserContext.getSettings()) == false) {
466+
throw new IllegalArgumentException(
467+
String.format(
468+
Locale.ROOT,
469+
"Set '%s' as true as part of index settings to use star tree index",
470+
IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey()
471+
)
472+
);
473+
}
465474
Iterator<Map.Entry<String, Object>> iterator = compositeNode.entrySet().iterator();
466475
if (compositeNode.size() > StarTreeIndexSettings.STAR_TREE_MAX_FIELDS_SETTING.get(parserContext.getSettings())) {
467476
throw new IllegalArgumentException(

‎server/src/test/java/org/opensearch/cluster/metadata/TranslogFlushIntervalSettingsTests.java

+16-3
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,18 @@ public void testValidSettings() {
3131
Settings requestSettings = Settings.builder()
3232
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "50mb")
3333
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
34+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
3435
.build();
3536

3637
// This should not throw an exception
3738
MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex(requestSettings, clusterSettings);
3839
}
3940

4041
public void testDefaultTranslogFlushSetting() {
41-
Settings requestSettings = Settings.builder().put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true).build();
42+
Settings requestSettings = Settings.builder()
43+
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
44+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
45+
.build();
4246

4347
// This should not throw an exception
4448
IllegalArgumentException ex = expectThrows(
@@ -61,6 +65,7 @@ public void testNullTranslogFlushSetting() {
6165
Settings requestSettings = Settings.builder()
6266
.putNull(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey())
6367
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
68+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
6469
.build();
6570

6671
// This should not throw an exception
@@ -75,6 +80,7 @@ public void testExceedingMaxFlushSize() {
7580
Settings requestSettings = Settings.builder()
7681
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "150mb")
7782
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
83+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
7884
.build();
7985

8086
IllegalArgumentException ex = expectThrows(
@@ -88,6 +94,7 @@ public void testEqualToMaxFlushSize() {
8894
Settings requestSettings = Settings.builder()
8995
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "100mb")
9096
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
97+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
9198
.build();
9299

93100
// This should not throw an exception
@@ -99,7 +106,10 @@ public void testUpdateIndexThresholdFlushSize() {
99106
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "100mb")
100107
.build();
101108

102-
Settings indexSettings = Settings.builder().put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true).build();
109+
Settings indexSettings = Settings.builder()
110+
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
111+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
112+
.build();
103113

104114
// This should not throw an exception
105115
assertTrue(
@@ -116,7 +126,10 @@ public void testUpdateFlushSizeAboveThresholdWithCompositeIndex() {
116126
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "131mb")
117127
.build();
118128

119-
Settings indexSettings = Settings.builder().put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true).build();
129+
Settings indexSettings = Settings.builder()
130+
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
131+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
132+
.build();
120133

121134
Optional<String> err = MetadataCreateIndexService.validateTranslogFlushIntervalSettingsForCompositeIndex(
122135
requestSettings,

‎server/src/test/java/org/opensearch/index/codec/composite912/datacube/startree/AbstractStarTreeDVFormatTests.java

+1
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ public static MapperService createMapperService(XContentBuilder builder) throws
100100
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
101101
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
102102
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
103+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
103104
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
104105
.build();
105106
IndexMetadata indexMetadata = IndexMetadata.builder("test").settings(settings).putMapping(builder.toString()).build();

‎server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java

+2
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
package org.opensearch.index.mapper;
3434

35+
import org.opensearch.cluster.metadata.IndexMetadata;
3536
import org.opensearch.common.compress.CompressedXContent;
3637
import org.opensearch.common.settings.Settings;
3738
import org.opensearch.common.util.FeatureFlags;
@@ -542,6 +543,7 @@ public void testCompositeFields() throws Exception {
542543

543544
Settings settings = Settings.builder()
544545
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
546+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
545547
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
546548
.build();
547549

‎server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java

+3
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
package org.opensearch.index.mapper;
1010

11+
import org.opensearch.cluster.metadata.IndexMetadata;
1112
import org.opensearch.common.CheckedConsumer;
1213
import org.opensearch.common.Rounding;
1314
import org.opensearch.common.settings.ClusterSettings;
@@ -67,6 +68,7 @@ public void teardown() {
6768
protected Settings getIndexSettings() {
6869
return Settings.builder()
6970
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
71+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
7072
.put(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
7173
.put(SETTINGS)
7274
.build();
@@ -138,6 +140,7 @@ public void testValidValueForCompositeIndex() {
138140
Settings settings = Settings.builder()
139141
.put(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "256mb")
140142
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
143+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
141144
.put(COMPOSITE_INDEX_MAX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), "512mb")
142145
.build();
143146

‎server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java

+3
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ public void testQueryParsingForMetricAggregations() throws IOException {
9696
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
9797
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
9898
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
99+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
99100
.build();
100101
CreateIndexRequestBuilder builder = client().admin()
101102
.indices()
@@ -246,6 +247,7 @@ public void testQueryParsingForDateHistogramAggregations() throws IOException {
246247
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
247248
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
248249
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
250+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
249251
.build();
250252
CreateIndexRequestBuilder builder = client().admin()
251253
.indices()
@@ -494,6 +496,7 @@ public void testInvalidQueryParsingForDateHistogramAggregations() throws IOExcep
494496
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
495497
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
496498
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
499+
.put(IndexMetadata.INDEX_APPEND_ONLY_ENABLED_SETTING.getKey(), true)
497500
.build();
498501
CreateIndexRequestBuilder builder = client().admin()
499502
.indices()

0 commit comments

Comments
 (0)