Skip to content

Commit 5504f32

Browse files
sumitasrsumitasr
authored
Merge branch '2.x' into backport/backport-15508-to-2.x
Signed-off-by: Sumit Bansal <sumitsb@amazon.com>
2 parents a71278e + bb0a497 commit 5504f32

File tree

12 files changed

+153
-29
lines changed

12 files changed

+153
-29
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
5151
- [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218))
5252
- Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471))
5353
- Optimize NodeIndicesStats output behind flag ([#14454](https://github.com/opensearch-project/OpenSearch/pull/14454))
54+
- MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637))
5455
- ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508))
5556

5657
### Dependencies

‎server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

+1
Original file line numberDiff line numberDiff line change
@@ -551,6 +551,7 @@ public void apply(Settings value, Settings current, Settings previous) {
551551
SearchService.MAX_AGGREGATION_REWRITE_FILTERS,
552552
SearchService.INDICES_MAX_CLAUSE_COUNT_SETTING,
553553
SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD,
554+
SearchService.KEYWORD_INDEX_OR_DOC_VALUES_ENABLED,
554555
CreatePitController.PIT_INIT_KEEP_ALIVE,
555556
Node.WRITE_PORTS_FILE_SETTING,
556557
Node.NODE_NAME_SETTING,

‎server/src/main/java/org/opensearch/index/IndexService.java

+20-2
Original file line numberDiff line numberDiff line change
@@ -959,7 +959,8 @@ public QueryShardContext newQueryShardContext(
959959
IndexSearcher searcher,
960960
LongSupplier nowInMillis,
961961
String clusterAlias,
962-
boolean validate
962+
boolean validate,
963+
boolean keywordIndexOrDocValuesEnabled
963964
) {
964965
final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(
965966
index().getName(),
@@ -985,10 +986,27 @@ public QueryShardContext newQueryShardContext(
985986
indexNameMatcher,
986987
allowExpensiveQueries,
987988
valuesSourceRegistry,
988-
validate
989+
validate,
990+
keywordIndexOrDocValuesEnabled
989991
);
990992
}
991993

994+
/**
995+
* Creates a new QueryShardContext.
996+
* <p>
997+
* Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make
998+
* {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
999+
*/
1000+
public QueryShardContext newQueryShardContext(
1001+
int shardId,
1002+
IndexSearcher searcher,
1003+
LongSupplier nowInMillis,
1004+
String clusterAlias,
1005+
boolean validate
1006+
) {
1007+
return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, validate, false);
1008+
}
1009+
9921010
/**
9931011
* The {@link ThreadPool} to use for this index.
9941012
*/

‎server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java

+15
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,9 @@ public Query termsQuery(List<?> values, QueryShardContext context) {
392392
failIfNotIndexedAndNoDocValues();
393393
// has index and doc_values enabled
394394
if (isSearchable() && hasDocValues()) {
395+
if (!context.keywordFieldIndexOrDocValuesEnabled()) {
396+
return super.termsQuery(values, context);
397+
}
395398
BytesRef[] bytesRefs = new BytesRef[values.size()];
396399
for (int i = 0; i < bytesRefs.length; i++) {
397400
bytesRefs[i] = indexedValueForSearch(values.get(i));
@@ -429,6 +432,9 @@ public Query prefixQuery(
429432
}
430433
failIfNotIndexedAndNoDocValues();
431434
if (isSearchable() && hasDocValues()) {
435+
if (!context.keywordFieldIndexOrDocValuesEnabled()) {
436+
return super.prefixQuery(value, method, caseInsensitive, context);
437+
}
432438
Query indexQuery = super.prefixQuery(value, method, caseInsensitive, context);
433439
Query dvQuery = super.prefixQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, context);
434440
return new IndexOrDocValuesQuery(indexQuery, dvQuery);
@@ -461,6 +467,9 @@ public Query regexpQuery(
461467
}
462468
failIfNotIndexedAndNoDocValues();
463469
if (isSearchable() && hasDocValues()) {
470+
if (!context.keywordFieldIndexOrDocValuesEnabled()) {
471+
return super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context);
472+
}
464473
Query indexQuery = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context);
465474
Query dvQuery = super.regexpQuery(
466475
value,
@@ -549,6 +558,9 @@ public Query fuzzyQuery(
549558
);
550559
}
551560
if (isSearchable() && hasDocValues()) {
561+
if (!context.keywordFieldIndexOrDocValuesEnabled()) {
562+
return super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context);
563+
}
552564
Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context);
553565
Query dvQuery = super.fuzzyQuery(
554566
value,
@@ -591,6 +603,9 @@ public Query wildcardQuery(
591603
// wildcard
592604
// query text
593605
if (isSearchable() && hasDocValues()) {
606+
if (!context.keywordFieldIndexOrDocValuesEnabled()) {
607+
return super.wildcardQuery(value, method, caseInsensitive, true, context);
608+
}
594609
Query indexQuery = super.wildcardQuery(value, method, caseInsensitive, true, context);
595610
Query dvQuery = super.wildcardQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, true, context);
596611
return new IndexOrDocValuesQuery(indexQuery, dvQuery);

‎server/src/main/java/org/opensearch/index/query/QueryShardContext.java

+63-3
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ public class QueryShardContext extends QueryRewriteContext {
126126
private final ValuesSourceRegistry valuesSourceRegistry;
127127
private BitSetProducer parentFilter;
128128
private DerivedFieldResolver derivedFieldResolver;
129+
private boolean keywordIndexOrDocValuesEnabled;
129130

130131
public QueryShardContext(
131132
int shardId,
@@ -209,7 +210,55 @@ public QueryShardContext(
209210
),
210211
allowExpensiveQueries,
211212
valuesSourceRegistry,
212-
validate
213+
validate,
214+
false
215+
);
216+
}
217+
218+
public QueryShardContext(
219+
int shardId,
220+
IndexSettings indexSettings,
221+
BigArrays bigArrays,
222+
BitsetFilterCache bitsetFilterCache,
223+
TriFunction<MappedFieldType, String, Supplier<SearchLookup>, IndexFieldData<?>> indexFieldDataLookup,
224+
MapperService mapperService,
225+
SimilarityService similarityService,
226+
ScriptService scriptService,
227+
NamedXContentRegistry xContentRegistry,
228+
NamedWriteableRegistry namedWriteableRegistry,
229+
Client client,
230+
IndexSearcher searcher,
231+
LongSupplier nowInMillis,
232+
String clusterAlias,
233+
Predicate<String> indexNameMatcher,
234+
BooleanSupplier allowExpensiveQueries,
235+
ValuesSourceRegistry valuesSourceRegistry,
236+
boolean validate,
237+
boolean keywordIndexOrDocValuesEnabled
238+
) {
239+
this(
240+
shardId,
241+
indexSettings,
242+
bigArrays,
243+
bitsetFilterCache,
244+
indexFieldDataLookup,
245+
mapperService,
246+
similarityService,
247+
scriptService,
248+
xContentRegistry,
249+
namedWriteableRegistry,
250+
client,
251+
searcher,
252+
nowInMillis,
253+
indexNameMatcher,
254+
new Index(
255+
RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()),
256+
indexSettings.getIndex().getUUID()
257+
),
258+
allowExpensiveQueries,
259+
valuesSourceRegistry,
260+
validate,
261+
keywordIndexOrDocValuesEnabled
213262
);
214263
}
215264

@@ -232,7 +281,8 @@ public QueryShardContext(QueryShardContext source) {
232281
source.fullyQualifiedIndex,
233282
source.allowExpensiveQueries,
234283
source.valuesSourceRegistry,
235-
source.validate()
284+
source.validate(),
285+
source.keywordIndexOrDocValuesEnabled
236286
);
237287
}
238288

@@ -254,7 +304,8 @@ private QueryShardContext(
254304
Index fullyQualifiedIndex,
255305
BooleanSupplier allowExpensiveQueries,
256306
ValuesSourceRegistry valuesSourceRegistry,
257-
boolean validate
307+
boolean validate,
308+
boolean keywordIndexOrDocValuesEnabled
258309
) {
259310
super(xContentRegistry, namedWriteableRegistry, client, nowInMillis, validate);
260311
this.shardId = shardId;
@@ -278,6 +329,7 @@ private QueryShardContext(
278329
emptyList(),
279330
indexSettings.isDerivedFieldAllowed()
280331
);
332+
this.keywordIndexOrDocValuesEnabled = keywordIndexOrDocValuesEnabled;
281333
}
282334

283335
private void reset() {
@@ -425,6 +477,14 @@ public MappedFieldType getDerivedFieldType(String fieldName) {
425477
throw new UnsupportedOperationException("Use resolveDerivedFieldType() instead.");
426478
}
427479

480+
public boolean keywordFieldIndexOrDocValuesEnabled() {
481+
return keywordIndexOrDocValuesEnabled;
482+
}
483+
484+
public void setKeywordFieldIndexOrDocValuesEnabled(boolean keywordIndexOrDocValuesEnabled) {
485+
this.keywordIndexOrDocValuesEnabled = keywordIndexOrDocValuesEnabled;
486+
}
487+
428488
public void setAllowUnmappedFields(boolean allowUnmappedFields) {
429489
this.allowUnmappedFields = allowUnmappedFields;
430490
}

‎server/src/main/java/org/opensearch/search/DefaultSearchContext.java

+17-1
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@
121121
import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL;
122122
import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO;
123123
import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE;
124+
import static org.opensearch.search.SearchService.KEYWORD_INDEX_OR_DOC_VALUES_ENABLED;
124125
import static org.opensearch.search.SearchService.MAX_AGGREGATION_REWRITE_FILTERS;
125126

126127
/**
@@ -207,6 +208,7 @@ final class DefaultSearchContext extends SearchContext {
207208
private final SetOnce<Boolean> requestShouldUseConcurrentSearch = new SetOnce<>();
208209
private final int maxAggRewriteFilters;
209210
private final int cardinalityAggregationPruningThreshold;
211+
private final boolean keywordIndexOrDocValuesEnabled;
210212

211213
DefaultSearchContext(
212214
ReaderContext readerContext,
@@ -257,14 +259,16 @@ final class DefaultSearchContext extends SearchContext {
257259
this.searcher,
258260
request::nowInMillis,
259261
shardTarget.getClusterAlias(),
260-
validate
262+
validate,
263+
evaluateKeywordIndexOrDocValuesEnabled()
261264
);
262265
queryBoost = request.indexBoost();
263266
this.lowLevelCancellation = lowLevelCancellation;
264267
this.requestToAggReduceContextBuilder = requestToAggReduceContextBuilder;
265268

266269
this.maxAggRewriteFilters = evaluateFilterRewriteSetting();
267270
this.cardinalityAggregationPruningThreshold = evaluateCardinalityAggregationPruningThreshold();
271+
this.keywordIndexOrDocValuesEnabled = evaluateKeywordIndexOrDocValuesEnabled();
268272
this.concurrentSearchDeciderFactories = concurrentSearchDeciderFactories;
269273
}
270274

@@ -1125,10 +1129,22 @@ public int cardinalityAggregationPruningThreshold() {
11251129
return cardinalityAggregationPruningThreshold;
11261130
}
11271131

1132+
@Override
1133+
public boolean keywordIndexOrDocValuesEnabled() {
1134+
return keywordIndexOrDocValuesEnabled;
1135+
}
1136+
11281137
private int evaluateCardinalityAggregationPruningThreshold() {
11291138
if (clusterService != null) {
11301139
return clusterService.getClusterSettings().get(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD);
11311140
}
11321141
return 0;
11331142
}
1143+
1144+
public boolean evaluateKeywordIndexOrDocValuesEnabled() {
1145+
if (clusterService != null) {
1146+
return clusterService.getClusterSettings().get(KEYWORD_INDEX_OR_DOC_VALUES_ENABLED);
1147+
}
1148+
return false;
1149+
}
11341150
}

‎server/src/main/java/org/opensearch/search/SearchService.java

+8
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,13 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
338338
Property.NodeScope
339339
);
340340

341+
public static final Setting<Boolean> KEYWORD_INDEX_OR_DOC_VALUES_ENABLED = Setting.boolSetting(
342+
"search.keyword_index_or_doc_values_enabled",
343+
false,
344+
Property.Dynamic,
345+
Property.NodeScope
346+
);
347+
341348
public static final int DEFAULT_SIZE = 10;
342349
public static final int DEFAULT_FROM = 0;
343350

@@ -1174,6 +1181,7 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
11741181
context.getIndexSettings().isDerivedFieldAllowed() && allowDerivedField
11751182
);
11761183
context.setDerivedFieldResolver(derivedFieldResolver);
1184+
context.setKeywordFieldIndexOrDocValuesEnabled(searchContext.keywordIndexOrDocValuesEnabled());
11771185
searchContext.getQueryShardContext().setDerivedFieldResolver(derivedFieldResolver);
11781186
Rewriteable.rewrite(request.getRewriteable(), context, true);
11791187
assert searchContext.getQueryShardContext().isCacheable();

‎server/src/main/java/org/opensearch/search/internal/SearchContext.java

+5
Original file line numberDiff line numberDiff line change
@@ -530,4 +530,9 @@ public int maxAggRewriteFilters() {
530530
public int cardinalityAggregationPruningThreshold() {
531531
return 0;
532532
}
533+
534+
public boolean keywordIndexOrDocValuesEnabled() {
535+
return false;
536+
}
537+
533538
}

‎server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ public void testTermsQuery() {
136136
new TermInSetQuery("field", terms),
137137
new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, "field", terms)
138138
);
139-
assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), null));
139+
assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES));
140140

141141
MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap());
142142
Query expectedIndex = new TermInSetQuery("field", terms);
@@ -225,7 +225,7 @@ public void testRegexpQuery() {
225225
new RegexpQuery(new Term("field", "foo.*")),
226226
new RegexpQuery(new Term("field", "foo.*"), 0, 0, RegexpQuery.DEFAULT_PROVIDER, 10, MultiTermQuery.DOC_VALUES_REWRITE)
227227
),
228-
ft.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)
228+
ft.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)
229229
);
230230

231231
Query indexExpected = new RegexpQuery(new Term("field", "foo.*"));
@@ -267,7 +267,7 @@ public void testFuzzyQuery() {
267267
new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true),
268268
new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE)
269269
),
270-
ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC)
270+
ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)
271271
);
272272

273273
Query indexExpected = new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true);
@@ -308,7 +308,7 @@ public void testWildCardQuery() {
308308
MultiTermQuery.DOC_VALUES_REWRITE
309309
)
310310
);
311-
assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC));
311+
assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC_ENABLE_INDEX_DOC_VALUES));
312312

313313
Query indexExpected = new WildcardQuery(new Term("field", new BytesRef("foo*")));
314314
MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap());

‎server/src/test/java/org/opensearch/index/search/NestedHelperTests.java

+6-4
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@
5757
import java.io.IOException;
5858
import java.util.Collections;
5959

60+
import static org.opensearch.index.mapper.FieldTypeTestCase.MOCK_QSC_ENABLE_INDEX_DOC_VALUES;
61+
6062
public class NestedHelperTests extends OpenSearchSingleNodeTestCase {
6163

6264
IndexService indexService;
@@ -132,28 +134,28 @@ public void testMatchNo() {
132134
}
133135

134136
public void testTermsQuery() {
135-
Query termsQuery = mapperService.fieldType("foo").termsQuery(Collections.singletonList("bar"), null);
137+
Query termsQuery = mapperService.fieldType("foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES);
136138
assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery));
137139
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1"));
138140
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2"));
139141
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3"));
140142
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing"));
141143

142-
termsQuery = mapperService.fieldType("nested1.foo").termsQuery(Collections.singletonList("bar"), null);
144+
termsQuery = mapperService.fieldType("nested1.foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES);
143145
assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery));
144146
assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1"));
145147
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2"));
146148
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3"));
147149
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing"));
148150

149-
termsQuery = mapperService.fieldType("nested2.foo").termsQuery(Collections.singletonList("bar"), null);
151+
termsQuery = mapperService.fieldType("nested2.foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES);
150152
assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery));
151153
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1"));
152154
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2"));
153155
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3"));
154156
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing"));
155157

156-
termsQuery = mapperService.fieldType("nested3.foo").termsQuery(Collections.singletonList("bar"), null);
158+
termsQuery = mapperService.fieldType("nested3.foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES);
157159
assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery));
158160
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1"));
159161
assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2"));

0 commit comments

Comments
 (0)