Skip to content

Commit 2f8cb07

Browse files
bowenlan-amznrishabhmaurya
bowenlan-amzn
and
rishabhmaurya
authored
Support Dynamic Pruning in Cardinality Aggregation (opensearch-project#13821)
* Cardinality aggregation dynamic pruning changes Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * Reading Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * remaining disjunction scorer full understand Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * utilize competitive iterator api to perform pruning Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * handle missing input Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * add change log Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * clean up Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * Clean up Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * Test fix Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * Do all the scoring within Cardinality Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * clean unnecessary Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * fix Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * Add dynamic flag for this feature Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * Add random test, small bug fix Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * address comment Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * Address comments Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> * address comments Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> --------- Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> Co-authored-by: Rishabh Maurya <rishabhmaurya05@gmail.com>
1 parent 990ddc3 commit 2f8cb07

File tree

10 files changed

+624
-11
lines changed

10 files changed

+624
-11
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2929
- Move Remote Store Migration from DocRep to GA and modify remote migration settings name ([#14100](https://github.com/opensearch-project/OpenSearch/pull/14100))
3030
- Derived field object type support ([#13720](https://github.com/opensearch-project/OpenSearch/pull/13720))
3131
- [Query Insights] Add cpu and memory metrics to top n queries ([#13739](https://github.com/opensearch-project/OpenSearch/pull/13739))
32+
- Support Dynamic Pruning in Cardinality Aggregation ([#13821](https://github.com/opensearch-project/OpenSearch/pull/13821))
3233

3334
### Dependencies
3435
- Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))

‎server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java

+32
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
3636

37+
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
3738
import org.opensearch.action.index.IndexRequestBuilder;
3839
import org.opensearch.action.search.SearchResponse;
3940
import org.opensearch.common.settings.Settings;
@@ -59,6 +60,7 @@
5960
import static java.util.Collections.emptyMap;
6061
import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
6162
import static org.opensearch.index.query.QueryBuilders.matchAllQuery;
63+
import static org.opensearch.search.SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD;
6264
import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
6365
import static org.opensearch.search.aggregations.AggregationBuilders.cardinality;
6466
import static org.opensearch.search.aggregations.AggregationBuilders.global;
@@ -255,6 +257,36 @@ public void testSingleValuedString() throws Exception {
255257
assertCount(count, numDocs);
256258
}
257259

260+
public void testDisableDynamicPruning() throws Exception {
261+
SearchResponse response = client().prepareSearch("idx")
262+
.addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value"))
263+
.get();
264+
assertSearchResponse(response);
265+
266+
Cardinality count1 = response.getAggregations().get("cardinality");
267+
268+
final ClusterUpdateSettingsResponse updateSettingResponse = client().admin()
269+
.cluster()
270+
.prepareUpdateSettings()
271+
.setTransientSettings(Settings.builder().put(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD.getKey(), 0))
272+
.get();
273+
assertEquals(updateSettingResponse.getTransientSettings().get(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD.getKey()), "0");
274+
275+
response = client().prepareSearch("idx")
276+
.addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value"))
277+
.get();
278+
assertSearchResponse(response);
279+
Cardinality count2 = response.getAggregations().get("cardinality");
280+
281+
assertEquals(count1, count2);
282+
283+
client().admin()
284+
.cluster()
285+
.prepareUpdateSettings()
286+
.setTransientSettings(Settings.builder().putNull(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD.getKey()))
287+
.get();
288+
}
289+
258290
public void testSingleValuedNumeric() throws Exception {
259291
SearchResponse response = client().prepareSearch("idx")
260292
.addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField()))

‎server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

+1
Original file line numberDiff line numberDiff line change
@@ -540,6 +540,7 @@ public void apply(Settings value, Settings current, Settings previous) {
540540
SearchService.MAX_OPEN_PIT_CONTEXT,
541541
SearchService.MAX_PIT_KEEPALIVE_SETTING,
542542
SearchService.MAX_AGGREGATION_REWRITE_FILTERS,
543+
SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD,
543544
CreatePitController.PIT_INIT_KEEP_ALIVE,
544545
Node.WRITE_PORTS_FILE_SETTING,
545546
Node.NODE_NAME_SETTING,

‎server/src/main/java/org/opensearch/search/DefaultSearchContext.java

+15
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@
106106
import java.util.function.Function;
107107
import java.util.function.LongSupplier;
108108

109+
import static org.opensearch.search.SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD;
109110
import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
110111
import static org.opensearch.search.SearchService.MAX_AGGREGATION_REWRITE_FILTERS;
111112

@@ -189,6 +190,7 @@ final class DefaultSearchContext extends SearchContext {
189190
private final boolean concurrentSearchSettingsEnabled;
190191
private final SetOnce<Boolean> requestShouldUseConcurrentSearch = new SetOnce<>();
191192
private final int maxAggRewriteFilters;
193+
private final int cardinalityAggregationPruningThreshold;
192194

193195
DefaultSearchContext(
194196
ReaderContext readerContext,
@@ -244,6 +246,7 @@ final class DefaultSearchContext extends SearchContext {
244246
this.requestToAggReduceContextBuilder = requestToAggReduceContextBuilder;
245247

246248
this.maxAggRewriteFilters = evaluateFilterRewriteSetting();
249+
this.cardinalityAggregationPruningThreshold = evaluateCardinalityAggregationPruningThreshold();
247250
}
248251

249252
@Override
@@ -1010,4 +1013,16 @@ private int evaluateFilterRewriteSetting() {
10101013
}
10111014
return 0;
10121015
}
1016+
1017+
@Override
1018+
public int cardinalityAggregationPruningThreshold() {
1019+
return cardinalityAggregationPruningThreshold;
1020+
}
1021+
1022+
private int evaluateCardinalityAggregationPruningThreshold() {
1023+
if (clusterService != null) {
1024+
return clusterService.getClusterSettings().get(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD);
1025+
}
1026+
return 0;
1027+
}
10131028
}

‎server/src/main/java/org/opensearch/search/SearchService.java

+9
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,15 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
288288
Property.NodeScope
289289
);
290290

291+
// value 0 can disable dynamic pruning optimization in cardinality aggregation
292+
public static final Setting<Integer> CARDINALITY_AGGREGATION_PRUNING_THRESHOLD = Setting.intSetting(
293+
"search.dynamic_pruning.cardinality_aggregation.max_allowed_cardinality",
294+
100,
295+
0,
296+
Property.Dynamic,
297+
Property.NodeScope
298+
);
299+
291300
public static final int DEFAULT_SIZE = 10;
292301
public static final int DEFAULT_FROM = 0;
293302

0 commit comments

Comments
 (0)