Skip to content

Commit 5ad0f5d

Browse files
Support Dynamic Pruning in Cardinality Aggregation (opensearch-project#13821) (opensearch-project#14203)
* Cardinality aggregation dynamic pruning changes * Reading * remaining disjunction scorer full understand * utilize competitive iterator api to perform pruning * handle missing input * add change log * clean up * Clean up * Test fix * Do all the scoring within Cardinality * clean unnecessary * fix * Add dynamic flag for this feature * Add random test, small bug fix * address comment * Address comments * address comments --------- Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> Co-authored-by: Rishabh Maurya <rishabhmaurya05@gmail.com>
1 parent c38dfef commit 5ad0f5d

File tree

10 files changed

+624
-11
lines changed

10 files changed

+624
-11
lines changed

CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
3131
- Add support for query level resource usage tracking ([#13172](https://github.com/opensearch-project/OpenSearch/pull/13172))
3232
- [Query Insights] Add cpu and memory metrics to top n queries ([#13739](https://github.com/opensearch-project/OpenSearch/pull/13739))
3333
- Derived field object type support ([#13720](https://github.com/opensearch-project/OpenSearch/pull/13720))
34+
- Support Dynamic Pruning in Cardinality Aggregation ([#13821](https://github.com/opensearch-project/OpenSearch/pull/13821))
3435

3536
### Dependencies
3637
- Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))

server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityIT.java

+32
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434

3535
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
3636

37+
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
3738
import org.opensearch.action.index.IndexRequestBuilder;
3839
import org.opensearch.action.search.SearchResponse;
3940
import org.opensearch.common.settings.Settings;
@@ -59,6 +60,7 @@
5960
import static java.util.Collections.emptyMap;
6061
import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
6162
import static org.opensearch.index.query.QueryBuilders.matchAllQuery;
63+
import static org.opensearch.search.SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD;
6264
import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
6365
import static org.opensearch.search.aggregations.AggregationBuilders.cardinality;
6466
import static org.opensearch.search.aggregations.AggregationBuilders.global;
@@ -255,6 +257,36 @@ public void testSingleValuedString() throws Exception {
255257
assertCount(count, numDocs);
256258
}
257259

260+
public void testDisableDynamicPruning() throws Exception {
261+
SearchResponse response = client().prepareSearch("idx")
262+
.addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value"))
263+
.get();
264+
assertSearchResponse(response);
265+
266+
Cardinality count1 = response.getAggregations().get("cardinality");
267+
268+
final ClusterUpdateSettingsResponse updateSettingResponse = client().admin()
269+
.cluster()
270+
.prepareUpdateSettings()
271+
.setTransientSettings(Settings.builder().put(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD.getKey(), 0))
272+
.get();
273+
assertEquals(updateSettingResponse.getTransientSettings().get(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD.getKey()), "0");
274+
275+
response = client().prepareSearch("idx")
276+
.addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value"))
277+
.get();
278+
assertSearchResponse(response);
279+
Cardinality count2 = response.getAggregations().get("cardinality");
280+
281+
assertEquals(count1, count2);
282+
283+
client().admin()
284+
.cluster()
285+
.prepareUpdateSettings()
286+
.setTransientSettings(Settings.builder().putNull(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD.getKey()))
287+
.get();
288+
}
289+
258290
public void testSingleValuedNumeric() throws Exception {
259291
SearchResponse response = client().prepareSearch("idx")
260292
.addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField()))

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

+1
Original file line numberDiff line numberDiff line change
@@ -541,6 +541,7 @@ public void apply(Settings value, Settings current, Settings previous) {
541541
SearchService.MAX_OPEN_PIT_CONTEXT,
542542
SearchService.MAX_PIT_KEEPALIVE_SETTING,
543543
SearchService.MAX_AGGREGATION_REWRITE_FILTERS,
544+
SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD,
544545
CreatePitController.PIT_INIT_KEEP_ALIVE,
545546
Node.WRITE_PORTS_FILE_SETTING,
546547
Node.NODE_NAME_SETTING,

server/src/main/java/org/opensearch/search/DefaultSearchContext.java

+15
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@
106106
import java.util.function.Function;
107107
import java.util.function.LongSupplier;
108108

109+
import static org.opensearch.search.SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD;
109110
import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
110111
import static org.opensearch.search.SearchService.MAX_AGGREGATION_REWRITE_FILTERS;
111112

@@ -189,6 +190,7 @@ final class DefaultSearchContext extends SearchContext {
189190
private final boolean concurrentSearchSettingsEnabled;
190191
private final SetOnce<Boolean> requestShouldUseConcurrentSearch = new SetOnce<>();
191192
private final int maxAggRewriteFilters;
193+
private final int cardinalityAggregationPruningThreshold;
192194

193195
DefaultSearchContext(
194196
ReaderContext readerContext,
@@ -244,6 +246,7 @@ final class DefaultSearchContext extends SearchContext {
244246
this.requestToAggReduceContextBuilder = requestToAggReduceContextBuilder;
245247

246248
this.maxAggRewriteFilters = evaluateFilterRewriteSetting();
249+
this.cardinalityAggregationPruningThreshold = evaluateCardinalityAggregationPruningThreshold();
247250
}
248251

249252
@Override
@@ -1010,4 +1013,16 @@ private int evaluateFilterRewriteSetting() {
10101013
}
10111014
return 0;
10121015
}
1016+
1017+
@Override
1018+
public int cardinalityAggregationPruningThreshold() {
1019+
return cardinalityAggregationPruningThreshold;
1020+
}
1021+
1022+
private int evaluateCardinalityAggregationPruningThreshold() {
1023+
if (clusterService != null) {
1024+
return clusterService.getClusterSettings().get(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD);
1025+
}
1026+
return 0;
1027+
}
10131028
}

server/src/main/java/org/opensearch/search/SearchService.java

+9
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,15 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
289289
Property.NodeScope
290290
);
291291

292+
// value 0 can disable dynamic pruning optimization in cardinality aggregation
293+
public static final Setting<Integer> CARDINALITY_AGGREGATION_PRUNING_THRESHOLD = Setting.intSetting(
294+
"search.dynamic_pruning.cardinality_aggregation.max_allowed_cardinality",
295+
100,
296+
0,
297+
Property.Dynamic,
298+
Property.NodeScope
299+
);
300+
292301
public static final int DEFAULT_SIZE = 10;
293302
public static final int DEFAULT_FROM = 0;
294303

0 commit comments

Comments
 (0)