Skip to content

Commit a9c03c2

Browse files
[Backport 2.x] Apply fast date histogram optimization at the segment level (opensearch-project#12279) (opensearch-project#12284)
* Apply fast date histogram optimization at the segment level (opensearch-project#12073) --------- (cherry picked from commit 9a0a69f) * Fix: reset the filter built at segment level for date histogram optimization (opensearch-project#12267) --------- --------- (cherry picked from commit 25c2fde) Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com> Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent eafa416 commit a9c03c2

File tree

9 files changed

+728
-179
lines changed

9 files changed

+728
-179
lines changed

server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/DateHistogramIT.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -177,9 +177,9 @@ public void setupSuiteScopeCluster() throws Exception {
177177
indexDoc(2, 15, 3), // date: Feb 15, dates: Feb 15, Mar 16
178178
indexDoc(3, 2, 4), // date: Mar 2, dates: Mar 2, Apr 3
179179
indexDoc(3, 15, 5), // date: Mar 15, dates: Mar 15, Apr 16
180-
indexDoc(3, 23, 6)
180+
indexDoc(3, 23, 6) // date: Mar 23, dates: Mar 23, Apr 24
181181
)
182-
); // date: Mar 23, dates: Mar 23, Apr 24
182+
);
183183
indexRandom(true, builders);
184184
ensureSearchable();
185185
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.search.aggregations.bucket;
10+
11+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
12+
13+
import org.opensearch.action.index.IndexRequestBuilder;
14+
import org.opensearch.action.search.SearchResponse;
15+
import org.opensearch.common.settings.Settings;
16+
import org.opensearch.common.time.DateFormatter;
17+
import org.opensearch.index.query.QueryBuilder;
18+
import org.opensearch.index.query.QueryBuilders;
19+
import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
20+
import org.opensearch.search.aggregations.bucket.histogram.Histogram;
21+
import org.opensearch.test.OpenSearchIntegTestCase;
22+
import org.opensearch.test.ParameterizedDynamicSettingsOpenSearchIntegTestCase;
23+
24+
import java.time.ZoneOffset;
25+
import java.time.ZonedDateTime;
26+
import java.util.ArrayList;
27+
import java.util.Arrays;
28+
import java.util.Collection;
29+
import java.util.HashMap;
30+
import java.util.HashSet;
31+
import java.util.List;
32+
import java.util.Map;
33+
import java.util.Set;
34+
35+
import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
36+
import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
37+
import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram;
38+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
39+
40+
@OpenSearchIntegTestCase.SuiteScopeTestCase
41+
public class FilterRewriteIT extends ParameterizedDynamicSettingsOpenSearchIntegTestCase {
42+
43+
// simulate segment level match all
44+
private static final QueryBuilder QUERY = QueryBuilders.termQuery("match", true);
45+
private static final Map<String, Long> expected = new HashMap<>();
46+
47+
public FilterRewriteIT(Settings dynamicSettings) {
48+
super(dynamicSettings);
49+
}
50+
51+
@ParametersFactory
52+
public static Collection<Object[]> parameters() {
53+
return Arrays.asList(
54+
new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), false).build() },
55+
new Object[] { Settings.builder().put(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true).build() }
56+
);
57+
}
58+
59+
@Override
60+
protected void setupSuiteScopeCluster() throws Exception {
61+
assertAcked(client().admin().indices().prepareCreate("idx").get());
62+
63+
final int segmentCount = randomIntBetween(2, 10);
64+
final Set<Long> longTerms = new HashSet();
65+
66+
final Map<String, Integer> dateTerms = new HashMap<>();
67+
for (int i = 0; i < segmentCount; i++) {
68+
final List<IndexRequestBuilder> indexRequests = new ArrayList<>();
69+
70+
long longTerm;
71+
do {
72+
longTerm = randomInt(segmentCount * 2);
73+
} while (!longTerms.add(longTerm));
74+
ZonedDateTime time = ZonedDateTime.of(2024, 1, ((int) longTerm % 20) + 1, 0, 0, 0, 0, ZoneOffset.UTC);
75+
String dateTerm = DateFormatter.forPattern("yyyy-MM-dd").format(time);
76+
77+
final int frequency = randomBoolean() ? 1 : randomIntBetween(2, 20);
78+
for (int j = 0; j < frequency; j++) {
79+
indexRequests.add(
80+
client().prepareIndex("idx")
81+
.setSource(jsonBuilder().startObject().field("date", dateTerm).field("match", true).endObject())
82+
);
83+
}
84+
expected.put(dateTerm + "T00:00:00.000Z", (long) frequency);
85+
86+
indexRandom(true, false, indexRequests);
87+
}
88+
89+
ensureSearchable();
90+
}
91+
92+
public void testMinDocCountOnDateHistogram() throws Exception {
93+
final SearchResponse allResponse = client().prepareSearch("idx")
94+
.setSize(0)
95+
.setQuery(QUERY)
96+
.addAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(0))
97+
.get();
98+
99+
final Histogram allHisto = allResponse.getAggregations().get("histo");
100+
Map<String, Long> results = new HashMap<>();
101+
allHisto.getBuckets().forEach(bucket -> results.put(bucket.getKeyAsString(), bucket.getDocCount()));
102+
103+
for (Map.Entry<String, Long> entry : expected.entrySet()) {
104+
assertEquals(entry.getValue(), results.get(entry.getKey()));
105+
}
106+
}
107+
}

0 commit comments

Comments
 (0)