Skip to content

Commit 2c43988

Browse files
author
Jay Deng
committed
Parallelize build agg
1 parent afd3969 commit 2c43988

File tree

12 files changed

+488
-32
lines changed

12 files changed

+488
-32
lines changed

‎modules/lang-painless/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin
3333

3434
apply plugin: 'opensearch.validate-rest-spec'
3535
apply plugin: 'opensearch.yaml-rest-test'
36+
apply plugin: 'opensearch.internal-cluster-test'
3637

3738
opensearchplugin {
3839
description 'An easy, safe and fast scripting language for OpenSearch'

‎modules/lang-painless/src/internalClusterTest/java/org/opensearch/painless/TemporaryIT.java

+327
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.painless;
10+
11+
import org.opensearch.action.search.SearchResponse;
12+
import org.opensearch.cluster.metadata.IndexMetadata;
13+
import org.opensearch.common.settings.Settings;
14+
import org.opensearch.common.xcontent.XContentFactory;
15+
import org.opensearch.core.xcontent.MediaTypeRegistry;
16+
import org.opensearch.core.xcontent.XContentBuilder;
17+
import org.opensearch.index.query.QueryBuilder;
18+
import org.opensearch.plugins.Plugin;
19+
import org.opensearch.script.Script;
20+
import org.opensearch.script.ScriptType;
21+
import org.opensearch.search.SearchService;
22+
import org.opensearch.search.aggregations.AggregationBuilder;
23+
import org.opensearch.search.aggregations.AggregationBuilders;
24+
import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder;
25+
import org.opensearch.search.sort.FieldSortBuilder;
26+
import org.opensearch.search.sort.SortOrder;
27+
import org.opensearch.test.OpenSearchIntegTestCase;
28+
29+
import java.io.IOException;
30+
import java.util.Collection;
31+
import java.util.Collections;
32+
import java.util.List;
33+
34+
import static org.opensearch.index.query.QueryBuilders.boolQuery;
35+
import static org.opensearch.index.query.QueryBuilders.termQuery;
36+
import static org.opensearch.search.aggregations.PipelineAggregatorBuilders.bucketSort;
37+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
38+
39+
public class TemporaryIT extends OpenSearchIntegTestCase {
40+
@Override
41+
protected Settings nodeSettings(int nodeOrdinal) {
42+
return Settings.builder()
43+
.put(super.nodeSettings(nodeOrdinal))
44+
.put(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true)
45+
.build();
46+
}
47+
48+
@Override
49+
protected Collection<Class<? extends Plugin>> nodePlugins() {
50+
return List.of(PainlessModulePlugin.class);
51+
}
52+
53+
private void createTestIndex() throws IOException {
54+
XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()
55+
.startObject()
56+
.field("dynamic", "false")
57+
.startObject("_meta")
58+
.field("schema_version", 5)
59+
.endObject()
60+
.startObject("properties")
61+
.startObject("anomaly_grade")
62+
.field("type", "double")
63+
.endObject()
64+
.startObject("anomaly_score")
65+
.field("type", "double")
66+
.endObject()
67+
.startObject("approx_anomaly_start_time")
68+
.field("type", "date")
69+
.field("format", "strict_date_time||epoch_millis")
70+
.endObject()
71+
.startObject("confidence")
72+
.field("type", "double")
73+
.endObject()
74+
.startObject("data_end_time")
75+
.field("type", "date")
76+
.field("format", "strict_date_time||epoch_millis")
77+
.endObject()
78+
.startObject("data_start_time")
79+
.field("type", "date")
80+
.field("format", "strict_date_time||epoch_millis")
81+
.endObject()
82+
.startObject("detector_id")
83+
.field("type", "keyword")
84+
.endObject()
85+
.startObject("entity")
86+
.field("type", "nested")
87+
.startObject("properties")
88+
.startObject("name")
89+
.field("type", "keyword")
90+
.endObject()
91+
.startObject("value")
92+
.field("type", "keyword")
93+
.endObject()
94+
.endObject()
95+
.endObject()
96+
.startObject("error")
97+
.field("type", "text")
98+
.endObject()
99+
.startObject("execution_end_time")
100+
.field("type", "date")
101+
.field("format", "strict_date_time||epoch_millis")
102+
.endObject()
103+
.startObject("execution_start_time")
104+
.field("type", "date")
105+
.field("format", "strict_date_time||epoch_millis")
106+
.endObject()
107+
.startObject("expected_values")
108+
.field("type", "nested")
109+
.startObject("properties")
110+
.startObject("likelihood")
111+
.field("type", "double")
112+
.endObject()
113+
.startObject("value_list")
114+
.field("type", "nested")
115+
.startObject("properties")
116+
.startObject("data")
117+
.field("type", "double")
118+
.endObject()
119+
.startObject("feature_id")
120+
.field("type", "keyword")
121+
.endObject()
122+
.endObject()
123+
.endObject()
124+
.endObject()
125+
.endObject()
126+
.startObject("feature_data")
127+
.field("type", "nested")
128+
.startObject("properties")
129+
.startObject("data")
130+
.field("type", "double")
131+
.endObject()
132+
.startObject("feature_id")
133+
.field("type", "keyword")
134+
.endObject()
135+
.endObject()
136+
.endObject()
137+
.startObject("is_anomaly")
138+
.field("type", "boolean")
139+
.endObject()
140+
.startObject("model_id")
141+
.field("type", "keyword")
142+
.endObject()
143+
.startObject("past_values")
144+
.field("type", "nested")
145+
.startObject("properties")
146+
.startObject("data")
147+
.field("type", "double")
148+
.endObject()
149+
.startObject("feature_id")
150+
.field("type", "keyword")
151+
.endObject()
152+
.endObject()
153+
.endObject()
154+
.startObject("relevant_attribution")
155+
.field("type", "nested")
156+
.startObject("properties")
157+
.startObject("data")
158+
.field("type", "double")
159+
.endObject()
160+
.startObject("feature_id")
161+
.field("type", "keyword")
162+
.endObject()
163+
.endObject()
164+
.endObject()
165+
.startObject("schema_version")
166+
.field("type", "integer")
167+
.endObject()
168+
.startObject("task_id")
169+
.field("type", "keyword")
170+
.endObject()
171+
.startObject("threshold")
172+
.field("type", "double")
173+
.endObject()
174+
.startObject("user")
175+
.field("type", "nested")
176+
.startObject("properties")
177+
.startObject("backend_roles")
178+
.field("type", "text")
179+
.startObject("fields")
180+
.startObject("keyword")
181+
.field("type", "keyword")
182+
.endObject()
183+
.endObject()
184+
.endObject()
185+
.startObject("custom_attribute_names")
186+
.field("type", "text")
187+
.startObject("fields")
188+
.startObject("keyword")
189+
.field("type", "keyword")
190+
.endObject()
191+
.endObject()
192+
.endObject()
193+
.startObject("name")
194+
.field("type", "text")
195+
.startObject("fields")
196+
.startObject("keyword")
197+
.field("type", "keyword")
198+
.field("ignore_above", 256)
199+
.endObject()
200+
.endObject()
201+
.endObject()
202+
.startObject("roles")
203+
.field("type", "text")
204+
.startObject("fields")
205+
.startObject("keyword")
206+
.field("type", "keyword")
207+
.endObject()
208+
.endObject()
209+
.endObject()
210+
.endObject()
211+
.endObject()
212+
.endObject()
213+
.endObject();
214+
215+
assertAcked(
216+
prepareCreate("test").setMapping(xContentBuilder)
217+
.setSettings(
218+
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
219+
)
220+
);
221+
}
222+
223+
private void indexTestData() {
224+
client().prepareIndex("test")
225+
.setId("gRbUF")
226+
.setSource(
227+
"{\"detector_id\":\"VqbXro0B0N8KJjAbG28Y\",\"schema_version\":0,\"data_start_time\":5,\"data_end_time\":5,\"feature_data\":[{\"feature_id\":\"WQgvo\",\"feature_name\":\"PVhgc\",\"data\":0.9212883816892278},{\"feature_id\":\"JulWB\",\"feature_name\":\"HgOGN\",\"data\":0.27831399526601086}],\"execution_start_time\":5,\"execution_end_time\":5,\"anomaly_score\":0.5,\"anomaly_grade\":0.8,\"confidence\":0.1705822118682151,\"entity\":[{\"name\":\"ip-field\",\"value\":\"1.2.3.4\"},{\"name\":\"keyword-field\",\"value\":\"field-1\"}],\"user\":{\"name\":\"PBJzgZpg\",\"backend_roles\":[\"giOWwAZcpU\"],\"roles\":[\"all_access\"],\"custom_attribute_names\":[\"attribute=test\"],\"user_requested_tenant\":null},\"approx_anomaly_start_time\":1708035355000,\"relevant_attribution\":[{\"feature_id\":\"piyfg\",\"data\":0.7797511350635153},{\"feature_id\":\"pFhPl\",\"data\":0.680814523323366}],\"past_values\":[{\"feature_id\":\"mECeN\",\"data\":0.8577224651498027},{\"feature_id\":\"SSHho\",\"data\":0.36525036781711573}],\"expected_values\":[{\"likelihood\":0.712699398152217,\"value_list\":[{\"feature_id\":\"wOPWI\",\"data\":0.09344528571943234},{\"feature_id\":\"HMZbM\",\"data\":0.8899196238445849}]}],\"threshold\":7.513042281539716}",
228+
MediaTypeRegistry.JSON
229+
)
230+
.get();
231+
client().prepareIndex("test")
232+
.setId("vWCJa")
233+
.setSource(
234+
"{\"detector_id\":\"VqbXro0B0N8KJjAbG28Y\",\"schema_version\":0,\"data_start_time\":5,\"data_end_time\":5,\"feature_data\":[{\"feature_id\":\"Lmcsm\",\"feature_name\":\"iDXfc\",\"data\":0.9674434291471465},{\"feature_id\":\"qSUQl\",\"feature_name\":\"qbEoF\",\"data\":0.6504223878706881}],\"execution_start_time\":5,\"execution_end_time\":5,\"anomaly_score\":0.5,\"anomaly_grade\":0.5,\"confidence\":0.06614591879270315,\"entity\":[{\"name\":\"ip-field\",\"value\":\"5.6.7.8\"},{\"name\":\"keyword-field\",\"value\":\"field-2\"}],\"user\":{\"name\":\"dJHBbnuu\",\"backend_roles\":[\"HXqCilWVMf\"],\"roles\":[\"all_access\"],\"custom_attribute_names\":[\"attribute=test\"],\"user_requested_tenant\":null},\"approx_anomaly_start_time\":1708035355000,\"relevant_attribution\":[{\"feature_id\":\"Ufhtc\",\"data\":0.08750171412108843},{\"feature_id\":\"uyJWb\",\"data\":0.9333680688095377}],\"past_values\":[{\"feature_id\":\"qskfI\",\"data\":0.970802420410941},{\"feature_id\":\"gYdme\",\"data\":0.847333030542884}],\"expected_values\":[{\"likelihood\":0.001994250912530804,\"value_list\":[{\"feature_id\":\"pnLad\",\"data\":0.1614332721050905},{\"feature_id\":\"BtBBh\",\"data\":0.5734485976838636}]}],\"threshold\":8.580216939299472}",
235+
MediaTypeRegistry.JSON
236+
)
237+
.get();
238+
client().prepareIndex("test")
239+
.setId("VnVkC")
240+
.setSource(
241+
"{\"detector_id\":\"VqbXro0B0N8KJjAbG28Y\",\"schema_version\":0,\"data_start_time\":5,\"data_end_time\":5,\"feature_data\":[{\"feature_id\":\"IqHwm\",\"feature_name\":\"LCnRh\",\"data\":0.8929177514663842},{\"feature_id\":\"IcaxA\",\"feature_name\":\"HLuxV\",\"data\":0.8975549333747292}],\"execution_start_time\":5,\"execution_end_time\":5,\"anomaly_score\":0.5,\"anomaly_grade\":0.2,\"confidence\":0.06244189871920458,\"entity\":[{\"name\":\"ip-field\",\"value\":\"5.6.7.8\"},{\"name\":\"keyword-field\",\"value\":\"field-2\"}],\"user\":{\"name\":\"IBhQUsrP\",\"backend_roles\":[\"AeewVXqCYO\"],\"roles\":[\"all_access\"],\"custom_attribute_names\":[\"attribute=test\"],\"user_requested_tenant\":null},\"approx_anomaly_start_time\":1708035355000,\"relevant_attribution\":[{\"feature_id\":\"EptJC\",\"data\":0.6875058309428451},{\"feature_id\":\"IKFpg\",\"data\":0.3419015294070341}],\"past_values\":[{\"feature_id\":\"KnVpN\",\"data\":0.7255993126008243},{\"feature_id\":\"NxgkL\",\"data\":0.6884725049479412}],\"expected_values\":[{\"likelihood\":0.7352436055910023,\"value_list\":[{\"feature_id\":\"Cvddb\",\"data\":0.7457298326060673},{\"feature_id\":\"QhtZU\",\"data\":0.7327525344956058}]}],\"threshold\":6.517648854225251}",
242+
MediaTypeRegistry.JSON
243+
)
244+
.get();
245+
refresh("test");
246+
}
247+
248+
public void test() throws Exception {
249+
createTestIndex();
250+
indexTestData();
251+
252+
/**
253+
* curl "localhost:57523/.opendistro-anomaly-results/_search?pretty" -H 'Content-Type: application/json' -d'
254+
* quote> {
255+
* "query": {
256+
* "bool": {
257+
* "filter": {
258+
* "term": {
259+
* "detector_id": "Ue39ro0BJngQavFLX2Q-"
260+
* }
261+
* }
262+
* }
263+
* },
264+
* "aggs": {
265+
* "multi_buckets": {
266+
* "composite": {
267+
* "sources": [{
268+
* "keyword-field": {
269+
* "terms": {
270+
* "script": {
271+
* "source": "String value = null; if (params == null || params._source == null || params._source.entity == null) { return \"\"; } for (item in params._source.entity) { if (item[\"name\"] == \"keyword-field\") { value = item['value']; break; } } return value;",
272+
* "lang": "painless"
273+
* }
274+
* }
275+
* }
276+
* }]
277+
* },
278+
* "aggregations": {
279+
* "max": {
280+
* "max": {
281+
* "field": "anomaly_grade"
282+
* }
283+
* },
284+
* "multi_buckets_sort": {
285+
* "bucket_sort": {
286+
* "sort": [{
287+
* "max": {
288+
* "order": "desc"
289+
* }
290+
* }],
291+
* "size": 10
292+
* }
293+
* }
294+
* }
295+
* }
296+
* }
297+
* }'
298+
*/
299+
300+
QueryBuilder query = boolQuery().filter(termQuery("detector_id", "VqbXro0B0N8KJjAbG28Y"));
301+
302+
AggregationBuilder agg = AggregationBuilders.composite(
303+
"multi_buckets",
304+
Collections.singletonList(
305+
new TermsValuesSourceBuilder("keyword-field").script(
306+
new Script(
307+
ScriptType.INLINE,
308+
"painless",
309+
"String value = null; if (params == null || params._source == null || params._source.entity == null) { return \"\"; } for (item in params._source.entity) { if (item[\"name\"] == \"keyword-field\") { value = item['value']; break; } } return value;",
310+
Collections.emptyMap()
311+
)
312+
)
313+
)
314+
)
315+
.subAggregation(AggregationBuilders.max("max").field("anomaly_grade"))
316+
.subAggregation(
317+
bucketSort("multi_buckets_sort", Collections.singletonList(new FieldSortBuilder("max").order(SortOrder.DESC))).size(10)
318+
);
319+
320+
// System.out.println(query);
321+
// System.out.println(agg);
322+
323+
SearchResponse response = client().prepareSearch("test").setQuery(query).addAggregation(agg).get();
324+
325+
// System.out.println(response);
326+
}
327+
}

0 commit comments

Comments
 (0)