Skip to content

Commit ab8ebcd

Browse files
author
Jay Deng
committed
TMP
1 parent afd3969 commit ab8ebcd

File tree

8 files changed

+303
-26
lines changed

8 files changed

+303
-26
lines changed

‎modules/lang-painless/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin
3333

3434
apply plugin: 'opensearch.validate-rest-spec'
3535
apply plugin: 'opensearch.yaml-rest-test'
36+
apply plugin: 'opensearch.internal-cluster-test'
3637

3738
opensearchplugin {
3839
description 'An easy, safe and fast scripting language for OpenSearch'

‎modules/lang-painless/src/internalClusterTest/java/org/opensearch/painless/TemporaryIT.java

+217
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.painless;
10+
11+
import org.opensearch.action.search.SearchResponse;
12+
import org.opensearch.cluster.metadata.IndexMetadata;
13+
import org.opensearch.common.settings.Settings;
14+
import org.opensearch.common.xcontent.XContentFactory;
15+
import org.opensearch.core.xcontent.MediaTypeRegistry;
16+
import org.opensearch.core.xcontent.XContentBuilder;
17+
import org.opensearch.index.query.QueryBuilder;
18+
import org.opensearch.plugins.Plugin;
19+
import org.opensearch.script.Script;
20+
import org.opensearch.script.ScriptType;
21+
import org.opensearch.search.SearchService;
22+
import org.opensearch.search.aggregations.AggregationBuilder;
23+
import org.opensearch.search.aggregations.AggregationBuilders;
24+
import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder;
25+
import org.opensearch.search.sort.FieldSortBuilder;
26+
import org.opensearch.search.sort.SortOrder;
27+
import org.opensearch.test.OpenSearchIntegTestCase;
28+
29+
import java.io.IOException;
30+
import java.util.Collection;
31+
import java.util.Collections;
32+
import java.util.List;
33+
34+
import static org.opensearch.index.query.QueryBuilders.boolQuery;
35+
import static org.opensearch.index.query.QueryBuilders.termQuery;
36+
import static org.opensearch.search.aggregations.PipelineAggregatorBuilders.bucketSort;
37+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
38+
39+
public class TemporaryIT extends OpenSearchIntegTestCase {
40+
@Override
41+
protected Settings nodeSettings(int nodeOrdinal) {
42+
return Settings.builder()
43+
.put(super.nodeSettings(nodeOrdinal))
44+
.put(SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(), true)
45+
.build();
46+
}
47+
48+
@Override
49+
protected Collection<Class<? extends Plugin>> nodePlugins() {
50+
return List.of(PainlessModulePlugin.class);
51+
}
52+
53+
private void createTestIndex() throws IOException {
54+
XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject()
55+
.field("dynamic", "false")
56+
.startObject("_meta")
57+
.field("schema_version", 5)
58+
.endObject()
59+
.startObject("properties")
60+
.startObject("anomaly_grade").field("type", "double").endObject()
61+
.startObject("anomaly_score").field("type", "double").endObject()
62+
.startObject("approx_anomaly_start_time").field("type", "date").field("format", "strict_date_time||epoch_millis").endObject()
63+
.startObject("confidence").field("type", "double").endObject()
64+
.startObject("data_end_time").field("type", "date").field("format", "strict_date_time||epoch_millis").endObject()
65+
.startObject("data_start_time").field("type", "date").field("format", "strict_date_time||epoch_millis").endObject()
66+
.startObject("detector_id").field("type", "keyword").endObject()
67+
.startObject("entity").field("type", "nested")
68+
.startObject("properties")
69+
.startObject("name").field("type", "keyword").endObject()
70+
.startObject("value").field("type", "keyword").endObject()
71+
.endObject()
72+
.endObject()
73+
.startObject("error").field("type", "text").endObject()
74+
.startObject("execution_end_time").field("type", "date").field("format", "strict_date_time||epoch_millis").endObject()
75+
.startObject("execution_start_time").field("type", "date").field("format", "strict_date_time||epoch_millis").endObject()
76+
.startObject("expected_values").field("type", "nested")
77+
.startObject("properties")
78+
.startObject("likelihood").field("type", "double").endObject()
79+
.startObject("value_list").field("type", "nested")
80+
.startObject("properties")
81+
.startObject("data").field("type", "double").endObject()
82+
.startObject("feature_id").field("type", "keyword").endObject()
83+
.endObject()
84+
.endObject()
85+
.endObject()
86+
.endObject()
87+
.startObject("feature_data").field("type", "nested")
88+
.startObject("properties")
89+
.startObject("data").field("type", "double").endObject()
90+
.startObject("feature_id").field("type", "keyword").endObject()
91+
.endObject()
92+
.endObject()
93+
.startObject("is_anomaly").field("type", "boolean").endObject()
94+
.startObject("model_id").field("type", "keyword").endObject()
95+
.startObject("past_values").field("type", "nested")
96+
.startObject("properties")
97+
.startObject("data").field("type", "double").endObject()
98+
.startObject("feature_id").field("type", "keyword").endObject()
99+
.endObject()
100+
.endObject()
101+
.startObject("relevant_attribution").field("type", "nested")
102+
.startObject("properties")
103+
.startObject("data").field("type", "double").endObject()
104+
.startObject("feature_id").field("type", "keyword").endObject()
105+
.endObject()
106+
.endObject()
107+
.startObject("schema_version").field("type", "integer").endObject()
108+
.startObject("task_id").field("type", "keyword").endObject()
109+
.startObject("threshold").field("type", "double").endObject()
110+
.startObject("user").field("type", "nested")
111+
.startObject("properties")
112+
.startObject("backend_roles").field("type", "text").startObject("fields").startObject("keyword").field("type", "keyword").endObject().endObject().endObject()
113+
.startObject("custom_attribute_names").field("type", "text").startObject("fields").startObject("keyword").field("type", "keyword").endObject().endObject().endObject()
114+
.startObject("name").field("type", "text").startObject("fields").startObject("keyword").field("type", "keyword").field("ignore_above", 256).endObject().endObject().endObject()
115+
.startObject("roles").field("type", "text").startObject("fields").startObject("keyword").field("type", "keyword").endObject().endObject().endObject()
116+
.endObject()
117+
.endObject()
118+
.endObject()
119+
.endObject();
120+
121+
assertAcked(
122+
prepareCreate("test")
123+
.setMapping(xContentBuilder)
124+
.setSettings(
125+
Settings.builder()
126+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
127+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
128+
)
129+
);
130+
}
131+
132+
private void indexTestData() {
133+
client().prepareIndex("test").setId("gRbUF").setSource("{\"detector_id\":\"VqbXro0B0N8KJjAbG28Y\",\"schema_version\":0,\"data_start_time\":5,\"data_end_time\":5,\"feature_data\":[{\"feature_id\":\"WQgvo\",\"feature_name\":\"PVhgc\",\"data\":0.9212883816892278},{\"feature_id\":\"JulWB\",\"feature_name\":\"HgOGN\",\"data\":0.27831399526601086}],\"execution_start_time\":5,\"execution_end_time\":5,\"anomaly_score\":0.5,\"anomaly_grade\":0.8,\"confidence\":0.1705822118682151,\"entity\":[{\"name\":\"ip-field\",\"value\":\"1.2.3.4\"},{\"name\":\"keyword-field\",\"value\":\"field-1\"}],\"user\":{\"name\":\"PBJzgZpg\",\"backend_roles\":[\"giOWwAZcpU\"],\"roles\":[\"all_access\"],\"custom_attribute_names\":[\"attribute=test\"],\"user_requested_tenant\":null},\"approx_anomaly_start_time\":1708035355000,\"relevant_attribution\":[{\"feature_id\":\"piyfg\",\"data\":0.7797511350635153},{\"feature_id\":\"pFhPl\",\"data\":0.680814523323366}],\"past_values\":[{\"feature_id\":\"mECeN\",\"data\":0.8577224651498027},{\"feature_id\":\"SSHho\",\"data\":0.36525036781711573}],\"expected_values\":[{\"likelihood\":0.712699398152217,\"value_list\":[{\"feature_id\":\"wOPWI\",\"data\":0.09344528571943234},{\"feature_id\":\"HMZbM\",\"data\":0.8899196238445849}]}],\"threshold\":7.513042281539716}", MediaTypeRegistry.JSON).get();
134+
client().prepareIndex("test").setId("vWCJa").setSource("{\"detector_id\":\"VqbXro0B0N8KJjAbG28Y\",\"schema_version\":0,\"data_start_time\":5,\"data_end_time\":5,\"feature_data\":[{\"feature_id\":\"Lmcsm\",\"feature_name\":\"iDXfc\",\"data\":0.9674434291471465},{\"feature_id\":\"qSUQl\",\"feature_name\":\"qbEoF\",\"data\":0.6504223878706881}],\"execution_start_time\":5,\"execution_end_time\":5,\"anomaly_score\":0.5,\"anomaly_grade\":0.5,\"confidence\":0.06614591879270315,\"entity\":[{\"name\":\"ip-field\",\"value\":\"5.6.7.8\"},{\"name\":\"keyword-field\",\"value\":\"field-2\"}],\"user\":{\"name\":\"dJHBbnuu\",\"backend_roles\":[\"HXqCilWVMf\"],\"roles\":[\"all_access\"],\"custom_attribute_names\":[\"attribute=test\"],\"user_requested_tenant\":null},\"approx_anomaly_start_time\":1708035355000,\"relevant_attribution\":[{\"feature_id\":\"Ufhtc\",\"data\":0.08750171412108843},{\"feature_id\":\"uyJWb\",\"data\":0.9333680688095377}],\"past_values\":[{\"feature_id\":\"qskfI\",\"data\":0.970802420410941},{\"feature_id\":\"gYdme\",\"data\":0.847333030542884}],\"expected_values\":[{\"likelihood\":0.001994250912530804,\"value_list\":[{\"feature_id\":\"pnLad\",\"data\":0.1614332721050905},{\"feature_id\":\"BtBBh\",\"data\":0.5734485976838636}]}],\"threshold\":8.580216939299472}", MediaTypeRegistry.JSON).get();
135+
client().prepareIndex("test").setId("VnVkC").setSource("{\"detector_id\":\"VqbXro0B0N8KJjAbG28Y\",\"schema_version\":0,\"data_start_time\":5,\"data_end_time\":5,\"feature_data\":[{\"feature_id\":\"IqHwm\",\"feature_name\":\"LCnRh\",\"data\":0.8929177514663842},{\"feature_id\":\"IcaxA\",\"feature_name\":\"HLuxV\",\"data\":0.8975549333747292}],\"execution_start_time\":5,\"execution_end_time\":5,\"anomaly_score\":0.5,\"anomaly_grade\":0.2,\"confidence\":0.06244189871920458,\"entity\":[{\"name\":\"ip-field\",\"value\":\"5.6.7.8\"},{\"name\":\"keyword-field\",\"value\":\"field-2\"}],\"user\":{\"name\":\"IBhQUsrP\",\"backend_roles\":[\"AeewVXqCYO\"],\"roles\":[\"all_access\"],\"custom_attribute_names\":[\"attribute=test\"],\"user_requested_tenant\":null},\"approx_anomaly_start_time\":1708035355000,\"relevant_attribution\":[{\"feature_id\":\"EptJC\",\"data\":0.6875058309428451},{\"feature_id\":\"IKFpg\",\"data\":0.3419015294070341}],\"past_values\":[{\"feature_id\":\"KnVpN\",\"data\":0.7255993126008243},{\"feature_id\":\"NxgkL\",\"data\":0.6884725049479412}],\"expected_values\":[{\"likelihood\":0.7352436055910023,\"value_list\":[{\"feature_id\":\"Cvddb\",\"data\":0.7457298326060673},{\"feature_id\":\"QhtZU\",\"data\":0.7327525344956058}]}],\"threshold\":6.517648854225251}", MediaTypeRegistry.JSON).get();
136+
refresh("test");
137+
}
138+
139+
public void test() throws Exception {
140+
createTestIndex();
141+
indexTestData();
142+
143+
/**
144+
* curl "localhost:57523/.opendistro-anomaly-results/_search?pretty" -H 'Content-Type: application/json' -d'
145+
* quote> {
146+
* "query": {
147+
* "bool": {
148+
* "filter": {
149+
* "term": {
150+
* "detector_id": "Ue39ro0BJngQavFLX2Q-"
151+
* }
152+
* }
153+
* }
154+
* },
155+
* "aggs": {
156+
* "multi_buckets": {
157+
* "composite": {
158+
* "sources": [{
159+
* "keyword-field": {
160+
* "terms": {
161+
* "script": {
162+
* "source": "String value = null; if (params == null || params._source == null || params._source.entity == null) { return \"\"; } for (item in params._source.entity) { if (item[\"name\"] == \"keyword-field\") { value = item['value']; break; } } return value;",
163+
* "lang": "painless"
164+
* }
165+
* }
166+
* }
167+
* }]
168+
* },
169+
* "aggregations": {
170+
* "max": {
171+
* "max": {
172+
* "field": "anomaly_grade"
173+
* }
174+
* },
175+
* "multi_buckets_sort": {
176+
* "bucket_sort": {
177+
* "sort": [{
178+
* "max": {
179+
* "order": "desc"
180+
* }
181+
* }],
182+
* "size": 10
183+
* }
184+
* }
185+
* }
186+
* }
187+
* }
188+
* }'
189+
*/
190+
191+
QueryBuilder query = boolQuery().filter(
192+
termQuery("detector_id", "VqbXro0B0N8KJjAbG28Y")
193+
);
194+
195+
AggregationBuilder agg = AggregationBuilders.composite("multi_buckets",
196+
Collections.singletonList(new TermsValuesSourceBuilder("keyword-field")
197+
.script(new Script(ScriptType.INLINE, "painless", "String value = null; if (params == null || params._source == null || params._source.entity == null) { return \"\"; } for (item in params._source.entity) { if (item[\"name\"] == \"keyword-field\") { value = item['value']; break; } } return value;", Collections.emptyMap()))
198+
)
199+
).subAggregation(
200+
AggregationBuilders.max("max").field("anomaly_grade")
201+
).subAggregation(
202+
bucketSort("multi_buckets_sort", Collections.singletonList(new FieldSortBuilder("max").order(SortOrder.DESC)))
203+
.size(10)
204+
);
205+
206+
207+
System.out.println(query);
208+
System.out.println(agg);
209+
210+
SearchResponse response = client().prepareSearch("test")
211+
.setQuery(query)
212+
.addAggregation(agg)
213+
.get();
214+
215+
System.out.println(response);
216+
}
217+
}

0 commit comments

Comments
 (0)