Skip to content

Commit 0c8f804

Browse files
deshsidddeshsidd
authored
Add additional grouping ITs and refactor (#89)
Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com>
1 parent 6d0ec13 commit 0c8f804

File tree

5 files changed

+176
-99
lines changed

5 files changed

+176
-99
lines changed

‎src/test/java/org/opensearch/plugin/insights/QueryInsightsRestTestCase.java

+58
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import java.util.List;
1515
import java.util.Map;
1616
import java.util.Optional;
17+
import java.util.function.Supplier;
1718
import java.util.regex.Matcher;
1819
import java.util.regex.Pattern;
1920
import java.util.stream.Collectors;
@@ -45,6 +46,7 @@
4546
import org.opensearch.core.xcontent.MediaType;
4647
import org.opensearch.core.xcontent.NamedXContentRegistry;
4748
import org.opensearch.core.xcontent.XContentParser;
49+
import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
4850
import org.opensearch.test.rest.OpenSearchRestTestCase;
4951

5052
public abstract class QueryInsightsRestTestCase extends OpenSearchRestTestCase {
@@ -320,4 +322,60 @@ protected void waitForEmptyTopQueriesResponse() throws IOException, InterruptedE
320322
}
321323
}
322324

325+
protected void assertTopQueriesCount(int expectedTopQueriesCount, String type) throws IOException, InterruptedException {
326+
// Ensure records are drained to the top queries service
327+
Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
328+
329+
// run five times to make sure the records are drained to the top queries services
330+
for (int i = 0; i < 5; i++) {
331+
String responseBody = getTopQueries(type);
332+
333+
int topNArraySize = countTopQueries(responseBody);
334+
335+
if (topNArraySize < expectedTopQueriesCount) {
336+
// Ensure records are drained to the top queries service
337+
Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
338+
continue;
339+
}
340+
341+
// Validate that all queries are listed separately (no grouping)
342+
Assert.assertEquals(expectedTopQueriesCount, topNArraySize);
343+
}
344+
}
345+
346+
protected String getTopQueries(String type) throws IOException {
347+
// Base URL
348+
String endpoint = "/_insights/top_queries?pretty";
349+
350+
if (type != null) {
351+
switch (type) {
352+
case "cpu":
353+
case "memory":
354+
case "latency":
355+
endpoint = "/_insights/top_queries?type=" + type + "&pretty";
356+
break;
357+
case "all":
358+
// Keep the default endpoint (no type parameter)
359+
break;
360+
default:
361+
// Throw an exception if the type is invalid
362+
throw new IllegalArgumentException("Invalid type: " + type + ". Valid types are 'all', 'cpu', 'memory', or 'latency'.");
363+
}
364+
}
365+
366+
Request request = new Request("GET", endpoint);
367+
Response response = client().performRequest(request);
368+
369+
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
370+
371+
String responseBody = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
372+
return responseBody;
373+
}
374+
375+
protected void updateClusterSettings(Supplier<String> settingsSupplier) throws IOException {
376+
Request request = new Request("PUT", "/_cluster/settings");
377+
request.setJsonEntity(settingsSupplier.get());
378+
Response response = client().performRequest(request);
379+
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
380+
}
323381
}

‎src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxQueryGrouperByNoneIT.java

+2-26
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,7 @@
88
package org.opensearch.plugin.insights.core.service.grouper;
99

1010
import java.io.IOException;
11-
import java.nio.charset.StandardCharsets;
12-
import org.junit.Assert;
13-
import org.opensearch.client.Request;
14-
import org.opensearch.client.Response;
1511
import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
16-
import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
1712

1813
/**
1914
* ITs for Grouping Top Queries by none
@@ -29,33 +24,14 @@ public void testGroupingByNone() throws IOException, InterruptedException {
2924

3025
waitForEmptyTopQueriesResponse();
3126

32-
// Enable top N feature and grouping by none
33-
Request request = new Request("PUT", "/_cluster/settings");
34-
request.setJsonEntity(groupByNoneSettings());
35-
Response response = client().performRequest(request);
36-
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
27+
updateClusterSettings(this::groupByNoneSettings);
3728

3829
// Search
3930
doSearch("range", 2);
4031
doSearch("match", 6);
4132
doSearch("term", 4);
4233

43-
// Ensure records are drained to the top queries service
44-
Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
45-
46-
// run five times to make sure the records are drained to the top queries services
47-
for (int i = 0; i < 5; i++) {
48-
// Get Top Queries and validate
49-
request = new Request("GET", "/_insights/top_queries?pretty");
50-
response = client().performRequest(request);
51-
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
52-
String top_requests = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
53-
54-
int top_n_array_size = countTopQueries(top_requests);
55-
56-
// Validate that all queries are listed separately (no grouping)
57-
Assert.assertEquals(12, top_n_array_size);
58-
}
34+
assertTopQueriesCount(12, "latency");
5935
}
6036

6137
private String groupByNoneSettings() {

‎src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxQueryGrouperBySimilarityIT.java

+2-27
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,10 @@
88
package org.opensearch.plugin.insights.core.service.grouper;
99

1010
import java.io.IOException;
11-
import java.nio.charset.StandardCharsets;
12-
import org.junit.Assert;
1311
import org.opensearch.client.Request;
1412
import org.opensearch.client.Response;
1513
import org.opensearch.client.ResponseException;
1614
import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
17-
import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
1815

1916
/**
2017
* ITs for Grouping Top Queries by similarity
@@ -30,36 +27,14 @@ public void testGroupingBySimilarity() throws IOException, InterruptedException
3027

3128
waitForEmptyTopQueriesResponse();
3229

33-
// Enable top N feature and grouping feature
34-
Request request = new Request("PUT", "/_cluster/settings");
35-
request.setJsonEntity(defaultTopQueryGroupingSettings());
36-
Response response = client().performRequest(request);
37-
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
30+
updateClusterSettings(this::defaultTopQueryGroupingSettings);
3831

3932
// Search
4033
doSearch("range", 2);
4134
doSearch("match", 6);
4235
doSearch("term", 4);
4336

44-
// run five times to make sure the records are drained to the top queries services
45-
for (int i = 0; i < 5; i++) {
46-
// Get Top Queries
47-
request = new Request("GET", "/_insights/top_queries?pretty");
48-
response = client().performRequest(request);
49-
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
50-
51-
String responseBody = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
52-
53-
// Extract and count top_queries
54-
int topNArraySize = countTopQueries(responseBody);
55-
56-
if (topNArraySize == 0) {
57-
Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
58-
continue;
59-
}
60-
61-
Assert.assertEquals(3, topNArraySize);
62-
}
37+
assertTopQueriesCount(3, "latency");
6338
}
6439

6540
/**

‎src/test/java/org/opensearch/plugin/insights/core/service/grouper/MinMaxQueryGrouperIT.java

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
package org.opensearch.plugin.insights.core.service.grouper;
9+
10+
import java.io.IOException;
11+
import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
12+
13+
public class MinMaxQueryGrouperIT extends QueryInsightsRestTestCase {
14+
/**
15+
* Grouping by none should not group queries
16+
* @throws IOException
17+
* @throws InterruptedException
18+
*/
19+
public void testNoneToSimilarityGroupingTransition() throws IOException, InterruptedException {
20+
21+
waitForEmptyTopQueriesResponse();
22+
23+
updateClusterSettings(this::defaultTopQueriesSettings);
24+
25+
// Search
26+
doSearch("range", 2);
27+
doSearch("match", 6);
28+
doSearch("term", 4);
29+
30+
assertTopQueriesCount(12, "latency");
31+
32+
updateClusterSettings(this::defaultTopQueryGroupingSettings);
33+
34+
// Top queries should be drained due to grouping change from NONE -> SIMILARITY
35+
assertTopQueriesCount(0, "latency");
36+
37+
// Search
38+
doSearch("range", 2);
39+
doSearch("match", 6);
40+
doSearch("term", 4);
41+
42+
// 3 groups
43+
assertTopQueriesCount(3, "latency");
44+
}
45+
46+
public void testSimilarityToNoneGroupingTransition() throws IOException, InterruptedException {
47+
48+
waitForEmptyTopQueriesResponse();
49+
50+
updateClusterSettings(this::defaultTopQueryGroupingSettings);
51+
52+
// Search
53+
doSearch("range", 2);
54+
doSearch("match", 6);
55+
doSearch("term", 4);
56+
57+
assertTopQueriesCount(3, "latency");
58+
59+
updateClusterSettings(this::defaultTopQueriesSettings);
60+
61+
// Top queries should be drained due to grouping change from SIMILARITY -> NONE
62+
assertTopQueriesCount(0, "latency");
63+
64+
// Search
65+
doSearch("range", 2);
66+
doSearch("match", 6);
67+
doSearch("term", 4);
68+
69+
assertTopQueriesCount(12, "latency");
70+
}
71+
72+
public void testSimilarityMaxGroupsChanged() throws IOException, InterruptedException {
73+
74+
waitForEmptyTopQueriesResponse();
75+
76+
updateClusterSettings(this::defaultTopQueryGroupingSettings);
77+
78+
// Search
79+
doSearch("range", 2);
80+
doSearch("match", 6);
81+
doSearch("term", 4);
82+
83+
assertTopQueriesCount(3, "latency");
84+
85+
// Change max groups exluding topn setting
86+
updateClusterSettings(this::updateMaxGroupsExcludingTopNSetting);
87+
88+
// Top queries should be drained due to max group change
89+
assertTopQueriesCount(0, "latency");
90+
91+
// Search
92+
doSearch("range", 2);
93+
doSearch("match", 6);
94+
doSearch("term", 4);
95+
96+
assertTopQueriesCount(3, "latency");
97+
}
98+
99+
protected String updateMaxGroupsExcludingTopNSetting() {
100+
return "{\n"
101+
+ " \"persistent\" : {\n"
102+
+ " \"search.insights.top_queries.max_groups_excluding_topn\" : 1\n"
103+
+ " }\n"
104+
+ "}";
105+
}
106+
}

‎src/test/java/org/opensearch/plugin/insights/rules/resthandler/top_queries/TopQueriesRestIT.java

+8-46
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
package org.opensearch.plugin.insights.rules.resthandler.top_queries;
1010

1111
import java.io.IOException;
12-
import java.nio.charset.StandardCharsets;
1312
import java.util.List;
1413
import java.util.Map;
1514
import org.junit.Assert;
@@ -20,7 +19,6 @@
2019
import org.opensearch.common.xcontent.json.JsonXContent;
2120
import org.opensearch.core.xcontent.NamedXContentRegistry;
2221
import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
23-
import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
2422

2523
/** Rest Action tests for Top Queries */
2624
public class TopQueriesRestIT extends QueryInsightsRestTestCase {
@@ -52,55 +50,19 @@ public void testTopQueriesResponses() throws IOException, InterruptedException {
5250
waitForEmptyTopQueriesResponse();
5351

5452
// Enable Top N Queries feature
55-
Request request = new Request("PUT", "/_cluster/settings");
56-
request.setJsonEntity(defaultTopQueriesSettings());
57-
Response response = client().performRequest(request);
58-
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
59-
doSearch(2);
60-
// run five times to make sure the records are drained to the top queries services
61-
for (int i = 0; i < 5; i++) {
62-
// Get Top Queries
63-
request = new Request("GET", "/_insights/top_queries?pretty");
64-
response = client().performRequest(request);
65-
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
66-
String topRequests = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
67-
Assert.assertTrue(topRequests.contains("top_queries"));
53+
updateClusterSettings(this::defaultTopQueriesSettings);
6854

69-
int topNArraySize = countTopQueries(topRequests);
55+
doSearch(2);
7056

71-
if (topNArraySize == 0) {
72-
Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
73-
continue;
74-
}
75-
Assert.assertEquals(2, topNArraySize);
76-
}
57+
assertTopQueriesCount(2, "latency");
7758

7859
// Enable Top N Queries by resource usage
79-
request = new Request("PUT", "/_cluster/settings");
80-
request.setJsonEntity(topQueriesByResourceUsagesSettings());
81-
response = client().performRequest(request);
82-
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
60+
updateClusterSettings(this::topQueriesByResourceUsagesSettings);
61+
8362
// Do Search
8463
doSearch(2);
8564

86-
// Run five times to make sure the records are drained to the top queries services
87-
for (int i = 0; i < 5; i++) {
88-
// Get Top Queries
89-
request = new Request("GET", "/_insights/top_queries?type=cpu&pretty");
90-
response = client().performRequest(request);
91-
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
92-
String topRequests = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
93-
Assert.assertTrue(topRequests.contains("top_queries"));
94-
95-
// Use the countTopQueries method to determine the number of top queries
96-
int topNArraySize = countTopQueries(topRequests);
97-
98-
if (topNArraySize == 0) {
99-
Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
100-
continue;
101-
}
102-
Assert.assertEquals(2, topNArraySize);
103-
}
65+
assertTopQueriesCount(2, "cpu");
10466
}
10567

10668
/**
@@ -125,10 +87,10 @@ private String topQueriesByResourceUsagesSettings() {
12587
return "{\n"
12688
+ " \"persistent\" : {\n"
12789
+ " \"search.insights.top_queries.memory.enabled\" : \"true\",\n"
128-
+ " \"search.insights.top_queries.memory.window_size\" : \"600s\",\n"
90+
+ " \"search.insights.top_queries.memory.window_size\" : \"1m\",\n"
12991
+ " \"search.insights.top_queries.memory.top_n_size\" : \"5\",\n"
13092
+ " \"search.insights.top_queries.cpu.enabled\" : \"true\",\n"
131-
+ " \"search.insights.top_queries.cpu.window_size\" : \"600s\",\n"
93+
+ " \"search.insights.top_queries.cpu.window_size\" : \"1m\",\n"
13294
+ " \"search.insights.top_queries.cpu.top_n_size\" : 5,\n"
13395
+ " \"search.insights.top_queries.group_by\" : \"none\"\n"
13496
+ " }\n"

0 commit comments

Comments
 (0)