Skip to content

Commit 2c02a2d

Browse files
ansjcyansjcy
authored
Add more integ tests for query insights (#71)
Signed-off-by: Chenyang Ji <cyji@amazon.com>
1 parent b55d760 commit 2c02a2d

File tree

5 files changed

+346
-187
lines changed

5 files changed

+346
-187
lines changed

‎build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,7 @@ integTest {
263263
if (System.getProperty("security.enabled") == "true" || System.getProperty("https") == "true") {
264264
// Exclude this IT, because they executed in another task (:integTestWithSecurity)
265265
exclude 'org/opensearch/plugin/insights/rules/resthandler/top_queries/TopQueriesRestIT.class'
266+
exclude 'org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterIT.class'
266267
}
267268
}
268269

‎gradle/wrapper/gradle-wrapper.properties

+1-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
distributionBase=GRADLE_USER_HOME
1010
distributionPath=wrapper/dists
11-
distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
11+
distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
1212
zipStoreBase=GRADLE_USER_HOME
1313
zipStorePath=wrapper/dists
14-
distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961

‎src/test/java/org/opensearch/plugin/insights/QueryInsightsRestTestCase.java

+216
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.plugin.insights;
10+
11+
import java.io.IOException;
12+
import java.util.Collections;
13+
import java.util.List;
14+
import java.util.Map;
15+
import java.util.Optional;
16+
import java.util.stream.Collectors;
17+
import org.apache.hc.client5.http.auth.AuthScope;
18+
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
19+
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
20+
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager;
21+
import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
22+
import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
23+
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
24+
import org.apache.hc.core5.http.Header;
25+
import org.apache.hc.core5.http.HttpHost;
26+
import org.apache.hc.core5.http.message.BasicHeader;
27+
import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
28+
import org.apache.hc.core5.reactor.ssl.TlsDetails;
29+
import org.apache.hc.core5.ssl.SSLContextBuilder;
30+
import org.apache.hc.core5.util.Timeout;
31+
import org.junit.After;
32+
import org.junit.Assert;
33+
import org.junit.Before;
34+
import org.opensearch.client.Request;
35+
import org.opensearch.client.Response;
36+
import org.opensearch.client.RestClient;
37+
import org.opensearch.client.RestClientBuilder;
38+
import org.opensearch.common.settings.Settings;
39+
import org.opensearch.common.unit.TimeValue;
40+
import org.opensearch.common.util.concurrent.ThreadContext;
41+
import org.opensearch.core.xcontent.DeprecationHandler;
42+
import org.opensearch.core.xcontent.MediaType;
43+
import org.opensearch.core.xcontent.NamedXContentRegistry;
44+
import org.opensearch.core.xcontent.XContentParser;
45+
import org.opensearch.test.rest.OpenSearchRestTestCase;
46+
47+
public abstract class QueryInsightsRestTestCase extends OpenSearchRestTestCase {
48+
protected static final String QUERY_INSIGHTS_INDICES_PREFIX = "top_queries";
49+
50+
protected boolean isHttps() {
51+
return Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false);
52+
}
53+
54+
@Override
55+
protected String getProtocol() {
56+
return isHttps() ? "https" : "http";
57+
}
58+
59+
@Override
60+
protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
61+
RestClientBuilder builder = RestClient.builder(hosts);
62+
if (isHttps()) {
63+
configureHttpsClient(builder, settings);
64+
} else {
65+
configureClient(builder, settings);
66+
}
67+
68+
builder.setStrictDeprecationMode(false);
69+
return builder.build();
70+
}
71+
72+
protected static void configureClient(RestClientBuilder builder, Settings settings) throws IOException {
73+
String userName = System.getProperty("user");
74+
String password = System.getProperty("password");
75+
if (userName != null && password != null) {
76+
builder.setHttpClientConfigCallback(httpClientBuilder -> {
77+
BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
78+
credentialsProvider.setCredentials(
79+
new AuthScope(null, -1),
80+
new UsernamePasswordCredentials(userName, password.toCharArray())
81+
);
82+
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
83+
});
84+
}
85+
OpenSearchRestTestCase.configureClient(builder, settings);
86+
}
87+
88+
protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException {
89+
// Similar to client configuration with OpenSearch:
90+
// https://github.com/opensearch-project/OpenSearch/blob/2.15.1/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java#L841-L863
91+
builder.setHttpClientConfigCallback(httpClientBuilder -> {
92+
String userName = Optional.ofNullable(System.getProperty("user"))
93+
.orElseThrow(() -> new RuntimeException("user name is missing"));
94+
String password = Optional.ofNullable(System.getProperty("password"))
95+
.orElseThrow(() -> new RuntimeException("password is missing"));
96+
BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
97+
final AuthScope anyScope = new AuthScope(null, -1);
98+
credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(userName, password.toCharArray()));
99+
try {
100+
final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create()
101+
.setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
102+
.setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build())
103+
// See https://issues.apache.org/jira/browse/HTTPCLIENT-2219
104+
.setTlsDetailsFactory(sslEngine -> new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()))
105+
.build();
106+
final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create()
107+
.setTlsStrategy(tlsStrategy)
108+
.build();
109+
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager);
110+
} catch (Exception e) {
111+
throw new RuntimeException(e);
112+
}
113+
});
114+
Map<String, String> headers = ThreadContext.buildDefaultHeaders(settings);
115+
Header[] defaultHeaders = new Header[headers.size()];
116+
int i = 0;
117+
for (Map.Entry<String, String> entry : headers.entrySet()) {
118+
defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue());
119+
}
120+
builder.setDefaultHeaders(defaultHeaders);
121+
final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT);
122+
final TimeValue socketTimeout = TimeValue.parseTimeValue(
123+
socketTimeoutString == null ? "60s" : socketTimeoutString,
124+
CLIENT_SOCKET_TIMEOUT
125+
);
126+
builder.setRequestConfigCallback(
127+
conf -> conf.setResponseTimeout(Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis())))
128+
);
129+
if (settings.hasValue(CLIENT_PATH_PREFIX)) {
130+
builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX));
131+
}
132+
}
133+
134+
/**
135+
* wipeAllIndices won't work since it cannot delete security index. Use
136+
* wipeAllQueryInsightsIndices instead.
137+
*/
138+
@Override
139+
protected boolean preserveIndicesUponCompletion() {
140+
return true;
141+
}
142+
143+
@Before
144+
public void runBeforeEachTest() throws IOException {
145+
// Create documents for search
146+
Request request = new Request("POST", "/my-index-0/_doc");
147+
request.setJsonEntity(createDocumentsBody());
148+
Response response = client().performRequest(request);
149+
150+
Assert.assertEquals(201, response.getStatusLine().getStatusCode());
151+
}
152+
153+
@SuppressWarnings("unchecked")
154+
@After
155+
public void wipeAllQueryInsightsIndices() throws Exception {
156+
Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all"));
157+
MediaType mediaType = MediaType.fromMediaType(response.getEntity().getContentType());
158+
try (
159+
XContentParser parser = mediaType.xContent()
160+
.createParser(
161+
NamedXContentRegistry.EMPTY,
162+
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
163+
response.getEntity().getContent()
164+
)
165+
) {
166+
XContentParser.Token token = parser.nextToken();
167+
List<Map<String, Object>> parserList = null;
168+
if (token == XContentParser.Token.START_ARRAY) {
169+
parserList = parser.listOrderedMap().stream().map(obj -> (Map<String, Object>) obj).collect(Collectors.toList());
170+
} else {
171+
parserList = Collections.singletonList(parser.mapOrdered());
172+
}
173+
174+
for (Map<String, Object> index : parserList) {
175+
final String indexName = (String) index.get("index");
176+
if (indexName.startsWith(QUERY_INSIGHTS_INDICES_PREFIX)) {
177+
adminClient().performRequest(new Request("DELETE", "/" + indexName));
178+
}
179+
}
180+
}
181+
}
182+
183+
protected String defaultTopQueriesSettings() {
184+
return "{\n"
185+
+ " \"persistent\" : {\n"
186+
+ " \"search.insights.top_queries.latency.enabled\" : \"true\",\n"
187+
+ " \"search.insights.top_queries.latency.window_size\" : \"600s\",\n"
188+
+ " \"search.insights.top_queries.latency.top_n_size\" : 5\n"
189+
+ " }\n"
190+
+ "}";
191+
}
192+
193+
protected String createDocumentsBody() {
194+
return "{\n"
195+
+ " \"@timestamp\": \"2099-11-15T13:12:00\",\n"
196+
+ " \"message\": \"this is document 1\",\n"
197+
+ " \"user\": {\n"
198+
+ " \"id\": \"cyji\"\n"
199+
+ " }\n"
200+
+ "}";
201+
}
202+
203+
protected String searchBody() {
204+
return "{}";
205+
}
206+
207+
protected void doSearch(int times) throws IOException {
208+
for (int i = 0; i < times; i++) {
209+
// Do Search
210+
Request request = new Request("GET", "/my-index-0/_search?size=20&pretty");
211+
request.setJsonEntity(searchBody());
212+
Response response = client().performRequest(request);
213+
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
214+
}
215+
}
216+
}

‎src/test/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterIT.java

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.plugin.insights.core.exporter;
10+
11+
import java.io.IOException;
12+
import org.junit.Assert;
13+
import org.opensearch.client.Request;
14+
import org.opensearch.client.Response;
15+
import org.opensearch.client.ResponseException;
16+
import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
17+
18+
/** Rest Action tests for query */
19+
public class QueryInsightsExporterIT extends QueryInsightsRestTestCase {
20+
/**
21+
* Test Top Queries setting endpoints
22+
*
23+
* @throws IOException IOException
24+
*/
25+
public void testQueryInsightsExporterSettings() throws IOException {
26+
// test invalid settings
27+
for (String setting : invalidExporterSettings()) {
28+
Request request = new Request("PUT", "/_cluster/settings");
29+
request.setJsonEntity(setting);
30+
try {
31+
client().performRequest(request);
32+
fail("Should not succeed with invalid exporter settings");
33+
} catch (ResponseException e) {
34+
assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
35+
}
36+
}
37+
38+
// Test enable Top N Queries feature
39+
Request request = new Request("PUT", "/_cluster/settings");
40+
request.setJsonEntity(defaultExporterSettings());
41+
Response response = client().performRequest(request);
42+
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
43+
}
44+
45+
private String defaultExporterSettings() {
46+
return "{\n"
47+
+ " \"persistent\" : {\n"
48+
+ " \"search.insights.top_queries.latency.exporter.config.index\" : \"YYYY.MM.dd\",\n"
49+
+ " \"search.insights.top_queries.latency.exporter.type\" : \"local_index\"\n"
50+
+ " }\n"
51+
+ "}";
52+
}
53+
54+
private String[] invalidExporterSettings() {
55+
return new String[] {
56+
"{\n"
57+
+ " \"persistent\" : {\n"
58+
+ " \"search.insights.top_queries.latency.exporter.type\" : invalid_type\n"
59+
+ " }\n"
60+
+ "}",
61+
"{\n"
62+
+ " \"persistent\" : {\n"
63+
+ " \"search.insights.top_queries.latency.exporter.type\" : local_index,\n"
64+
+ " \"search.insights.top_queries.latency.exporter.config.index\" : \"1a2b\"\n"
65+
+ " }\n"
66+
+ "}" };
67+
}
68+
}

0 commit comments

Comments
 (0)