Skip to content

Commit 4d896cc

Browse files
LilyCaroline17LilyCaroline17
authored
Top n queries historical queries from local index and time range (#84)
* Added support for historical top n queries Signed-off-by: Emily Guo <emilyguo@amazon.com> Handle indexing errors and accurate search parsing Signed-off-by: Emily Guo <emilyguo@amazon.com> Correctly filter in window queries when from and to exist Signed-off-by: Emily Guo <emilyguo@amazon.com> Remove search requests from top n queries Signed-off-by: Emily Guo <emilyguo@amazon.com> Removed comments Signed-off-by: Emily Guo <emilyguo@amazon.com> Comments for new functions Signed-off-by: Emily Guo <emilyguo@amazon.com> Comments for new functions Signed-off-by: Emily Guo <emilyguo@amazon.com> Updated comments Signed-off-by: Emily Guo <emilyguo@amazon.com> * Added unit tests Signed-off-by: Emily Guo <emilyguo@amazon.com> * Update LocalIndexExporter.java Signed-off-by: Emily Guo <35637792+LilyCaroline17@users.noreply.github.com> * Update LocalIndexReaderTests.java Signed-off-by: Emily Guo <35637792+LilyCaroline17@users.noreply.github.com> * Update QueryInsightsReaderFactoryTests.java Signed-off-by: Emily Guo <35637792+LilyCaroline17@users.noreply.github.com> * Address comments, change getTimeRange into getFrom and getTo, apply filters to historical, and rename functions Signed-off-by: Emily Guo <LilyCaroline1717@gmail.com> * Fix test cases Signed-off-by: Emily Guo <LilyCaroline1717@gmail.com> --------- Signed-off-by: Emily Guo <emilyguo@amazon.com> Signed-off-by: Emily Guo <35637792+LilyCaroline17@users.noreply.github.com> Signed-off-by: Emily Guo <LilyCaroline1717@gmail.com>
1 parent ca0d999 commit 4d896cc

19 files changed

+953
-43
lines changed

‎src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,8 @@ public Collection<Object> createComponents(
7979
clusterService.getClusterSettings(),
8080
threadPool,
8181
client,
82-
metricsRegistry
82+
metricsRegistry,
83+
xContentRegistry
8384
);
8485
return List.of(queryInsightsService, new QueryInsightsListener(clusterService, queryInsightsService, false));
8586
}

‎src/main/java/org/opensearch/plugin/insights/core/reader/LocalIndexReader.java

+130
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.plugin.insights.core.reader;
10+
11+
import java.util.ArrayList;
12+
import java.util.List;
13+
import org.apache.logging.log4j.LogManager;
14+
import org.apache.logging.log4j.Logger;
15+
import org.joda.time.DateTime;
16+
import org.joda.time.DateTimeZone;
17+
import org.joda.time.format.DateTimeFormatter;
18+
import org.opensearch.action.search.SearchRequest;
19+
import org.opensearch.action.search.SearchResponse;
20+
import org.opensearch.client.Client;
21+
import org.opensearch.core.xcontent.NamedXContentRegistry;
22+
import org.opensearch.index.IndexNotFoundException;
23+
import org.opensearch.index.query.MatchQueryBuilder;
24+
import org.opensearch.index.query.QueryBuilder;
25+
import org.opensearch.index.query.QueryBuilders;
26+
import org.opensearch.index.query.RangeQueryBuilder;
27+
import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
28+
import org.opensearch.search.SearchHit;
29+
import org.opensearch.search.builder.SearchSourceBuilder;
30+
31+
/**
32+
* Local index reader for reading query insights data from local OpenSearch indices.
33+
*/
34+
public final class LocalIndexReader implements QueryInsightsReader {
35+
/**
36+
* Logger of the local index reader
37+
*/
38+
private final Logger logger = LogManager.getLogger();
39+
private final Client client;
40+
private DateTimeFormatter indexPattern;
41+
private final NamedXContentRegistry namedXContentRegistry;
42+
43+
/**
44+
* Constructor of LocalIndexReader
45+
*
46+
* @param client OS client
47+
* @param indexPattern the pattern of index to read from
48+
* @param namedXContentRegistry for parsing purposes
49+
*/
50+
public LocalIndexReader(final Client client, final DateTimeFormatter indexPattern, final NamedXContentRegistry namedXContentRegistry) {
51+
this.indexPattern = indexPattern;
52+
this.client = client;
53+
this.namedXContentRegistry = namedXContentRegistry;
54+
}
55+
56+
/**
57+
* Getter of indexPattern
58+
*
59+
* @return indexPattern
60+
*/
61+
public DateTimeFormatter getIndexPattern() {
62+
return indexPattern;
63+
}
64+
65+
/**
66+
* Setter of indexPattern
67+
*
68+
* @param indexPattern index pattern
69+
* @return the current LocalIndexReader
70+
*/
71+
public LocalIndexReader setIndexPattern(DateTimeFormatter indexPattern) {
72+
this.indexPattern = indexPattern;
73+
return this;
74+
}
75+
76+
/**
77+
* Export a list of SearchQueryRecord from local index
78+
*
79+
* @param from start timestamp
80+
* @param to end timestamp
81+
* @return list of SearchQueryRecords whose timestamps fall between from and to
82+
*/
83+
@Override
84+
public List<SearchQueryRecord> read(final String from, final String to) {
85+
List<SearchQueryRecord> records = new ArrayList<>();
86+
if (from == null || to == null) {
87+
return records;
88+
}
89+
final DateTime start = DateTime.parse(from);
90+
DateTime end = DateTime.parse(to);
91+
if (end.compareTo(DateTime.now(DateTimeZone.UTC)) > 0) {
92+
end = DateTime.now(DateTimeZone.UTC);
93+
}
94+
DateTime curr = start;
95+
while (curr.compareTo(end.plusDays(1).withTimeAtStartOfDay()) < 0) {
96+
String index = getDateTimeFromFormat(curr);
97+
SearchRequest searchRequest = new SearchRequest(index);
98+
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
99+
MatchQueryBuilder excludeQuery = QueryBuilders.matchQuery("indices", "top_queries*");
100+
RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("timestamp").from(start.getMillis()).to(end.getMillis());
101+
QueryBuilder query = QueryBuilders.boolQuery().must(rangeQuery).mustNot(excludeQuery);
102+
searchSourceBuilder.query(query);
103+
searchRequest.source(searchSourceBuilder);
104+
try {
105+
SearchResponse searchResponse = client.search(searchRequest).actionGet();
106+
for (SearchHit hit : searchResponse.getHits()) {
107+
SearchQueryRecord record = SearchQueryRecord.getRecord(hit, namedXContentRegistry);
108+
records.add(record);
109+
}
110+
} catch (IndexNotFoundException ignored) {} catch (Exception e) {
111+
logger.error("Unable to parse search hit: ", e);
112+
}
113+
curr = curr.plusDays(1);
114+
115+
}
116+
return records;
117+
}
118+
119+
/**
120+
* Close the reader sink
121+
*/
122+
@Override
123+
public void close() {
124+
logger.debug("Closing the LocalIndexReader..");
125+
}
126+
127+
private String getDateTimeFromFormat(DateTime current) {
128+
return indexPattern.print(current);
129+
}
130+
}

‎src/main/java/org/opensearch/plugin/insights/core/reader/QueryInsightsReader.java

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.plugin.insights.core.reader;
10+
11+
import java.io.Closeable;
12+
import java.util.List;
13+
import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
14+
15+
/**
16+
* Base interface for Query Insights readers
17+
*/
18+
public interface QueryInsightsReader extends Closeable {
19+
/**
20+
* Reader a list of SearchQueryRecord
21+
*
22+
* @param from string
23+
* @param to string
24+
* @return List of SearchQueryRecord
25+
*/
26+
List<SearchQueryRecord> read(final String from, final String to);
27+
}

‎src/main/java/org/opensearch/plugin/insights/core/reader/QueryInsightsReaderFactory.java

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.plugin.insights.core.reader;
10+
11+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_N_QUERIES_INDEX_PATTERN;
12+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORT_INDEX;
13+
14+
import java.io.IOException;
15+
import java.util.HashSet;
16+
import java.util.Locale;
17+
import java.util.Set;
18+
import org.apache.logging.log4j.LogManager;
19+
import org.apache.logging.log4j.Logger;
20+
import org.joda.time.format.DateTimeFormat;
21+
import org.opensearch.client.Client;
22+
import org.opensearch.common.settings.Settings;
23+
import org.opensearch.core.xcontent.NamedXContentRegistry;
24+
25+
/**
26+
* Factory class for validating and creating Readers based on provided settings
27+
*/
28+
public class QueryInsightsReaderFactory {
29+
/**
30+
* Logger of the query insights Reader factory
31+
*/
32+
private final Logger logger = LogManager.getLogger();
33+
final private Client client;
34+
final private Set<QueryInsightsReader> Readers;
35+
36+
/**
37+
* Constructor of QueryInsightsReaderFactory
38+
*
39+
* @param client OS client
40+
*/
41+
public QueryInsightsReaderFactory(final Client client) {
42+
this.client = client;
43+
this.Readers = new HashSet<>();
44+
}
45+
46+
/**
47+
* Validate Reader sink config
48+
*
49+
* @param settings Reader sink config {@link Settings}
50+
* @throws IllegalArgumentException if provided Reader sink config settings are invalid
51+
*/
52+
public void validateReaderConfig(final Settings settings) throws IllegalArgumentException {
53+
final String indexPattern = settings.get(EXPORT_INDEX, DEFAULT_TOP_N_QUERIES_INDEX_PATTERN);
54+
if (indexPattern.isEmpty()) {
55+
throw new IllegalArgumentException("Empty index pattern configured for the Reader");
56+
}
57+
try {
58+
DateTimeFormat.forPattern(indexPattern);
59+
} catch (Exception e) {
60+
throw new IllegalArgumentException(
61+
String.format(Locale.ROOT, "Invalid index pattern [%s] configured for the Reader", indexPattern)
62+
);
63+
}
64+
}
65+
66+
/**
67+
* Create a Reader based on provided parameters
68+
*
69+
* @param indexPattern the index pattern if creating an index Reader
70+
* @param namedXContentRegistry for parsing purposes
71+
* @return QueryInsightsReader the created Reader
72+
*/
73+
public QueryInsightsReader createReader(String indexPattern, NamedXContentRegistry namedXContentRegistry) {
74+
QueryInsightsReader Reader = new LocalIndexReader(client, DateTimeFormat.forPattern(indexPattern), namedXContentRegistry);
75+
this.Readers.add(Reader);
76+
return Reader;
77+
}
78+
79+
/**
80+
* Update a Reader based on provided parameters
81+
*
82+
* @param Reader The Reader to update
83+
* @param indexPattern the index pattern if creating an index Reader
84+
* @return QueryInsightsReader the updated Reader sink
85+
*/
86+
public QueryInsightsReader updateReader(QueryInsightsReader Reader, String indexPattern) {
87+
if (Reader.getClass() == LocalIndexReader.class) {
88+
((LocalIndexReader) Reader).setIndexPattern(DateTimeFormat.forPattern(indexPattern));
89+
}
90+
return Reader;
91+
}
92+
93+
/**
94+
* Close a Reader
95+
*
96+
* @param Reader the Reader to close
97+
*/
98+
public void closeReader(QueryInsightsReader Reader) throws IOException {
99+
if (Reader != null) {
100+
Reader.close();
101+
this.Readers.remove(Reader);
102+
}
103+
}
104+
105+
/**
106+
* Close all Readers
107+
*
108+
*/
109+
public void closeAllReaders() {
110+
for (QueryInsightsReader Reader : Readers) {
111+
try {
112+
closeReader(Reader);
113+
} catch (IOException e) {
114+
logger.error("Fail to close query insights Reader, error: ", e);
115+
}
116+
}
117+
}
118+
}

‎src/main/java/org/opensearch/plugin/insights/core/reader/package-info.java

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
/**
10+
* Query Insights reader
11+
*/
12+
package org.opensearch.plugin.insights.core.reader;

0 commit comments

Comments
 (0)