Skip to content

Commit f5285b8

Browse files
dzane17dzane17
committed
Top N indices auto deletion config & functionality
Signed-off-by: David Zane <davizane@amazon.com>
1 parent f570d9f commit f5285b8

12 files changed

+326
-33
lines changed

‎src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public Collection<Object> createComponents(
8282
OperationalMetricsCounter.initialize(clusterService.getClusterName().toString(), metricsRegistry);
8383
// create top n queries service
8484
final QueryInsightsService queryInsightsService = new QueryInsightsService(
85-
clusterService.getClusterSettings(),
85+
clusterService,
8686
threadPool,
8787
client,
8888
metricsRegistry,
@@ -145,6 +145,7 @@ public List<Setting<?>> getSettings() {
145145
QueryInsightsSettings.TOP_N_QUERIES_GROUPING_FIELD_NAME,
146146
QueryInsightsSettings.TOP_N_QUERIES_GROUPING_FIELD_TYPE,
147147
QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING,
148+
QueryInsightsSettings.TOP_N_EXPORTER_DELETE_AFTER,
148149
QueryCategorizationSettings.SEARCH_QUERY_FIELD_TYPE_CACHE_SIZE_KEY
149150
);
150151
}

‎src/main/java/org/opensearch/plugin/insights/core/exporter/LocalIndexExporter.java

+101-8
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,25 @@
88

99
package org.opensearch.plugin.insights.core.exporter;
1010

11+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_DELETE_AFTER_VALUE;
12+
13+
import java.time.Instant;
14+
import java.time.LocalDate;
1115
import java.time.ZoneOffset;
1216
import java.time.ZonedDateTime;
1317
import java.time.format.DateTimeFormatter;
18+
import java.time.format.DateTimeParseException;
1419
import java.util.List;
20+
import java.util.Map;
21+
import java.util.concurrent.TimeUnit;
1522
import org.apache.logging.log4j.LogManager;
1623
import org.apache.logging.log4j.Logger;
24+
import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
1725
import org.opensearch.action.bulk.BulkRequestBuilder;
1826
import org.opensearch.action.bulk.BulkResponse;
1927
import org.opensearch.action.index.IndexRequest;
2028
import org.opensearch.client.Client;
29+
import org.opensearch.cluster.metadata.IndexMetadata;
2130
import org.opensearch.common.unit.TimeValue;
2231
import org.opensearch.common.xcontent.XContentFactory;
2332
import org.opensearch.core.action.ActionListener;
@@ -36,6 +45,7 @@ public final class LocalIndexExporter implements QueryInsightsExporter {
3645
private final Logger logger = LogManager.getLogger();
3746
private final Client client;
3847
private DateTimeFormatter indexPattern;
48+
private int deleteAfter;
3949

4050
/**
4151
* Constructor of LocalIndexExporter
@@ -46,6 +56,7 @@ public final class LocalIndexExporter implements QueryInsightsExporter {
4656
public LocalIndexExporter(final Client client, final DateTimeFormatter indexPattern) {
4757
this.indexPattern = indexPattern;
4858
this.client = client;
59+
this.deleteAfter = DEFAULT_DELETE_AFTER_VALUE;
4960
}
5061

5162
/**
@@ -61,11 +72,9 @@ public DateTimeFormatter getIndexPattern() {
6172
* Setter of indexPattern
6273
*
6374
* @param indexPattern index pattern
64-
* @return the current LocalIndexExporter
6575
*/
66-
public LocalIndexExporter setIndexPattern(DateTimeFormatter indexPattern) {
76+
void setIndexPattern(DateTimeFormatter indexPattern) {
6777
this.indexPattern = indexPattern;
68-
return this;
6978
}
7079

7180
/**
@@ -75,15 +84,15 @@ public LocalIndexExporter setIndexPattern(DateTimeFormatter indexPattern) {
7584
*/
7685
@Override
7786
public void export(final List<SearchQueryRecord> records) {
78-
if (records == null || records.size() == 0) {
87+
if (records == null || records.isEmpty()) {
7988
return;
8089
}
8190
try {
82-
final String index = getDateTimeFromFormat();
91+
final String indexName = buildLocalIndexName();
8392
final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk().setTimeout(TimeValue.timeValueMinutes(1));
8493
for (SearchQueryRecord record : records) {
8594
bulkRequestBuilder.add(
86-
new IndexRequest(index).source(record.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS))
95+
new IndexRequest(indexName).source(record.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS))
8796
);
8897
}
8998
bulkRequestBuilder.execute(new ActionListener<BulkResponse>() {
@@ -110,7 +119,91 @@ public void close() {
110119
logger.debug("Closing the LocalIndexExporter..");
111120
}
112121

113-
private String getDateTimeFromFormat() {
114-
return indexPattern.format(ZonedDateTime.now(ZoneOffset.UTC));
122+
/**
123+
* Builds the local index name using the current UTC datetime
124+
*
125+
* @return A string representing the index name in the format "top_queries-YYYY.MM.dd-01234".
126+
*/
127+
String buildLocalIndexName() {
128+
return indexPattern.format(ZonedDateTime.now(ZoneOffset.UTC)) + "-" + generateLocalIndexDateHash();
129+
}
130+
131+
/**
132+
* Set local index exporter data retention period
133+
*
134+
* @param deleteAfter the number of days after which Top N local indices should be deleted
135+
*/
136+
public void setDeleteAfter(final int deleteAfter) {
137+
this.deleteAfter = deleteAfter;
138+
}
139+
140+
/**
141+
* Delete Top N local indices older than the configured data retention period
142+
*
143+
* @param indexMetadataMap Map of index name {@link String} to {@link IndexMetadata}
144+
*/
145+
public void deleteExpiredIndices(final Map<String, IndexMetadata> indexMetadataMap) {
146+
long expirationMillisLong = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(deleteAfter);
147+
for (Map.Entry<String, IndexMetadata> entry : indexMetadataMap.entrySet()) {
148+
String indexName = entry.getKey();
149+
if (!isTopQueriesIndex(indexName)) {
150+
continue;
151+
}
152+
if (entry.getValue().getCreationDate() <= expirationMillisLong) {
153+
// delete this index
154+
client.admin().indices().delete(new DeleteIndexRequest(indexName));
155+
}
156+
}
157+
}
158+
159+
/**
160+
* Validates if the input string is a Query Insights local index name
161+
* in the format "top_queries-YYYY.MM.dd-XXXXX".
162+
*
163+
* @param indexName the string to validate.
164+
* @return {@code true} if the string is valid, {@code false} otherwise.
165+
*/
166+
static boolean isTopQueriesIndex(String indexName) {
167+
// Split the input string by '-'
168+
String[] parts = indexName.split("-");
169+
170+
// Check if the string has exactly 3 parts
171+
if (parts.length != 3) {
172+
return false;
173+
}
174+
175+
// Validate the first part is "top_queries"
176+
if (!"top_queries".equals(parts[0])) {
177+
return false;
178+
}
179+
180+
// Validate the second part is a valid date in "YYYY.MM.dd" format
181+
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy.MM.dd");
182+
try {
183+
LocalDate.parse(parts[1], formatter);
184+
} catch (DateTimeParseException e) {
185+
return false;
186+
}
187+
188+
// Validate the third part is exactly 5 digits
189+
if (!parts[2].matches("\\d{5}")) {
190+
return false;
191+
}
192+
193+
return true;
194+
}
195+
196+
/**
197+
* Generates a consistent 5-digit numeric hash based on the current UTC date.
198+
* The generated hash is deterministic, meaning it will return the same result for the same date.
199+
*
200+
* @return A 5-digit numeric string representation of the current date's hash.
201+
*/
202+
public static String generateLocalIndexDateHash() {
203+
// Get the current date in UTC (yyyy-MM-dd format)
204+
String currentDate = DateTimeFormatter.ISO_LOCAL_DATE.format(Instant.now().atOffset(ZoneOffset.UTC).toLocalDate());
205+
206+
// Generate a 5-digit numeric hash from the date's hashCode
207+
return String.format("%05d", Math.abs(currentDate.hashCode()) % 100000);
115208
}
116209
}

‎src/main/java/org/opensearch/plugin/insights/core/metrics/OperationalMetric.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@
1313
public enum OperationalMetric {
1414
LOCAL_INDEX_READER_PARSING_EXCEPTIONS("Number of errors when parsing with LocalIndexReader"),
1515
LOCAL_INDEX_EXPORTER_BULK_FAILURES("Number of failures when ingesting Query Insights data to local indices"),
16+
LOCAL_INDEX_EXPORTER_DELETE_FAILURES("Number of failures when deleting local indices"),
1617
LOCAL_INDEX_EXPORTER_EXCEPTIONS("Number of exceptions in Query Insights LocalIndexExporter"),
1718
INVALID_EXPORTER_TYPE_FAILURES("Number of invalid exporter type failures"),
18-
INVALID_INDEX_PATTERN_EXCEPTIONS("Number of invalid index pattern exceptions"),
1919
DATA_INGEST_EXCEPTIONS("Number of exceptions during data ingest in Query Insights"),
2020
QUERY_CATEGORIZE_EXCEPTIONS("Number of exceptions when categorizing the queries"),
2121
EXPORTER_FAIL_TO_CLOSE_EXCEPTION("Number of failures when closing the exporter"),

‎src/main/java/org/opensearch/plugin/insights/core/reader/LocalIndexReader.java

+6-4
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88

99
package org.opensearch.plugin.insights.core.reader;
1010

11+
import static org.opensearch.plugin.insights.core.exporter.LocalIndexExporter.generateLocalIndexDateHash;
12+
1113
import java.time.ZoneOffset;
1214
import java.time.ZonedDateTime;
1315
import java.time.format.DateTimeFormatter;
@@ -99,8 +101,8 @@ public List<SearchQueryRecord> read(final String from, final String to) {
99101
}
100102
ZonedDateTime curr = start;
101103
while (curr.isBefore(end.plusDays(1).toLocalDate().atStartOfDay(end.getZone()))) {
102-
String index = getDateTimeFromFormat(curr);
103-
SearchRequest searchRequest = new SearchRequest(index);
104+
String indexName = buildLocalIndexName(curr);
105+
SearchRequest searchRequest = new SearchRequest(indexName);
104106
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
105107
MatchQueryBuilder excludeQuery = QueryBuilders.matchQuery("indices", "top_queries*");
106108
RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery("timestamp")
@@ -135,7 +137,7 @@ public void close() {
135137
logger.debug("Closing the LocalIndexReader..");
136138
}
137139

138-
private String getDateTimeFromFormat(ZonedDateTime current) {
139-
return current.format(indexPattern);
140+
private String buildLocalIndexName(ZonedDateTime current) {
141+
return current.format(indexPattern) + "-" + generateLocalIndexDateHash();
140142
}
141143
}

0 commit comments

Comments
 (0)