Skip to content

Commit fac3bbd

Browse files
jackiehanyangjackiehanyang
committed
add UTs
Signed-off-by: Jackie Han <jkhanjob@gmail.com>
1 parent d5ba7f5 commit fac3bbd

File tree

2 files changed

+171
-9
lines changed

2 files changed

+171
-9
lines changed

‎src/main/java/org/opensearch/timeseries/indices/IndexManagement.java

+3-7
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,6 @@ protected void deleteOldHistoryIndices(String indexPattern, TimeValue historyRet
308308
long indexAgeMillis = Instant.now().toEpochMilli() - creationTime;
309309
if (indexAgeMillis > historyRetentionPeriod.millis()) {
310310
String indexName = indexMetaData.getIndex().getName();
311-
System.out.println("indexName: " + indexName);
312311
candidates.add(indexName);
313312
if (latest < creationTime) {
314313
latest = creationTime;
@@ -318,7 +317,7 @@ protected void deleteOldHistoryIndices(String indexPattern, TimeValue historyRet
318317
}
319318
if (candidates.size() > 1) {
320319
// delete all indices except the last one because the last one may contain docs newer than the retention period
321-
//candidates.remove(latestToDelete);
320+
candidates.remove(latestToDelete);
322321
String[] toDelete = candidates.toArray(Strings.EMPTY_ARRAY);
323322
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(toDelete);
324323
adminClient.indices().delete(deleteIndexRequest, ActionListener.wrap(deleteIndexResponse -> {
@@ -1285,9 +1284,7 @@ private void handleCustomResultIndex(Config config, IndexType resultIndex) {
12851284

12861285
// add rollover conditions if found in config
12871286
if (config.getCustomResultIndexMinAge() != null) {
1288-
rolloverRequest.addMaxIndexAgeCondition(TimeValue.timeValueMinutes(1));
1289-
1290-
// rolloverRequest.addMaxIndexAgeCondition(TimeValue.timeValueDays(config.getCustomResultIndexMinAge()));
1287+
rolloverRequest.addMaxIndexAgeCondition(TimeValue.timeValueDays(config.getCustomResultIndexMinAge()));
12911288
}
12921289
if (config.getCustomResultIndexMinSize() != null) {
12931290
rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(config.getCustomResultIndexMinSize(), ByteSizeUnit.MB));
@@ -1339,8 +1336,7 @@ private void proceedWithRolloverAndDelete(
13391336
if (resultIndexAlias.startsWith(ADCommonName.CUSTOM_RESULT_INDEX_PREFIX) || resultIndexAlias.startsWith(CUSTOM_RESULT_INDEX_PREFIX)) {
13401337
// handle custom result index deletion
13411338
if (customResultIndexTtl != null) {
1342-
// deleteOldHistoryIndices(allResultIndicesPattern, TimeValue.timeValueHours(customResultIndexTtl * 24));
1343-
deleteOldHistoryIndices(allResultIndicesPattern, TimeValue.timeValueMinutes(1));
1339+
deleteOldHistoryIndices(allResultIndicesPattern, TimeValue.timeValueHours(customResultIndexTtl * 24));
13441340

13451341
}
13461342
} else {

‎src/test/java/org/opensearch/ad/indices/RolloverTests.java

+168-2
Original file line numberDiff line numberDiff line change
@@ -12,60 +12,100 @@
1212
package org.opensearch.ad.indices;
1313

1414
import static org.mockito.ArgumentMatchers.any;
15+
import static org.mockito.ArgumentMatchers.anyString;
1516
import static org.mockito.Mockito.doAnswer;
17+
import static org.mockito.Mockito.doReturn;
1618
import static org.mockito.Mockito.mock;
1719
import static org.mockito.Mockito.never;
1820
import static org.mockito.Mockito.times;
1921
import static org.mockito.Mockito.verify;
2022
import static org.mockito.Mockito.when;
23+
import static org.opensearch.timeseries.TestHelpers.createSearchResponse;
24+
import static org.opensearch.timeseries.constant.CommonName.CONFIG_INDEX;
2125

26+
import java.io.IOException;
27+
import java.io.InputStream;
2228
import java.time.Instant;
29+
import java.util.ArrayList;
2330
import java.util.Arrays;
2431
import java.util.Collections;
2532
import java.util.HashSet;
33+
import java.util.List;
34+
import java.util.Locale;
2635
import java.util.Map;
2736

37+
import com.google.common.collect.ImmutableList;
38+
import org.apache.lucene.search.TotalHits;
39+
import org.opensearch.Version;
2840
import org.opensearch.action.admin.cluster.state.ClusterStateRequest;
2941
import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
3042
import org.opensearch.action.admin.indices.create.CreateIndexRequest;
3143
import org.opensearch.action.admin.indices.rollover.Condition;
44+
import org.opensearch.action.admin.indices.rollover.MaxAgeCondition;
3245
import org.opensearch.action.admin.indices.rollover.MaxDocsCondition;
46+
import org.opensearch.action.admin.indices.rollover.MaxSizeCondition;
3347
import org.opensearch.action.admin.indices.rollover.RolloverRequest;
3448
import org.opensearch.action.admin.indices.rollover.RolloverResponse;
49+
import org.opensearch.action.search.SearchResponse;
50+
import org.opensearch.action.search.ShardSearchFailure;
3551
import org.opensearch.action.support.master.AcknowledgedResponse;
3652
import org.opensearch.ad.constant.ADCommonName;
53+
import org.opensearch.ad.mock.model.MockSimpleLog;
54+
import org.opensearch.ad.model.AnomalyDetector;
3755
import org.opensearch.ad.settings.AnomalyDetectorSettings;
3856
import org.opensearch.client.AdminClient;
3957
import org.opensearch.client.Client;
4058
import org.opensearch.client.ClusterAdminClient;
4159
import org.opensearch.client.IndicesAdminClient;
4260
import org.opensearch.cluster.ClusterName;
4361
import org.opensearch.cluster.ClusterState;
62+
import org.opensearch.cluster.metadata.AliasMetadata;
63+
import org.opensearch.cluster.metadata.IndexMetadata;
4464
import org.opensearch.cluster.metadata.Metadata;
4565
import org.opensearch.cluster.service.ClusterService;
66+
import org.opensearch.common.recycler.Recycler;
4667
import org.opensearch.common.settings.ClusterSettings;
4768
import org.opensearch.common.settings.Settings;
69+
import org.opensearch.common.unit.TimeValue;
4870
import org.opensearch.core.action.ActionListener;
71+
import org.opensearch.core.common.bytes.BytesArray;
72+
import org.opensearch.core.common.bytes.BytesReference;
73+
import org.opensearch.core.common.unit.ByteSizeUnit;
74+
import org.opensearch.core.common.unit.ByteSizeValue;
75+
import org.opensearch.core.xcontent.DeprecationHandler;
76+
import org.opensearch.core.xcontent.MediaType;
4977
import org.opensearch.core.xcontent.NamedXContentRegistry;
78+
import org.opensearch.core.xcontent.XContentParser;
79+
import org.opensearch.search.SearchHit;
80+
import org.opensearch.search.SearchHits;
81+
import org.opensearch.search.aggregations.AggregationBuilder;
82+
import org.opensearch.search.aggregations.InternalAggregations;
83+
import org.opensearch.search.internal.InternalSearchResponse;
5084
import org.opensearch.threadpool.ThreadPool;
5185
import org.opensearch.timeseries.AbstractTimeSeriesTest;
86+
import org.opensearch.timeseries.TestHelpers;
87+
import org.opensearch.timeseries.function.BiCheckedFunction;
88+
import org.opensearch.timeseries.model.Config;
89+
import org.opensearch.timeseries.model.Feature;
5290
import org.opensearch.timeseries.settings.TimeSeriesSettings;
5391
import org.opensearch.timeseries.util.DiscoveryNodeFilterer;
5492

5593
public class RolloverTests extends AbstractTimeSeriesTest {
5694
private ADIndexManagement adIndices;
5795
private IndicesAdminClient indicesClient;
5896
private ClusterAdminClient clusterAdminClient;
97+
private Client client;
5998
private ClusterName clusterName;
6099
private ClusterState clusterState;
61100
private ClusterService clusterService;
101+
private NamedXContentRegistry namedXContentRegistry;
62102
private long defaultMaxDocs;
63103
private int numberOfNodes;
64104

65105
@Override
66106
public void setUp() throws Exception {
67107
super.setUp();
68-
Client client = mock(Client.class);
108+
client = mock(Client.class);
69109
indicesClient = mock(IndicesAdminClient.class);
70110
AdminClient adminClient = mock(AdminClient.class);
71111
clusterService = mock(ClusterService.class);
@@ -98,14 +138,16 @@ public void setUp() throws Exception {
98138
numberOfNodes = 2;
99139
when(nodeFilter.getNumberOfEligibleDataNodes()).thenReturn(numberOfNodes);
100140

141+
namedXContentRegistry = TestHelpers.xContentRegistry();
142+
101143
adIndices = new ADIndexManagement(
102144
client,
103145
clusterService,
104146
threadPool,
105147
settings,
106148
nodeFilter,
107149
TimeSeriesSettings.MAX_UPDATE_RETRY_TIMES,
108-
NamedXContentRegistry.EMPTY
150+
namedXContentRegistry
109151
);
110152

111153
clusterAdminClient = mock(ClusterAdminClient.class);
@@ -248,4 +290,128 @@ public void testRetryingDelete() {
248290
// 1 group delete, 1 separate retry for each index to delete
249291
verify(indicesClient, times(2)).delete(any(), any());
250292
}
293+
294+
public void testNoCustomResultIndexFound_RolloverDefaultResultIndex_shouldSucceed() {
295+
setUpGetConfigs_withNoCustomResultIndexAlias();
296+
setUpRolloverSuccess();
297+
298+
adIndices.rolloverAndDeleteHistoryIndex();
299+
verify(indicesClient, times(1)).rolloverIndex(any(), any());
300+
verify(client, times(1)).search(any(), any());
301+
}
302+
303+
public void testCustomResultIndexFound_RolloverCustomResultIndex_withConditions_shouldSucceed() throws IOException {
304+
setUpGetConfigs_withCustomResultIndexAlias();
305+
setUpRolloverSuccessForCustomIndex();
306+
307+
adIndices.rolloverAndDeleteHistoryIndex();
308+
309+
verify(indicesClient, times(1)).rolloverIndex(any(), any());
310+
verify(client, times(1)).search(any(), any());
311+
}
312+
313+
private void setUpGetConfigs_withNoCustomResultIndexAlias() {
314+
Metadata.Builder metaBuilder = Metadata
315+
.builder()
316+
.put(indexMeta(".opendistro-anomaly-detectors", 1L, ADCommonName.ANOMALY_RESULT_INDEX_ALIAS), true);
317+
clusterState = ClusterState.builder(clusterName).metadata(metaBuilder.build()).build();
318+
when(clusterService.state()).thenReturn(clusterState);
319+
320+
String detectorString = "{\"name\":\"AhtYYGWTgqkzairTchcs\",\"description\":\"iIiAVPMyFgnFlEniLbMyfJxyoGvJAl\","
321+
+ "\"time_field\":\"HmdFH\",\"indices\":[\"ffsBF\"],\"filter_query\":{\"bool\":{\"filter\":[{\"exists\":"
322+
+ "{\"field\":\"value\",\"boost\":1}}],\"adjust_pure_negative\":true,\"boost\":1}},\"window_delay\":"
323+
+ "{\"period\":{\"interval\":2,\"unit\":\"Minutes\"}},\"shingle_size\":8,\"schema_version\":-512063255,"
324+
+ "\"feature_attributes\":[{\"feature_id\":\"OTYJs\",\"feature_name\":\"eYYCM\",\"feature_enabled\":false,"
325+
+ "\"aggregation_query\":{\"XzewX\":{\"value_count\":{\"field\":\"ok\"}}}}],\"recency_emphasis\":3342,"
326+
+ "\"history\":62,\"last_update_time\":1717192049845,\"category_field\":[\"Tcqcb\"],\"customResultIndexOrAlias\":"
327+
+ "\"\",\"imputation_option\":{\"method\":\"FIXED_VALUES\",\"defaultFill\""
328+
+ ":[],\"integerSensitive\":false},\"suggested_seasonality\":64,\"detection_interval\":{\"period\":"
329+
+ "{\"interval\":5,\"unit\":\"Minutes\"}},\"detector_type\":\"MULTI_ENTITY\",\"rules\":[]}";
330+
331+
doAnswer(invocation -> {
332+
ActionListener<SearchResponse> listener = invocation.getArgument(1);
333+
SearchHit config = SearchHit.fromXContent(TestHelpers.parser(detectorString));
334+
SearchHits searchHits = new SearchHits(new SearchHit[] { config }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), Float.NaN);
335+
InternalSearchResponse response = new InternalSearchResponse(
336+
searchHits,
337+
InternalAggregations.EMPTY,
338+
null,
339+
null,
340+
false,
341+
null,
342+
1
343+
);
344+
SearchResponse searchResponse = new SearchResponse(
345+
response,
346+
null,
347+
1,
348+
1,
349+
0,
350+
100,
351+
ShardSearchFailure.EMPTY_ARRAY,
352+
SearchResponse.Clusters.EMPTY
353+
);
354+
listener.onResponse(searchResponse);
355+
return null;
356+
}).when(client).search(any(), any());
357+
}
358+
359+
private void setUpRolloverSuccessForCustomIndex() {
360+
doAnswer(invocation -> {
361+
RolloverRequest request = invocation.getArgument(0);
362+
@SuppressWarnings("unchecked")
363+
ActionListener<RolloverResponse> listener = (ActionListener<RolloverResponse>) invocation.getArgument(1);
364+
365+
assertEquals("opensearch-ad-plugin-result-", request.indices()[0]);
366+
Map<String, Condition<?>> conditions = request.getConditions();
367+
assertEquals(2, conditions.size());
368+
assertEquals(new MaxAgeCondition(TimeValue.timeValueDays(7)), conditions.get(MaxAgeCondition.NAME));
369+
assertEquals(new MaxSizeCondition(new ByteSizeValue(51200, ByteSizeUnit.MB)), conditions.get(MaxSizeCondition.NAME));
370+
371+
CreateIndexRequest createIndexRequest = request.getCreateIndexRequest();
372+
assertEquals("<opensearch-ad-plugin-result--history-{now/d}-1>", createIndexRequest.index());
373+
assertTrue(createIndexRequest.mappings().contains("data_start_time"));
374+
listener.onResponse(new RolloverResponse(null, null, Collections.emptyMap(), request.isDryRun(), true, true, true));
375+
return null;
376+
}).when(indicesClient).rolloverIndex(any(), any());
377+
}
378+
379+
private void setUpGetConfigs_withCustomResultIndexAlias() throws IOException {
380+
IndexMetadata defaultResultIndex = IndexMetadata.builder(".opendistro-anomaly-detectors")
381+
.settings(settings(Version.CURRENT))
382+
.putAlias(AliasMetadata.builder(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS).writeIndex(true).build())
383+
.numberOfShards(1)
384+
.numberOfReplicas(0)
385+
.build();
386+
IndexMetadata customResultIndex = IndexMetadata.builder("opensearch-ad-plugin-result-test")
387+
.settings(settings(Version.CURRENT))
388+
.numberOfShards(1)
389+
.numberOfReplicas(0)
390+
.putAlias(AliasMetadata.builder(ADCommonName.CUSTOM_RESULT_INDEX_PREFIX).writeIndex(true).build())
391+
.build();
392+
393+
clusterState = ClusterState.builder(ClusterState.EMPTY_STATE)
394+
.metadata(Metadata.builder().put(defaultResultIndex, false).put(customResultIndex, false).build())
395+
.build();
396+
397+
when(clusterService.state()).thenReturn(clusterState);
398+
399+
String detectorStringWithCustomResultIndex = "{\"name\":\"todagtCMkwpcaedpyYUM\",\"description\":\"ClrcaMpuLfeDSlVduRcKlqPZyqWDBf\"," +
400+
"\"time_field\":\"dJRwh\",\"indices\":[\"eIrgWMqAED\"],\"feature_attributes\":[{\"feature_id\":\"lxYRN\"," +
401+
"\"feature_name\":\"eqSeU\",\"feature_enabled\":true,\"aggregation_query\":{\"aa\":{\"value_count\":{\"field\":\"ok\"}}}}]," +
402+
"\"detection_interval\":{\"period\":{\"interval\":425,\"unit\":\"Minutes\"}}," +
403+
"\"window_delay\":{\"period\":{\"interval\":973,\"unit\":\"Minutes\"}},\"shingle_size\":4,\"schema_version\":-1203962153," +
404+
"\"ui_metadata\":{\"JbAaV\":{\"feature_id\":\"rIFjS\",\"feature_name\":\"QXCmS\",\"feature_enabled\":false," +
405+
"\"aggregation_query\":{\"aa\":{\"value_count\":{\"field\":\"ok\"}}}}},\"last_update_time\":1568396089028," +
406+
"\"result_index\":\"opensearch-ad-plugin-result-\",\"result_index_min_size\":51200,\"result_index_min_age\":7}";
407+
408+
AnomalyDetector parsedDetector = AnomalyDetector.parse(TestHelpers.parser(detectorStringWithCustomResultIndex), "id", 1L, null, null);
409+
410+
doAnswer(invocation -> {
411+
Object[] args = invocation.getArguments();
412+
ActionListener<SearchResponse> listener = (ActionListener<SearchResponse>) args[1];
413+
listener.onResponse(createSearchResponse(parsedDetector));
414+
return null;
415+
}).when(client).search(any(), any());
416+
}
251417
}

0 commit comments

Comments
 (0)