Skip to content

Commit 6d39343

Browse files
committed
build query parameters using data_end_time
This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing. The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead. This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis. Testing done: 1. reproduced the issue and verified the fix. 2. added unit tests. 3. Confirmed that single stream detector result views remain functional post-changes. Signed-off-by: Kaituo Li <kaituo@amazon.com>
1 parent 48acb93 commit 6d39343

6 files changed

+97
-8
lines changed

.github/workflows/remote-integ-tests-workflow.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,6 @@ jobs:
128128
uses: cypress-io/github-action@v2
129129
with:
130130
working-directory: opensearch-dashboards-functional-test
131-
command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
131+
command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/*
132132
env:
133133
CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}

package.json

+3-4
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,16 @@
44
"description": "OpenSearch Anomaly Detection Dashboards Plugin",
55
"main": "index.js",
66
"config": {
7-
"plugin_version": "3.0.0.0",
8-
"plugin_name": "anomalyDetectionDashboards",
9-
"plugin_zip_name": "anomaly-detection-dashboards"
7+
"id": "anomalyDetectionDashboards",
8+
"zip_name": "anomaly-detection-dashboards"
109
},
1110
"scripts": {
1211
"osd": "node ../../scripts/osd",
1312
"opensearch": "node ../../scripts/opensearch",
1413
"lint": "node ../../scripts/eslint .",
1514
"plugin-helpers": "node ../../scripts/plugin_helpers",
1615
"test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
17-
"build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
16+
"build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_zip_name-$npm_package_version.zip && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip"
1817
},
1918
"lint-staged": {
2019
"*.{ts,tsx,js,jsx,json,css,md}": [

public/pages/DetectorResults/containers/AnomalyResults.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
255255
endDate: adjustedCurrentTime.valueOf(),
256256
} as DateRange;
257257

258+
// build result search query params relative to data end time
258259
const params = buildParamsForGetAnomalyResultsWithDateRange(
259260
featureDataPointsRange.startDate,
260261
featureDataPointsRange.endDate

public/pages/utils/__tests__/anomalyResultUtils.test.ts

+57
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
getFeatureMissingDataAnnotations,
1414
getFeatureDataPointsForDetector,
1515
parsePureAnomalies,
16+
buildParamsForGetAnomalyResultsWithDateRange,
1617
} from '../anomalyResultUtils';
1718
import { getRandomDetector } from '../../../redux/reducers/__tests__/utils';
1819
import {
@@ -22,11 +23,16 @@ import {
2223
AnomalyData,
2324
} from '../../../models/interfaces';
2425
import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants';
26+
import { MAX_ANOMALIES } from '../../../utils/constants';
27+
import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants';
2528

2629
describe('anomalyResultUtils', () => {
2730
let randomDetector_20_min: Detector;
2831
let randomDetector_20_sec: Detector;
2932
let feature_id = 'deny_max';
33+
const startTime = 1609459200000; // January 1, 2021
34+
const endTime = 1609545600000; // January 2, 2021
35+
3036
beforeAll(() => {
3137
randomDetector_20_min = {
3238
...getRandomDetector(true),
@@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => {
569575
)
570576
).toEqual([]);
571577
});
578+
test('should correctly build parameters with default options', () => {
579+
const expected = {
580+
from: 0,
581+
size: MAX_ANOMALIES,
582+
sortDirection: SORT_DIRECTION.DESC,
583+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
584+
startTime: startTime,
585+
endTime: endTime,
586+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
587+
anomalyThreshold: -1,
588+
entityList: undefined, // Default as an empty array stringified
589+
};
590+
591+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime);
592+
expect(result).toEqual(expected);
593+
});
594+
595+
test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => {
596+
const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }];
597+
const expected = {
598+
from: 0,
599+
size: MAX_ANOMALIES,
600+
sortDirection: SORT_DIRECTION.DESC,
601+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
602+
startTime: startTime,
603+
endTime: endTime,
604+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
605+
anomalyThreshold: 0, // because anomalyOnly is true
606+
entityList: JSON.stringify(entities),
607+
};
608+
609+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities);
610+
expect(result).toEqual(expected);
611+
});
612+
613+
test('should handle undefined `entityList` as an empty array JSON string', () => {
614+
const expected = {
615+
from: 0,
616+
size: MAX_ANOMALIES,
617+
sortDirection: SORT_DIRECTION.DESC,
618+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
619+
startTime: startTime,
620+
endTime: endTime,
621+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
622+
anomalyThreshold: -1, // default as anomalyOnly is false
623+
entityList: undefined, // Default for undefined entityList
624+
};
625+
626+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined);
627+
expect(result).toEqual(expected);
628+
});
572629
});
573630

574631
describe('parsePureAnomalies()', () => {

public/pages/utils/anomalyResultUtils.ts

+34-2
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,38 @@ export const getLiveAnomalyResults = (
118118
);
119119
};
120120

121+
/**
122+
* Builds search query parameters for retrieving anomaly results within a specified date range.
123+
*
124+
* This function constructs a parameter object for querying an anomaly detection system, filtering results
125+
* by a given start and end time. It supports filtering anomalies based on a threshold and can limit results to
126+
* specific entities if provided.
127+
*
128+
* In the context of anomaly results, the startTime and endTime parameters are used to compare against the data_end_time.
129+
* Using data_end_time instead of data_start_time is crucial because, within HC heatmap cells, the startTime and
130+
* endTime are derived from each cell's start and end times, which are determined based on the plotTime—coinciding
131+
* with the data_end_time. This alignment ensures that the temporal data within each heatmap cell accurately
132+
* reflects the intervals intended for analysis.
133+
*
134+
* @param startTime - The epoch time (in milliseconds) marking the start of the date range for the query.
135+
* @param endTime - The epoch time (in milliseconds) marking the end of the date range for the query.
136+
* @param anomalyOnly - Optional. If true, the query will return only results where anomalies are detected
137+
* (anomaly threshold is set to 0). If false or omitted, it will include all results
138+
* (anomaly threshold is set to -1). Default is `false`.
139+
* @param entityList - Optional. An array of entities to filter the results. If omitted, results are not filtered
140+
* by entities. Default is `undefined`.
141+
*
142+
* @returns An object containing the necessary parameters for the anomaly results search query. This object includes:
143+
* - `from`: The starting index for fetching results (always set to 0).
144+
* - `size`: The maximum number of anomalies to return (`MAX_ANOMALIES`).
145+
* - `sortDirection`: The sorting order of results, set to descending (`SORT_DIRECTION.DESC`).
146+
* - `sortField`: The field used to sort the data, set to data end time (`AD_DOC_FIELDS.DATA_END_TIME`).
147+
* - `startTime`: Passed start time for the search range.
148+
* - `endTime`: Passed end time for the search range.
149+
* - `fieldName`: Field used to query the data, set to data end time (`AD_DOC_FIELDS.DATA_END_TIME`).
150+
* - `anomalyThreshold`: The minimum score threshold for anomalies, dependent on `anomalyOnly` parameter.
151+
* - `entityList`: A JSON string representing the list of entities to filter the results by.
152+
*/
121153
export const buildParamsForGetAnomalyResultsWithDateRange = (
122154
startTime: number,
123155
endTime: number,
@@ -128,10 +160,10 @@ export const buildParamsForGetAnomalyResultsWithDateRange = (
128160
from: 0,
129161
size: MAX_ANOMALIES,
130162
sortDirection: SORT_DIRECTION.DESC,
131-
sortField: AD_DOC_FIELDS.DATA_START_TIME,
163+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
132164
startTime: startTime,
133165
endTime: endTime,
134-
fieldName: AD_DOC_FIELDS.DATA_START_TIME,
166+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
135167
anomalyThreshold: anomalyOnly ? 0 : -1,
136168
entityList: JSON.stringify(entityList),
137169
};

release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ You can use the plugin with the same version of the [Open Distro for Elasticsear
6161
- Tune AD result charts [PR #102](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/102)
6262
- Use annotation for live chart [PR #119](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/119)
6363
- Set fixed height for anomalies live chart [PR #123](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/123)
64-
- Use scientific notation when number less than 0.01 on live chart [PR #124](https://github.com/opendistro-for-elasticsearchanomaly-detection-kibana-plugin/pull/124)
64+
- Use scientific notation when number less than 0.01 on live chart [PR #124](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/124)
6565
- Use bucket aggregation for anomaly distribution [PR #126](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/126)
6666

6767
## Bug Fixes

0 commit comments

Comments
 (0)