Skip to content

Commit d5ad627

Browse files
committed
build query parameters using data_end_time
This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing. The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead. This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis. Testing done: 1. reproduced the issue and verified the fix. 2. added unit tests. 3. Confirmed that single stream detector result views remain functional post-changes. Signed-off-by: Kaituo Li <kaituo@amazon.com>
1 parent 48acb93 commit d5ad627

6 files changed

+117
-5
lines changed

.github/workflows/build-and-test-workflow.yml

+6
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ jobs:
8888
run: |
8989
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
9090
yarn osd bootstrap --single-version=loose
91+
- name: Set npm to use bash for shell
92+
if: ${{ matrix.os == 'windows-latest' }}
93+
run: |
94+
# Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json build script)
95+
# commands work as intended
96+
npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
9197
- name: Build the plugin
9298
run: |
9399
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin

.github/workflows/remote-integ-tests-workflow.yml

+18-2
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,11 @@ jobs:
7979
timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
8080
shell: bash
8181

82+
- name: Check OpenSearch Running on Linux
83+
if: ${{ matrix.os != 'windows-latest' }}
84+
run: curl http://localhost:9200/
85+
shell: bash
86+
8287
- name: Bootstrap the plugin
8388
run: |
8489
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
@@ -87,9 +92,20 @@ jobs:
8792
- name: Run OpenSearch Dashboards server
8893
run: |
8994
cd OpenSearch-Dashboards
90-
yarn start --no-base-path --no-watch &
95+
nohup yarn start --no-base-path --no-watch | tee dashboard.log &
9196
shell: bash
9297

98+
- name : Check If OpenSearch Dashboards Is Ready
99+
if: ${{ matrix.os != 'windows-latest' }}
100+
run: |
101+
if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
102+
echo "OpenSearch Dashboards compiled successfully."
103+
else
104+
echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
105+
exit 1
106+
fi
107+
working-directory: OpenSearch-Dashboards
108+
93109
# Window is slow so wait longer
94110
- name: Sleep until OSD server starts - windows
95111
if: ${{ matrix.os == 'windows-latest' }}
@@ -128,6 +144,6 @@ jobs:
128144
uses: cypress-io/github-action@v2
129145
with:
130146
working-directory: opensearch-dashboards-functional-test
131-
command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
147+
command: yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/anomaly-detection-dashboards-plugin/*.js'
132148
env:
133149
CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}

public/pages/DetectorResults/containers/AnomalyResults.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
255255
endDate: adjustedCurrentTime.valueOf(),
256256
} as DateRange;
257257

258+
// build result search query params relative to data end time
258259
const params = buildParamsForGetAnomalyResultsWithDateRange(
259260
featureDataPointsRange.startDate,
260261
featureDataPointsRange.endDate

public/pages/utils/__tests__/anomalyResultUtils.test.ts

+57
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
getFeatureMissingDataAnnotations,
1414
getFeatureDataPointsForDetector,
1515
parsePureAnomalies,
16+
buildParamsForGetAnomalyResultsWithDateRange,
1617
} from '../anomalyResultUtils';
1718
import { getRandomDetector } from '../../../redux/reducers/__tests__/utils';
1819
import {
@@ -22,11 +23,16 @@ import {
2223
AnomalyData,
2324
} from '../../../models/interfaces';
2425
import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants';
26+
import { MAX_ANOMALIES } from '../../../utils/constants';
27+
import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants';
2528

2629
describe('anomalyResultUtils', () => {
2730
let randomDetector_20_min: Detector;
2831
let randomDetector_20_sec: Detector;
2932
let feature_id = 'deny_max';
33+
const startTime = 1609459200000; // January 1, 2021
34+
const endTime = 1609545600000; // January 2, 2021
35+
3036
beforeAll(() => {
3137
randomDetector_20_min = {
3238
...getRandomDetector(true),
@@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => {
569575
)
570576
).toEqual([]);
571577
});
578+
test('should correctly build parameters with default options', () => {
579+
const expected = {
580+
from: 0,
581+
size: MAX_ANOMALIES,
582+
sortDirection: SORT_DIRECTION.DESC,
583+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
584+
startTime: startTime,
585+
endTime: endTime,
586+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
587+
anomalyThreshold: -1,
588+
entityList: undefined, // Default as an empty array stringified
589+
};
590+
591+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime);
592+
expect(result).toEqual(expected);
593+
});
594+
595+
test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => {
596+
const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }];
597+
const expected = {
598+
from: 0,
599+
size: MAX_ANOMALIES,
600+
sortDirection: SORT_DIRECTION.DESC,
601+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
602+
startTime: startTime,
603+
endTime: endTime,
604+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
605+
anomalyThreshold: 0, // because anomalyOnly is true
606+
entityList: JSON.stringify(entities),
607+
};
608+
609+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities);
610+
expect(result).toEqual(expected);
611+
});
612+
613+
test('should handle undefined `entityList` as an empty array JSON string', () => {
614+
const expected = {
615+
from: 0,
616+
size: MAX_ANOMALIES,
617+
sortDirection: SORT_DIRECTION.DESC,
618+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
619+
startTime: startTime,
620+
endTime: endTime,
621+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
622+
anomalyThreshold: -1, // default as anomalyOnly is false
623+
entityList: undefined, // Default for undefined entityList
624+
};
625+
626+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined);
627+
expect(result).toEqual(expected);
628+
});
572629
});
573630

574631
describe('parsePureAnomalies()', () => {

public/pages/utils/anomalyResultUtils.ts

+34-2
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,38 @@ export const getLiveAnomalyResults = (
118118
);
119119
};
120120

121+
/**
122+
* Builds search query parameters for retrieving anomaly results within a specified date range.
123+
*
124+
* This function constructs a parameter object for querying an anomaly detection system, filtering results
125+
* by a given start and end time. It supports filtering anomalies based on a threshold and can limit results to
126+
* specific entities if provided.
127+
*
128+
* In the context of anomaly results, the startTime and endTime parameters are used to compare against the data_end_time.
129+
* Using data_end_time instead of data_start_time is crucial because, within HC heatmap cells, the startTime and
130+
* endTime are derived from each cell's start and end times, which are determined based on the plotTime—coinciding
131+
* with the data_end_time. This alignment ensures that the temporal data within each heatmap cell accurately
132+
* reflects the intervals intended for analysis.
133+
*
134+
* @param startTime - The epoch time (in milliseconds) marking the start of the date range for the query.
135+
* @param endTime - The epoch time (in milliseconds) marking the end of the date range for the query.
136+
* @param anomalyOnly - Optional. If true, the query will return only results where anomalies are detected
137+
* (anomaly threshold is set to 0). If false or omitted, it will include all results
138+
* (anomaly threshold is set to -1). Default is `false`.
139+
* @param entityList - Optional. An array of entities to filter the results. If omitted, results are not filtered
140+
* by entities. Default is `undefined`.
141+
*
142+
* @returns An object containing the necessary parameters for the anomaly results search query. This object includes:
143+
* - `from`: The starting index for fetching results (always set to 0).
144+
* - `size`: The maximum number of anomalies to return (`MAX_ANOMALIES`).
145+
* - `sortDirection`: The sorting order of results, set to descending (`SORT_DIRECTION.DESC`).
146+
* - `sortField`: The field used to sort the data, set to data end time (`AD_DOC_FIELDS.DATA_END_TIME`).
147+
* - `startTime`: Passed start time for the search range.
148+
* - `endTime`: Passed end time for the search range.
149+
* - `fieldName`: Field used to query the data, set to data end time (`AD_DOC_FIELDS.DATA_END_TIME`).
150+
* - `anomalyThreshold`: The minimum score threshold for anomalies, dependent on `anomalyOnly` parameter.
151+
* - `entityList`: A JSON string representing the list of entities to filter the results by.
152+
*/
121153
export const buildParamsForGetAnomalyResultsWithDateRange = (
122154
startTime: number,
123155
endTime: number,
@@ -128,10 +160,10 @@ export const buildParamsForGetAnomalyResultsWithDateRange = (
128160
from: 0,
129161
size: MAX_ANOMALIES,
130162
sortDirection: SORT_DIRECTION.DESC,
131-
sortField: AD_DOC_FIELDS.DATA_START_TIME,
163+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
132164
startTime: startTime,
133165
endTime: endTime,
134-
fieldName: AD_DOC_FIELDS.DATA_START_TIME,
166+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
135167
anomalyThreshold: anomalyOnly ? 0 : -1,
136168
entityList: JSON.stringify(entityList),
137169
};

release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ You can use the plugin with the same version of the [Open Distro for Elasticsear
6161
- Tune AD result charts [PR #102](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/102)
6262
- Use annotation for live chart [PR #119](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/119)
6363
- Set fixed height for anomalies live chart [PR #123](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/123)
64-
- Use scientific notation when number less than 0.01 on live chart [PR #124](https://github.com/opendistro-for-elasticsearchanomaly-detection-kibana-plugin/pull/124)
64+
- Use scientific notation when number less than 0.01 on live chart [PR #124](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/124)
6565
- Use bucket aggregation for anomaly distribution [PR #126](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/126)
6666

6767
## Bug Fixes

0 commit comments

Comments
 (0)