Skip to content

Commit 9528c58

Browse files
kaituokaituo
committed
build query parameters using data_end_time
This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing. The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead. This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis. Testing done: 1. reproduced the issue and verified the fix. 2. added unit tests. 3. Confirmed that single stream detector result views remain functional post-changes. Signed-off-by: Kaituo Li <kaituo@amazon.com>
1 parent 48acb93 commit 9528c58

7 files changed

+160
-48
lines changed

‎.github/workflows/build-and-test-workflow.yml

+6
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ jobs:
8888
run: |
8989
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
9090
yarn osd bootstrap --single-version=loose
91+
- name: Set npm to use bash for shell
92+
if: ${{ matrix.os == 'windows-latest' }}
93+
run: |
94+
# Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json build script)
95+
# commands work as intended
96+
npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
9197
- name: Build the plugin
9298
run: |
9399
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin

‎.github/workflows/remote-integ-tests-workflow.yml

+56-40
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,14 @@ on:
99
pull_request:
1010
branches:
1111
- "*"
12+
13+
CI: 1
14+
# avoid warnings like "tput: No value for $TERM and no -T specified"
15+
TERM: xterm
16+
OPENSEARCH_DASHBOARDS_VERSION: 'main'
17+
OPENSEARCH_VERSION: '3.0.0'
18+
OPENSEARCH_PLUGIN_VERSION: '3.0.0.0'
19+
1220
jobs:
1321
test-without-security:
1422
name: Run integ tests without security
@@ -37,30 +45,39 @@ jobs:
3745
uses: actions/checkout@v2
3846
with:
3947
repository: opensearch-project/OpenSearch-Dashboards
40-
ref: '${{ github.base_ref }}'
48+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
4149
path: OpenSearch-Dashboards
50+
fetch-depth: 0
51+
filter: |
52+
cypress
53+
test
4254
4355
- name: Checkout Anomaly Detection OpenSearch Dashboards plugin
4456
uses: actions/checkout@v2
4557
with:
4658
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
4759

48-
- name: Setup Node
49-
uses: actions/setup-node@v3
60+
- id: tool-versions
61+
run: |
62+
echo "node_version=$(cat .node-version)" >> $GITHUB_OUTPUT
63+
echo "yarn_version=$(jq -r '.engines.yarn' package.json)" >> $GITHUB_OUTPUT
64+
working-directory: OpenSearch-Dashboards
65+
shell: bash
66+
67+
- uses: actions/setup-node@v1
5068
with:
51-
node-version-file: './OpenSearch-Dashboards/.nvmrc'
69+
node-version: ${{ steps.tool-versions.outputs.node_version }}
5270
registry-url: 'https://registry.npmjs.org'
5371

54-
- name: Install Yarn
55-
# Need to use bash to avoid having a windows/linux specific step
56-
shell: bash
72+
- name: Setup Opensearch Dashboards
5773
run: |
58-
YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
59-
echo "Installing yarn@$YARN_VERSION"
60-
npm i -g yarn@$YARN_VERSION
61-
62-
- run: node -v
63-
- run: yarn -v
74+
npm uninstall -g yarn
75+
echo "Installing yarn ${{ steps.tool-versions.outputs.yarn_version }}"
76+
npm i -g yarn@${{ steps.tool-versions.outputs.yarn_version }}
77+
yarn cache clean
78+
yarn add sha.js
79+
working-directory: OpenSearch-Dashboards
80+
shell: bash
6481

6582
- name: Checkout Anomaly-Detection
6683
uses: actions/checkout@v2
@@ -79,6 +96,11 @@ jobs:
7996
timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
8097
shell: bash
8198

99+
- name: Check OpenSearch Running on Linux
100+
if: ${{ matrix.os != 'windows-latest' }}
101+
run: curl http://localhost:9200/
102+
shell: bash
103+
82104
- name: Bootstrap the plugin
83105
run: |
84106
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
@@ -87,47 +109,41 @@ jobs:
87109
- name: Run OpenSearch Dashboards server
88110
run: |
89111
cd OpenSearch-Dashboards
90-
yarn start --no-base-path --no-watch &
112+
nohup yarn start --no-base-path --no-watch | tee dashboard.log &
91113
shell: bash
92114

115+
- name : Check If OpenSearch Dashboards Is Ready
116+
if: ${{ matrix.os != 'windows-latest' }}
117+
run: |
118+
if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
119+
echo "OpenSearch Dashboards compiled successfully."
120+
else
121+
echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
122+
exit 1
123+
fi
124+
working-directory: OpenSearch-Dashboards
125+
93126
# Window is slow so wait longer
94127
- name: Sleep until OSD server starts - windows
95128
if: ${{ matrix.os == 'windows-latest' }}
96129
run: Start-Sleep -s 400
97130
shell: powershell
98131

99-
- name: Sleep until OSD server starts - non-windows
100-
if: ${{ matrix.os != 'windows-latest' }}
101-
run: sleep 300
102-
shell: bash
103-
104132
- name: Checkout opensearch-dashboards-functional-test
105133
uses: actions/checkout@v2
106134
with:
107135
path: opensearch-dashboards-functional-test
108136
repository: opensearch-project/opensearch-dashboards-functional-test
109-
ref: '${{ github.base_ref }}'
137+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
138+
fetch-depth: 0
110139

111-
- name: Get Cypress version
112-
id: cypress_version
140+
- name: Install Cypress
113141
run: |
114-
echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')"
115-
116-
- name: Cache Cypress
117-
id: cache-cypress
118-
uses: actions/cache@v1
119-
with:
120-
path: ${{ matrix.cypress_cache_folder }}
121-
key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
122-
env:
123-
CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
124-
- run: npx cypress cache list
125-
- run: npx cypress cache path
142+
npm install cypress --save-dev
143+
shell: bash
144+
working-directory: opensearch-dashboards-functional-test
126145

127146
- name: Run AD cypress tests
128-
uses: cypress-io/github-action@v2
129-
with:
130-
working-directory: opensearch-dashboards-functional-test
131-
command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
132-
env:
133-
CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}
147+
run: |
148+
yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/anomaly-detection-dashboards-plugin/*.js'
149+
working-directory: opensearch-dashboards-functional-test

‎package.json

+5-5
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@
44
"description": "OpenSearch Anomaly Detection Dashboards Plugin",
55
"main": "index.js",
66
"config": {
7-
"plugin_version": "3.0.0.0",
8-
"plugin_name": "anomalyDetectionDashboards",
9-
"plugin_zip_name": "anomaly-detection-dashboards"
7+
"id": "anomalyDetectionDashboards",
8+
"zip_name": "anomaly-detection-dashboards"
109
},
1110
"scripts": {
1211
"osd": "node ../../scripts/osd",
1312
"opensearch": "node ../../scripts/opensearch",
1413
"lint": "node ../../scripts/eslint .",
1514
"plugin-helpers": "node ../../scripts/plugin_helpers",
1615
"test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
17-
"build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
16+
"build": "yarn plugin-helpers build",
17+
"postbuild": "echo Renaming artifact to [$npm_package_config_zip_name-$npm_package_version.zip] && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip"
1818
},
1919
"lint-staged": {
2020
"*.{ts,tsx,js,jsx,json,css,md}": [
@@ -56,4 +56,4 @@
5656
"browserify-sign": "^4.2.2",
5757
"axios": "^1.6.1"
5858
}
59-
}
59+
}

‎public/pages/DetectorResults/containers/AnomalyResults.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
255255
endDate: adjustedCurrentTime.valueOf(),
256256
} as DateRange;
257257

258+
// build result search query params relative to data end time
258259
const params = buildParamsForGetAnomalyResultsWithDateRange(
259260
featureDataPointsRange.startDate,
260261
featureDataPointsRange.endDate

‎public/pages/utils/__tests__/anomalyResultUtils.test.ts

+57
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
getFeatureMissingDataAnnotations,
1414
getFeatureDataPointsForDetector,
1515
parsePureAnomalies,
16+
buildParamsForGetAnomalyResultsWithDateRange,
1617
} from '../anomalyResultUtils';
1718
import { getRandomDetector } from '../../../redux/reducers/__tests__/utils';
1819
import {
@@ -22,11 +23,16 @@ import {
2223
AnomalyData,
2324
} from '../../../models/interfaces';
2425
import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants';
26+
import { MAX_ANOMALIES } from '../../../utils/constants';
27+
import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants';
2528

2629
describe('anomalyResultUtils', () => {
2730
let randomDetector_20_min: Detector;
2831
let randomDetector_20_sec: Detector;
2932
let feature_id = 'deny_max';
33+
const startTime = 1609459200000; // January 1, 2021
34+
const endTime = 1609545600000; // January 2, 2021
35+
3036
beforeAll(() => {
3137
randomDetector_20_min = {
3238
...getRandomDetector(true),
@@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => {
569575
)
570576
).toEqual([]);
571577
});
578+
test('should correctly build parameters with default options', () => {
579+
const expected = {
580+
from: 0,
581+
size: MAX_ANOMALIES,
582+
sortDirection: SORT_DIRECTION.DESC,
583+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
584+
startTime: startTime,
585+
endTime: endTime,
586+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
587+
anomalyThreshold: -1,
588+
entityList: undefined, // Default as an empty array stringified
589+
};
590+
591+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime);
592+
expect(result).toEqual(expected);
593+
});
594+
595+
test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => {
596+
const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }];
597+
const expected = {
598+
from: 0,
599+
size: MAX_ANOMALIES,
600+
sortDirection: SORT_DIRECTION.DESC,
601+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
602+
startTime: startTime,
603+
endTime: endTime,
604+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
605+
anomalyThreshold: 0, // because anomalyOnly is true
606+
entityList: JSON.stringify(entities),
607+
};
608+
609+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities);
610+
expect(result).toEqual(expected);
611+
});
612+
613+
test('should handle undefined `entityList` as an empty array JSON string', () => {
614+
const expected = {
615+
from: 0,
616+
size: MAX_ANOMALIES,
617+
sortDirection: SORT_DIRECTION.DESC,
618+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
619+
startTime: startTime,
620+
endTime: endTime,
621+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
622+
anomalyThreshold: -1, // default as anomalyOnly is false
623+
entityList: undefined, // Default for undefined entityList
624+
};
625+
626+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined);
627+
expect(result).toEqual(expected);
628+
});
572629
});
573630

574631
describe('parsePureAnomalies()', () => {

‎public/pages/utils/anomalyResultUtils.ts

+34-2
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,38 @@ export const getLiveAnomalyResults = (
118118
);
119119
};
120120

121+
/**
122+
* Builds search query parameters for retrieving anomaly results within a specified date range.
123+
*
124+
* This function constructs a parameter object for querying an anomaly detection system, filtering results
125+
* by a given start and end time. It supports filtering anomalies based on a threshold and can limit results to
126+
* specific entities if provided.
127+
*
128+
* In the context of anomaly results, the startTime and endTime parameters are used to compare against the data_end_time.
129+
* Using data_end_time instead of data_start_time is crucial because, within HC heatmap cells, the startTime and
130+
* endTime are derived from each cell's start and end times, which are determined based on the plotTime—coinciding
131+
* with the data_end_time. This alignment ensures that the temporal data within each heatmap cell accurately
132+
* reflects the intervals intended for analysis.
133+
*
134+
* @param startTime - The epoch time (in milliseconds) marking the start of the date range for the query.
135+
* @param endTime - The epoch time (in milliseconds) marking the end of the date range for the query.
136+
* @param anomalyOnly - Optional. If true, the query will return only results where anomalies are detected
137+
* (anomaly threshold is set to 0). If false or omitted, it will include all results
138+
* (anomaly threshold is set to -1). Default is `false`.
139+
* @param entityList - Optional. An array of entities to filter the results. If omitted, results are not filtered
140+
* by entities. Default is `undefined`.
141+
*
142+
* @returns An object containing the necessary parameters for the anomaly results search query. This object includes:
143+
* - `from`: The starting index for fetching results (always set to 0).
144+
* - `size`: The maximum number of anomalies to return (`MAX_ANOMALIES`).
145+
* - `sortDirection`: The sorting order of results, set to descending (`SORT_DIRECTION.DESC`).
146+
* - `sortField`: The field used to sort the data, set to data end time (`AD_DOC_FIELDS.DATA_END_TIME`).
147+
* - `startTime`: Passed start time for the search range.
148+
* - `endTime`: Passed end time for the search range.
149+
* - `fieldName`: Field used to query the data, set to data end time (`AD_DOC_FIELDS.DATA_END_TIME`).
150+
* - `anomalyThreshold`: The minimum score threshold for anomalies, dependent on `anomalyOnly` parameter.
151+
* - `entityList`: A JSON string representing the list of entities to filter the results by.
152+
*/
121153
export const buildParamsForGetAnomalyResultsWithDateRange = (
122154
startTime: number,
123155
endTime: number,
@@ -128,10 +160,10 @@ export const buildParamsForGetAnomalyResultsWithDateRange = (
128160
from: 0,
129161
size: MAX_ANOMALIES,
130162
sortDirection: SORT_DIRECTION.DESC,
131-
sortField: AD_DOC_FIELDS.DATA_START_TIME,
163+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
132164
startTime: startTime,
133165
endTime: endTime,
134-
fieldName: AD_DOC_FIELDS.DATA_START_TIME,
166+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
135167
anomalyThreshold: anomalyOnly ? 0 : -1,
136168
entityList: JSON.stringify(entityList),
137169
};

‎release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ You can use the plugin with the same version of the [Open Distro for Elasticsear
6161
- Tune AD result charts [PR #102](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/102)
6262
- Use annotation for live chart [PR #119](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/119)
6363
- Set fixed height for anomalies live chart [PR #123](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/123)
64-
- Use scientific notation when number less than 0.01 on live chart [PR #124](https://github.com/opendistro-for-elasticsearchanomaly-detection-kibana-plugin/pull/124)
64+
- Use scientific notation when number less than 0.01 on live chart [PR #124](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/124)
6565
- Use bucket aggregation for anomaly distribution [PR #126](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/126)
6666

6767
## Bug Fixes

0 commit comments

Comments
 (0)