Skip to content

Commit 1765d6c

Browse files
committed
build query parameters using data_end_time
This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing. The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead. This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis. Testing done: 1. reproduced the issue and verified the fix. 2. added unit tests. 3. Confirmed that single stream detector result views remain functional post-changes. Signed-off-by: Kaituo Li <kaituo@amazon.com>
1 parent 48acb93 commit 1765d6c

7 files changed

+215
-103
lines changed

.github/workflows/build-and-test-workflow.yml

+6
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ jobs:
8888
run: |
8989
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
9090
yarn osd bootstrap --single-version=loose
91+
- name: Set npm to use bash for shell
92+
if: ${{ matrix.os == 'windows-latest' }}
93+
run: |
94+
# Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json build script)
95+
# commands work as intended
96+
npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
9197
- name: Build the plugin
9298
run: |
9399
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,30 @@
1-
# Running AD integ tests stored in https://github.com/opensearch-project/opensearch-dashboards-functional-test
2-
# In the future we should pull dependencies from bundled build snapshots. Because that is not available
3-
# yet we build the cluster from source (besides core Opensearch, which is a pulled min artifact).
4-
name: Remote integ tests workflow
5-
on:
6-
push:
7-
branches:
8-
- "*"
9-
pull_request:
10-
branches:
11-
- "*"
1+
name: FTR E2E AD Workbench Test
2+
3+
on: [pull_request, push]
4+
5+
env:
6+
CI: 1
7+
# avoid warnings like "tput: No value for $TERM and no -T specified"
8+
TERM: xterm
9+
OPENSEARCH_DASHBOARDS_VERSION: 'main'
10+
OPENSEARCH_VERSION: '3.0.0'
11+
OPENSEARCH_PLUGIN_VERSION: '3.0.0.0'
12+
1213
jobs:
13-
test-without-security:
14-
name: Run integ tests without security
14+
tests:
15+
name: Run FTR E2E AD Workbench Tests
1516
strategy:
17+
fail-fast: false
1618
matrix:
17-
os: [ubuntu-latest, windows-latest]
18-
java: [11]
19-
include:
20-
- os: windows-latest
21-
cypress_cache_folder: ~/AppData/Local/Cypress/Cache
22-
- os: ubuntu-latest
23-
cypress_cache_folder: ~/.cache/Cypress
19+
os: [ ubuntu-latest ]
20+
jdk: [ 11 ]
2421
runs-on: ${{ matrix.os }}
25-
steps:
26-
- name: Set up Java 11
27-
uses: actions/setup-java@v3
28-
with:
29-
distribution: 'corretto'
30-
java-version: '11'
31-
32-
- name: Enable longer filenames
33-
if: ${{ matrix.os == 'windows-latest' }}
34-
run: git config --system core.longpaths true
35-
36-
- name: Checkout OpenSearch Dashboards
37-
uses: actions/checkout@v2
38-
with:
39-
repository: opensearch-project/OpenSearch-Dashboards
40-
ref: '${{ github.base_ref }}'
41-
path: OpenSearch-Dashboards
42-
43-
- name: Checkout Anomaly Detection OpenSearch Dashboards plugin
44-
uses: actions/checkout@v2
45-
with:
46-
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
4722

48-
- name: Setup Node
49-
uses: actions/setup-node@v3
23+
steps:
24+
- name: Set up JDK
25+
uses: actions/setup-java@v1
5026
with:
51-
node-version-file: './OpenSearch-Dashboards/.nvmrc'
52-
registry-url: 'https://registry.npmjs.org'
53-
54-
- name: Install Yarn
55-
# Need to use bash to avoid having a windows/linux specific step
56-
shell: bash
57-
run: |
58-
YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
59-
echo "Installing yarn@$YARN_VERSION"
60-
npm i -g yarn@$YARN_VERSION
61-
62-
- run: node -v
63-
- run: yarn -v
27+
java-version: ${{ matrix.jdk }}
6428

6529
- name: Checkout Anomaly-Detection
6630
uses: actions/checkout@v2
@@ -72,62 +36,114 @@ jobs:
7236
- name: Run OpenSearch with plugin
7337
run: |
7438
cd anomaly-detection
75-
CONFIG_PATH=../OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin/opensearch_dashboards.json
76-
OPENSEARCH_VERSION=$(node -p "require('$CONFIG_PATH').opensearchDashboardsVersion")-SNAPSHOT
77-
echo "Using OpenSearch version $OPENSEARCH_VERSION"
78-
./gradlew run -Dopensearch.version=$OPENSEARCH_VERSION &
39+
./gradlew run &
7940
timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
8041
shell: bash
42+
43+
- name: Check OpenSearch Running on Linux
44+
if: ${{ runner.os != 'Windows'}}
45+
run: curl http://localhost:9200/
46+
shell: bash
47+
48+
- name: Show OpenSearch Logs
49+
if: always()
50+
run: cat ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/logs/opensearch.log
51+
shell: bash
52+
53+
- name: Checkout OpenSearch Dashboards
54+
uses: actions/checkout@v2
55+
with:
56+
path: OpenSearch-Dashboards
57+
repository: opensearch-project/OpenSearch-Dashboards
58+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
59+
fetch-depth: 0
60+
filter: |
61+
cypress
62+
test
8163
82-
- name: Bootstrap the plugin
83-
run: |
84-
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
85-
yarn osd bootstrap --single-version=loose
64+
- name: Checkout AD in OpenSearch Dashboards Plugins Dir
65+
uses: actions/checkout@v2
66+
with:
67+
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
8668

87-
- name: Run OpenSearch Dashboards server
69+
- id: tool-versions
8870
run: |
89-
cd OpenSearch-Dashboards
90-
yarn start --no-base-path --no-watch &
71+
echo "node_version=$(cat .node-version)" >> $GITHUB_OUTPUT
72+
echo "yarn_version=$(jq -r '.engines.yarn' package.json)" >> $GITHUB_OUTPUT
73+
working-directory: OpenSearch-Dashboards
9174
shell: bash
9275

93-
# Window is slow so wait longer
94-
- name: Sleep until OSD server starts - windows
95-
if: ${{ matrix.os == 'windows-latest' }}
96-
run: Start-Sleep -s 400
97-
shell: powershell
76+
- uses: actions/setup-node@v1
77+
with:
78+
node-version: ${{ steps.tool-versions.outputs.node_version }}
79+
registry-url: 'https://registry.npmjs.org'
9880

99-
- name: Sleep until OSD server starts - non-windows
100-
if: ${{ matrix.os != 'windows-latest' }}
101-
run: sleep 300
81+
- name: Setup Opensearch Dashboards
82+
run: |
83+
npm uninstall -g yarn
84+
echo "Installing yarn ${{ steps.tool-versions.outputs.yarn_version }}"
85+
npm i -g yarn@${{ steps.tool-versions.outputs.yarn_version }}
86+
yarn cache clean
87+
yarn add sha.js
88+
working-directory: OpenSearch-Dashboards
10289
shell: bash
10390

104-
- name: Checkout opensearch-dashboards-functional-test
91+
- name: Boodstrap Opensearch Dashboards
92+
run: |
93+
yarn osd bootstrap --single-version=loose
94+
working-directory: OpenSearch-Dashboards
95+
96+
- name: Run Opensearch Dashboards with Query Workbench Installed
97+
run: |
98+
nohup yarn start --no-base-path --no-watch | tee dashboard.log &
99+
working-directory: OpenSearch-Dashboards
100+
101+
- name : Check If OpenSearch Dashboards Is Ready
102+
if: ${{ runner.os == 'Linux' }}
103+
run: |
104+
if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
105+
echo "OpenSearch Dashboards compiled successfully."
106+
else
107+
echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
108+
exit 1
109+
fi
110+
working-directory: OpenSearch-Dashboards
111+
112+
- name: Checkout Dashboards Functioanl Test Repo
105113
uses: actions/checkout@v2
106114
with:
107115
path: opensearch-dashboards-functional-test
108116
repository: opensearch-project/opensearch-dashboards-functional-test
109-
ref: '${{ github.base_ref }}'
117+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
118+
fetch-depth: 0
119+
120+
- name: Install Cypress
121+
run: |
122+
npm install cypress --save-dev
123+
shell: bash
124+
working-directory: opensearch-dashboards-functional-test
110125

111126
- name: Get Cypress version
112127
id: cypress_version
113128
run: |
114-
echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')"
129+
echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')"
130+
working-directory: opensearch-dashboards-functional-test
115131

116-
- name: Cache Cypress
117-
id: cache-cypress
118-
uses: actions/cache@v1
132+
- name: Run Cypress tests
133+
run: |
134+
yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/anomaly-detection-dashboards-plugin/*.js'
135+
working-directory: opensearch-dashboards-functional-test
136+
137+
- name: Capture failure screenshots
138+
uses: actions/upload-artifact@v1
139+
if: failure()
119140
with:
120-
path: ${{ matrix.cypress_cache_folder }}
121-
key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
122-
env:
123-
CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
124-
- run: npx cypress cache list
125-
- run: npx cypress cache path
126-
127-
- name: Run AD cypress tests
128-
uses: cypress-io/github-action@v2
141+
name: cypress-screenshots-${{ matrix.os }}
142+
path: opensearch-dashboards-functional-test/cypress/screenshots
143+
144+
- name: Capture failure test video
145+
uses: actions/upload-artifact@v1
146+
if: failure()
129147
with:
130-
working-directory: opensearch-dashboards-functional-test
131-
command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
132-
env:
133-
CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}
148+
name: cypress-videos-${{ matrix.os }}
149+
path: opensearch-dashboards-functional-test/cypress/videos

package.json

+5-5
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@
44
"description": "OpenSearch Anomaly Detection Dashboards Plugin",
55
"main": "index.js",
66
"config": {
7-
"plugin_version": "3.0.0.0",
8-
"plugin_name": "anomalyDetectionDashboards",
9-
"plugin_zip_name": "anomaly-detection-dashboards"
7+
"id": "anomalyDetectionDashboards",
8+
"zip_name": "anomaly-detection-dashboards"
109
},
1110
"scripts": {
1211
"osd": "node ../../scripts/osd",
1312
"opensearch": "node ../../scripts/opensearch",
1413
"lint": "node ../../scripts/eslint .",
1514
"plugin-helpers": "node ../../scripts/plugin_helpers",
1615
"test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
17-
"build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
16+
"build": "yarn plugin-helpers build",
17+
"postbuild": "echo Renaming artifact to [$npm_package_config_zip_name-$npm_package_version.zip] && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip"
1818
},
1919
"lint-staged": {
2020
"*.{ts,tsx,js,jsx,json,css,md}": [
@@ -56,4 +56,4 @@
5656
"browserify-sign": "^4.2.2",
5757
"axios": "^1.6.1"
5858
}
59-
}
59+
}

public/pages/DetectorResults/containers/AnomalyResults.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
255255
endDate: adjustedCurrentTime.valueOf(),
256256
} as DateRange;
257257

258+
// build result search query params relative to data end time
258259
const params = buildParamsForGetAnomalyResultsWithDateRange(
259260
featureDataPointsRange.startDate,
260261
featureDataPointsRange.endDate

public/pages/utils/__tests__/anomalyResultUtils.test.ts

+57
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
getFeatureMissingDataAnnotations,
1414
getFeatureDataPointsForDetector,
1515
parsePureAnomalies,
16+
buildParamsForGetAnomalyResultsWithDateRange,
1617
} from '../anomalyResultUtils';
1718
import { getRandomDetector } from '../../../redux/reducers/__tests__/utils';
1819
import {
@@ -22,11 +23,16 @@ import {
2223
AnomalyData,
2324
} from '../../../models/interfaces';
2425
import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants';
26+
import { MAX_ANOMALIES } from '../../../utils/constants';
27+
import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants';
2528

2629
describe('anomalyResultUtils', () => {
2730
let randomDetector_20_min: Detector;
2831
let randomDetector_20_sec: Detector;
2932
let feature_id = 'deny_max';
33+
const startTime = 1609459200000; // January 1, 2021
34+
const endTime = 1609545600000; // January 2, 2021
35+
3036
beforeAll(() => {
3137
randomDetector_20_min = {
3238
...getRandomDetector(true),
@@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => {
569575
)
570576
).toEqual([]);
571577
});
578+
test('should correctly build parameters with default options', () => {
579+
const expected = {
580+
from: 0,
581+
size: MAX_ANOMALIES,
582+
sortDirection: SORT_DIRECTION.DESC,
583+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
584+
startTime: startTime,
585+
endTime: endTime,
586+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
587+
anomalyThreshold: -1,
588+
entityList: undefined, // Default as an empty array stringified
589+
};
590+
591+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime);
592+
expect(result).toEqual(expected);
593+
});
594+
595+
test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => {
596+
const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }];
597+
const expected = {
598+
from: 0,
599+
size: MAX_ANOMALIES,
600+
sortDirection: SORT_DIRECTION.DESC,
601+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
602+
startTime: startTime,
603+
endTime: endTime,
604+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
605+
anomalyThreshold: 0, // because anomalyOnly is true
606+
entityList: JSON.stringify(entities),
607+
};
608+
609+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities);
610+
expect(result).toEqual(expected);
611+
});
612+
613+
test('should handle undefined `entityList` as an empty array JSON string', () => {
614+
const expected = {
615+
from: 0,
616+
size: MAX_ANOMALIES,
617+
sortDirection: SORT_DIRECTION.DESC,
618+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
619+
startTime: startTime,
620+
endTime: endTime,
621+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
622+
anomalyThreshold: -1, // default as anomalyOnly is false
623+
entityList: undefined, // Default for undefined entityList
624+
};
625+
626+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined);
627+
expect(result).toEqual(expected);
628+
});
572629
});
573630

574631
describe('parsePureAnomalies()', () => {

0 commit comments

Comments
 (0)