Skip to content

Commit acbf0fa

Browse files
committed
build query parameters using data_end_time
This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing. The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead. This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis. Testing done: 1. reproduced the issue and verified the fix. 2. added unit tests. 3. Confirmed that single stream detector result views remain functional post-changes. Signed-off-by: Kaituo Li <kaituo@amazon.com>
1 parent 48acb93 commit acbf0fa

7 files changed

+214
-99
lines changed

.github/workflows/build-and-test-workflow.yml

+6
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ jobs:
8888
run: |
8989
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
9090
yarn osd bootstrap --single-version=loose
91+
- name: Set npm to use bash for shell
92+
if: ${{ matrix.os == 'windows-latest' }}
93+
run: |
94+
# Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json build script)
95+
# commands work as intended
96+
npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
9197
- name: Build the plugin
9298
run: |
9399
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,30 @@
1-
# Running AD integ tests stored in https://github.com/opensearch-project/opensearch-dashboards-functional-test
2-
# In the future we should pull dependencies from bundled build snapshots. Because that is not available
3-
# yet we build the cluster from source (besides core Opensearch, which is a pulled min artifact).
4-
name: Remote integ tests workflow
5-
on:
6-
push:
7-
branches:
8-
- "*"
9-
pull_request:
10-
branches:
11-
- "*"
1+
name: FTR E2E AD Workbench Test
2+
3+
on: [pull_request, push]
4+
5+
env:
6+
CI: 1
7+
# avoid warnings like "tput: No value for $TERM and no -T specified"
8+
TERM: xterm
9+
OPENSEARCH_DASHBOARDS_VERSION: 'main'
10+
OPENSEARCH_VERSION: '3.0.0'
11+
OPENSEARCH_PLUGIN_VERSION: '3.0.0.0'
12+
1213
jobs:
13-
test-without-security:
14-
name: Run integ tests without security
14+
tests:
15+
name: Run FTR E2E AD Workbench Tests
1516
strategy:
17+
fail-fast: false
1618
matrix:
17-
os: [ubuntu-latest, windows-latest]
18-
java: [11]
19-
include:
20-
- os: windows-latest
21-
cypress_cache_folder: ~/AppData/Local/Cypress/Cache
22-
- os: ubuntu-latest
23-
cypress_cache_folder: ~/.cache/Cypress
19+
os: [ ubuntu-latest ]
20+
jdk: [ 11 ]
2421
runs-on: ${{ matrix.os }}
25-
steps:
26-
- name: Set up Java 11
27-
uses: actions/setup-java@v3
28-
with:
29-
distribution: 'corretto'
30-
java-version: '11'
31-
32-
- name: Enable longer filenames
33-
if: ${{ matrix.os == 'windows-latest' }}
34-
run: git config --system core.longpaths true
35-
36-
- name: Checkout OpenSearch Dashboards
37-
uses: actions/checkout@v2
38-
with:
39-
repository: opensearch-project/OpenSearch-Dashboards
40-
ref: '${{ github.base_ref }}'
41-
path: OpenSearch-Dashboards
42-
43-
- name: Checkout Anomaly Detection OpenSearch Dashboards plugin
44-
uses: actions/checkout@v2
45-
with:
46-
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
4722

48-
- name: Setup Node
49-
uses: actions/setup-node@v3
23+
steps:
24+
- name: Set up JDK
25+
uses: actions/setup-java@v1
5026
with:
51-
node-version-file: './OpenSearch-Dashboards/.nvmrc'
52-
registry-url: 'https://registry.npmjs.org'
53-
54-
- name: Install Yarn
55-
# Need to use bash to avoid having a windows/linux specific step
56-
shell: bash
57-
run: |
58-
YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
59-
echo "Installing yarn@$YARN_VERSION"
60-
npm i -g yarn@$YARN_VERSION
61-
62-
- run: node -v
63-
- run: yarn -v
27+
java-version: ${{ matrix.jdk }}
6428

6529
- name: Checkout Anomaly-Detection
6630
uses: actions/checkout@v2
@@ -78,56 +42,111 @@ jobs:
7842
./gradlew run -Dopensearch.version=$OPENSEARCH_VERSION &
7943
timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
8044
shell: bash
45+
46+
- name: Check OpenSearch Running on Linux
47+
if: ${{ runner.os != 'Windows'}}
48+
run: curl http://localhost:9200/
49+
shell: bash
50+
51+
- name: Show OpenSearch Logs
52+
if: always()
53+
run: cat ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/logs/opensearch.log
54+
shell: bash
55+
56+
- name: Checkout OpenSearch Dashboards
57+
uses: actions/checkout@v2
58+
with:
59+
path: OpenSearch-Dashboards
60+
repository: opensearch-project/OpenSearch-Dashboards
61+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
62+
fetch-depth: 0
63+
filter: |
64+
cypress
65+
test
8166
82-
- name: Bootstrap the plugin
83-
run: |
84-
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
85-
yarn osd bootstrap --single-version=loose
67+
- name: Checkout AD in OpenSearch Dashboards Plugins Dir
68+
uses: actions/checkout@v2
69+
with:
70+
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
8671

87-
- name: Run OpenSearch Dashboards server
72+
- id: tool-versions
8873
run: |
89-
cd OpenSearch-Dashboards
90-
yarn start --no-base-path --no-watch &
74+
echo "node_version=$(cat .node-version)" >> $GITHUB_OUTPUT
75+
echo "yarn_version=$(jq -r '.engines.yarn' package.json)" >> $GITHUB_OUTPUT
76+
working-directory: OpenSearch-Dashboards
9177
shell: bash
9278

93-
# Window is slow so wait longer
94-
- name: Sleep until OSD server starts - windows
95-
if: ${{ matrix.os == 'windows-latest' }}
96-
run: Start-Sleep -s 400
97-
shell: powershell
79+
- uses: actions/setup-node@v1
80+
with:
81+
node-version: ${{ steps.tool-versions.outputs.node_version }}
82+
registry-url: 'https://registry.npmjs.org'
9883

99-
- name: Sleep until OSD server starts - non-windows
100-
if: ${{ matrix.os != 'windows-latest' }}
101-
run: sleep 300
84+
- name: Setup Opensearch Dashboards
85+
run: |
86+
npm uninstall -g yarn
87+
echo "Installing yarn ${{ steps.tool-versions.outputs.yarn_version }}"
88+
npm i -g yarn@${{ steps.tool-versions.outputs.yarn_version }}
89+
yarn cache clean
90+
yarn add sha.js
91+
working-directory: OpenSearch-Dashboards
10292
shell: bash
10393

104-
- name: Checkout opensearch-dashboards-functional-test
94+
- name: Boodstrap Opensearch Dashboards
95+
run: |
96+
yarn osd bootstrap --single-version=loose
97+
working-directory: OpenSearch-Dashboards
98+
99+
- name: Run Opensearch Dashboards with Query Workbench Installed
100+
run: |
101+
nohup yarn start --no-base-path --no-watch | tee dashboard.log &
102+
working-directory: OpenSearch-Dashboards
103+
104+
- name : Check If OpenSearch Dashboards Is Ready
105+
if: ${{ runner.os == 'Linux' }}
106+
run: |
107+
if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
108+
echo "OpenSearch Dashboards compiled successfully."
109+
else
110+
echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
111+
exit 1
112+
fi
113+
working-directory: OpenSearch-Dashboards
114+
115+
- name: Checkout Dashboards Functioanl Test Repo
105116
uses: actions/checkout@v2
106117
with:
107118
path: opensearch-dashboards-functional-test
108119
repository: opensearch-project/opensearch-dashboards-functional-test
109-
ref: '${{ github.base_ref }}'
120+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
121+
fetch-depth: 0
122+
123+
- name: Install Cypress
124+
run: |
125+
npm install cypress --save-dev
126+
shell: bash
127+
working-directory: opensearch-dashboards-functional-test
110128

111129
- name: Get Cypress version
112130
id: cypress_version
113131
run: |
114-
echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')"
132+
echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')"
133+
working-directory: opensearch-dashboards-functional-test
115134

116-
- name: Cache Cypress
117-
id: cache-cypress
118-
uses: actions/cache@v1
135+
- name: Run Cypress tests
136+
run: |
137+
yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/anomaly-detection-dashboards-plugin/*.js'
138+
working-directory: opensearch-dashboards-functional-test
139+
140+
- name: Capture failure screenshots
141+
uses: actions/upload-artifact@v1
142+
if: failure()
119143
with:
120-
path: ${{ matrix.cypress_cache_folder }}
121-
key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
122-
env:
123-
CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
124-
- run: npx cypress cache list
125-
- run: npx cypress cache path
126-
127-
- name: Run AD cypress tests
128-
uses: cypress-io/github-action@v2
144+
name: cypress-screenshots-${{ matrix.os }}
145+
path: opensearch-dashboards-functional-test/cypress/screenshots
146+
147+
- name: Capture failure test video
148+
uses: actions/upload-artifact@v1
149+
if: failure()
129150
with:
130-
working-directory: opensearch-dashboards-functional-test
131-
command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
132-
env:
133-
CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}
151+
name: cypress-videos-${{ matrix.os }}
152+
path: opensearch-dashboards-functional-test/cypress/videos

package.json

+5-5
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@
44
"description": "OpenSearch Anomaly Detection Dashboards Plugin",
55
"main": "index.js",
66
"config": {
7-
"plugin_version": "3.0.0.0",
8-
"plugin_name": "anomalyDetectionDashboards",
9-
"plugin_zip_name": "anomaly-detection-dashboards"
7+
"id": "anomalyDetectionDashboards",
8+
"zip_name": "anomaly-detection-dashboards"
109
},
1110
"scripts": {
1211
"osd": "node ../../scripts/osd",
1312
"opensearch": "node ../../scripts/opensearch",
1413
"lint": "node ../../scripts/eslint .",
1514
"plugin-helpers": "node ../../scripts/plugin_helpers",
1615
"test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
17-
"build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
16+
"build": "yarn plugin-helpers build",
17+
"postbuild": "echo Renaming artifact to [$npm_package_config_zip_name-$npm_package_version.zip] && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip"
1818
},
1919
"lint-staged": {
2020
"*.{ts,tsx,js,jsx,json,css,md}": [
@@ -56,4 +56,4 @@
5656
"browserify-sign": "^4.2.2",
5757
"axios": "^1.6.1"
5858
}
59-
}
59+
}

public/pages/DetectorResults/containers/AnomalyResults.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
255255
endDate: adjustedCurrentTime.valueOf(),
256256
} as DateRange;
257257

258+
// build result search query params relative to data end time
258259
const params = buildParamsForGetAnomalyResultsWithDateRange(
259260
featureDataPointsRange.startDate,
260261
featureDataPointsRange.endDate

public/pages/utils/__tests__/anomalyResultUtils.test.ts

+57
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
getFeatureMissingDataAnnotations,
1414
getFeatureDataPointsForDetector,
1515
parsePureAnomalies,
16+
buildParamsForGetAnomalyResultsWithDateRange,
1617
} from '../anomalyResultUtils';
1718
import { getRandomDetector } from '../../../redux/reducers/__tests__/utils';
1819
import {
@@ -22,11 +23,16 @@ import {
2223
AnomalyData,
2324
} from '../../../models/interfaces';
2425
import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants';
26+
import { MAX_ANOMALIES } from '../../../utils/constants';
27+
import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants';
2528

2629
describe('anomalyResultUtils', () => {
2730
let randomDetector_20_min: Detector;
2831
let randomDetector_20_sec: Detector;
2932
let feature_id = 'deny_max';
33+
const startTime = 1609459200000; // January 1, 2021
34+
const endTime = 1609545600000; // January 2, 2021
35+
3036
beforeAll(() => {
3137
randomDetector_20_min = {
3238
...getRandomDetector(true),
@@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => {
569575
)
570576
).toEqual([]);
571577
});
578+
test('should correctly build parameters with default options', () => {
579+
const expected = {
580+
from: 0,
581+
size: MAX_ANOMALIES,
582+
sortDirection: SORT_DIRECTION.DESC,
583+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
584+
startTime: startTime,
585+
endTime: endTime,
586+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
587+
anomalyThreshold: -1,
588+
entityList: undefined, // Default as an empty array stringified
589+
};
590+
591+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime);
592+
expect(result).toEqual(expected);
593+
});
594+
595+
test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => {
596+
const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }];
597+
const expected = {
598+
from: 0,
599+
size: MAX_ANOMALIES,
600+
sortDirection: SORT_DIRECTION.DESC,
601+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
602+
startTime: startTime,
603+
endTime: endTime,
604+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
605+
anomalyThreshold: 0, // because anomalyOnly is true
606+
entityList: JSON.stringify(entities),
607+
};
608+
609+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities);
610+
expect(result).toEqual(expected);
611+
});
612+
613+
test('should handle undefined `entityList` as an empty array JSON string', () => {
614+
const expected = {
615+
from: 0,
616+
size: MAX_ANOMALIES,
617+
sortDirection: SORT_DIRECTION.DESC,
618+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
619+
startTime: startTime,
620+
endTime: endTime,
621+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
622+
anomalyThreshold: -1, // default as anomalyOnly is false
623+
entityList: undefined, // Default for undefined entityList
624+
};
625+
626+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined);
627+
expect(result).toEqual(expected);
628+
});
572629
});
573630

574631
describe('parsePureAnomalies()', () => {

0 commit comments

Comments
 (0)