Skip to content

Commit 334d526

Browse files
committed
build query parameters using data_end_time
This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing. The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead. This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis. Testing done: 1. reproduced the issue and verified the fix. 2. added unit tests. 3. Confirmed that single stream detector result views remain functional post-changes. Signed-off-by: Kaituo Li <kaituo@amazon.com>
1 parent 48acb93 commit 334d526

7 files changed

+210
-103
lines changed

.github/workflows/build-and-test-workflow.yml

+6
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ jobs:
8888
run: |
8989
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
9090
yarn osd bootstrap --single-version=loose
91+
- name: Set npm to use bash for shell
92+
if: ${{ matrix.os == 'windows-latest' }}
93+
run: |
94+
# Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json build script)
95+
# commands work as intended
96+
npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
9197
- name: Build the plugin
9298
run: |
9399
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,30 @@
1-
# Running AD integ tests stored in https://github.com/opensearch-project/opensearch-dashboards-functional-test
2-
# In the future we should pull dependencies from bundled build snapshots. Because that is not available
3-
# yet we build the cluster from source (besides core Opensearch, which is a pulled min artifact).
4-
name: Remote integ tests workflow
5-
on:
6-
push:
7-
branches:
8-
- "*"
9-
pull_request:
10-
branches:
11-
- "*"
1+
name: FTR E2E AD Workbench Test
2+
3+
on: [pull_request, push]
4+
5+
env:
6+
CI: 1
7+
# avoid warnings like "tput: No value for $TERM and no -T specified"
8+
TERM: xterm
9+
OPENSEARCH_DASHBOARDS_VERSION: 'main'
10+
OPENSEARCH_VERSION: '3.0.0'
11+
OPENSEARCH_PLUGIN_VERSION: '3.0.0.0'
12+
1213
jobs:
13-
test-without-security:
14-
name: Run integ tests without security
14+
tests:
15+
name: Run FTR E2E AD Workbench Tests
1516
strategy:
17+
fail-fast: false
1618
matrix:
17-
os: [ubuntu-latest, windows-latest]
18-
java: [11]
19-
include:
20-
- os: windows-latest
21-
cypress_cache_folder: ~/AppData/Local/Cypress/Cache
22-
- os: ubuntu-latest
23-
cypress_cache_folder: ~/.cache/Cypress
19+
os: [ ubuntu-latest ]
20+
jdk: [ 11 ]
2421
runs-on: ${{ matrix.os }}
25-
steps:
26-
- name: Set up Java 11
27-
uses: actions/setup-java@v3
28-
with:
29-
distribution: 'corretto'
30-
java-version: '11'
31-
32-
- name: Enable longer filenames
33-
if: ${{ matrix.os == 'windows-latest' }}
34-
run: git config --system core.longpaths true
35-
36-
- name: Checkout OpenSearch Dashboards
37-
uses: actions/checkout@v2
38-
with:
39-
repository: opensearch-project/OpenSearch-Dashboards
40-
ref: '${{ github.base_ref }}'
41-
path: OpenSearch-Dashboards
42-
43-
- name: Checkout Anomaly Detection OpenSearch Dashboards plugin
44-
uses: actions/checkout@v2
45-
with:
46-
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
4722

48-
- name: Setup Node
49-
uses: actions/setup-node@v3
23+
steps:
24+
- name: Set up JDK
25+
uses: actions/setup-java@v1
5026
with:
51-
node-version-file: './OpenSearch-Dashboards/.nvmrc'
52-
registry-url: 'https://registry.npmjs.org'
53-
54-
- name: Install Yarn
55-
# Need to use bash to avoid having a windows/linux specific step
56-
shell: bash
57-
run: |
58-
YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
59-
echo "Installing yarn@$YARN_VERSION"
60-
npm i -g yarn@$YARN_VERSION
61-
62-
- run: node -v
63-
- run: yarn -v
27+
java-version: ${{ matrix.jdk }}
6428

6529
- name: Checkout Anomaly-Detection
6630
uses: actions/checkout@v2
@@ -72,62 +36,109 @@ jobs:
7236
- name: Run OpenSearch with plugin
7337
run: |
7438
cd anomaly-detection
75-
CONFIG_PATH=../OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin/opensearch_dashboards.json
76-
OPENSEARCH_VERSION=$(node -p "require('$CONFIG_PATH').opensearchDashboardsVersion")-SNAPSHOT
77-
echo "Using OpenSearch version $OPENSEARCH_VERSION"
78-
./gradlew run -Dopensearch.version=$OPENSEARCH_VERSION &
39+
./gradlew run &
7940
timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
8041
shell: bash
42+
43+
- name: Check OpenSearch Running on Linux
44+
if: ${{ runner.os != 'Windows'}}
45+
run: curl http://localhost:9200/
46+
shell: bash
47+
48+
- name: Checkout OpenSearch Dashboards
49+
uses: actions/checkout@v2
50+
with:
51+
path: OpenSearch-Dashboards
52+
repository: opensearch-project/OpenSearch-Dashboards
53+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
54+
fetch-depth: 0
55+
filter: |
56+
cypress
57+
test
8158
82-
- name: Bootstrap the plugin
83-
run: |
84-
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
85-
yarn osd bootstrap --single-version=loose
59+
- name: Checkout AD in OpenSearch Dashboards Plugins Dir
60+
uses: actions/checkout@v2
61+
with:
62+
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
8663

87-
- name: Run OpenSearch Dashboards server
64+
- id: tool-versions
8865
run: |
89-
cd OpenSearch-Dashboards
90-
yarn start --no-base-path --no-watch &
66+
echo "node_version=$(cat .node-version)" >> $GITHUB_OUTPUT
67+
echo "yarn_version=$(jq -r '.engines.yarn' package.json)" >> $GITHUB_OUTPUT
68+
working-directory: OpenSearch-Dashboards
9169
shell: bash
9270

93-
# Window is slow so wait longer
94-
- name: Sleep until OSD server starts - windows
95-
if: ${{ matrix.os == 'windows-latest' }}
96-
run: Start-Sleep -s 400
97-
shell: powershell
71+
- uses: actions/setup-node@v1
72+
with:
73+
node-version: ${{ steps.tool-versions.outputs.node_version }}
74+
registry-url: 'https://registry.npmjs.org'
9875

99-
- name: Sleep until OSD server starts - non-windows
100-
if: ${{ matrix.os != 'windows-latest' }}
101-
run: sleep 300
76+
- name: Setup Opensearch Dashboards
77+
run: |
78+
npm uninstall -g yarn
79+
echo "Installing yarn ${{ steps.tool-versions.outputs.yarn_version }}"
80+
npm i -g yarn@${{ steps.tool-versions.outputs.yarn_version }}
81+
yarn cache clean
82+
yarn add sha.js
83+
working-directory: OpenSearch-Dashboards
10284
shell: bash
10385

104-
- name: Checkout opensearch-dashboards-functional-test
86+
- name: Boodstrap Opensearch Dashboards
87+
run: |
88+
yarn osd bootstrap --single-version=loose
89+
working-directory: OpenSearch-Dashboards
90+
91+
- name: Run Opensearch Dashboards with Query Workbench Installed
92+
run: |
93+
nohup yarn start --no-base-path --no-watch | tee dashboard.log &
94+
working-directory: OpenSearch-Dashboards
95+
96+
- name : Check If OpenSearch Dashboards Is Ready
97+
if: ${{ runner.os == 'Linux' }}
98+
run: |
99+
if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
100+
echo "OpenSearch Dashboards compiled successfully."
101+
else
102+
echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
103+
exit 1
104+
fi
105+
working-directory: OpenSearch-Dashboards
106+
107+
- name: Checkout Dashboards Functioanl Test Repo
105108
uses: actions/checkout@v2
106109
with:
107110
path: opensearch-dashboards-functional-test
108111
repository: opensearch-project/opensearch-dashboards-functional-test
109-
ref: '${{ github.base_ref }}'
112+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
113+
fetch-depth: 0
114+
115+
- name: Install Cypress
116+
run: |
117+
npm install cypress --save-dev
118+
shell: bash
119+
working-directory: opensearch-dashboards-functional-test
110120

111121
- name: Get Cypress version
112122
id: cypress_version
113123
run: |
114-
echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')"
124+
echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')"
125+
working-directory: opensearch-dashboards-functional-test
115126

116-
- name: Cache Cypress
117-
id: cache-cypress
118-
uses: actions/cache@v1
127+
- name: Run Cypress tests
128+
run: |
129+
yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/anomaly-detection-dashboards-plugin/*.js'
130+
working-directory: opensearch-dashboards-functional-test
131+
132+
- name: Capture failure screenshots
133+
uses: actions/upload-artifact@v1
134+
if: failure()
119135
with:
120-
path: ${{ matrix.cypress_cache_folder }}
121-
key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
122-
env:
123-
CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
124-
- run: npx cypress cache list
125-
- run: npx cypress cache path
126-
127-
- name: Run AD cypress tests
128-
uses: cypress-io/github-action@v2
136+
name: cypress-screenshots-${{ matrix.os }}
137+
path: opensearch-dashboards-functional-test/cypress/screenshots
138+
139+
- name: Capture failure test video
140+
uses: actions/upload-artifact@v1
141+
if: failure()
129142
with:
130-
working-directory: opensearch-dashboards-functional-test
131-
command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
132-
env:
133-
CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}
143+
name: cypress-videos-${{ matrix.os }}
144+
path: opensearch-dashboards-functional-test/cypress/videos

package.json

+5-5
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@
44
"description": "OpenSearch Anomaly Detection Dashboards Plugin",
55
"main": "index.js",
66
"config": {
7-
"plugin_version": "3.0.0.0",
8-
"plugin_name": "anomalyDetectionDashboards",
9-
"plugin_zip_name": "anomaly-detection-dashboards"
7+
"id": "anomalyDetectionDashboards",
8+
"zip_name": "anomaly-detection-dashboards"
109
},
1110
"scripts": {
1211
"osd": "node ../../scripts/osd",
1312
"opensearch": "node ../../scripts/opensearch",
1413
"lint": "node ../../scripts/eslint .",
1514
"plugin-helpers": "node ../../scripts/plugin_helpers",
1615
"test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
17-
"build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
16+
"build": "yarn plugin-helpers build",
17+
"postbuild": "echo Renaming artifact to [$npm_package_config_zip_name-$npm_package_version.zip] && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip"
1818
},
1919
"lint-staged": {
2020
"*.{ts,tsx,js,jsx,json,css,md}": [
@@ -56,4 +56,4 @@
5656
"browserify-sign": "^4.2.2",
5757
"axios": "^1.6.1"
5858
}
59-
}
59+
}

public/pages/DetectorResults/containers/AnomalyResults.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
255255
endDate: adjustedCurrentTime.valueOf(),
256256
} as DateRange;
257257

258+
// build result search query params relative to data end time
258259
const params = buildParamsForGetAnomalyResultsWithDateRange(
259260
featureDataPointsRange.startDate,
260261
featureDataPointsRange.endDate

public/pages/utils/__tests__/anomalyResultUtils.test.ts

+57
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
getFeatureMissingDataAnnotations,
1414
getFeatureDataPointsForDetector,
1515
parsePureAnomalies,
16+
buildParamsForGetAnomalyResultsWithDateRange,
1617
} from '../anomalyResultUtils';
1718
import { getRandomDetector } from '../../../redux/reducers/__tests__/utils';
1819
import {
@@ -22,11 +23,16 @@ import {
2223
AnomalyData,
2324
} from '../../../models/interfaces';
2425
import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants';
26+
import { MAX_ANOMALIES } from '../../../utils/constants';
27+
import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants';
2528

2629
describe('anomalyResultUtils', () => {
2730
let randomDetector_20_min: Detector;
2831
let randomDetector_20_sec: Detector;
2932
let feature_id = 'deny_max';
33+
const startTime = 1609459200000; // January 1, 2021
34+
const endTime = 1609545600000; // January 2, 2021
35+
3036
beforeAll(() => {
3137
randomDetector_20_min = {
3238
...getRandomDetector(true),
@@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => {
569575
)
570576
).toEqual([]);
571577
});
578+
test('should correctly build parameters with default options', () => {
579+
const expected = {
580+
from: 0,
581+
size: MAX_ANOMALIES,
582+
sortDirection: SORT_DIRECTION.DESC,
583+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
584+
startTime: startTime,
585+
endTime: endTime,
586+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
587+
anomalyThreshold: -1,
588+
entityList: undefined, // Default as an empty array stringified
589+
};
590+
591+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime);
592+
expect(result).toEqual(expected);
593+
});
594+
595+
test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => {
596+
const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }];
597+
const expected = {
598+
from: 0,
599+
size: MAX_ANOMALIES,
600+
sortDirection: SORT_DIRECTION.DESC,
601+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
602+
startTime: startTime,
603+
endTime: endTime,
604+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
605+
anomalyThreshold: 0, // because anomalyOnly is true
606+
entityList: JSON.stringify(entities),
607+
};
608+
609+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities);
610+
expect(result).toEqual(expected);
611+
});
612+
613+
test('should handle undefined `entityList` as an empty array JSON string', () => {
614+
const expected = {
615+
from: 0,
616+
size: MAX_ANOMALIES,
617+
sortDirection: SORT_DIRECTION.DESC,
618+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
619+
startTime: startTime,
620+
endTime: endTime,
621+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
622+
anomalyThreshold: -1, // default as anomalyOnly is false
623+
entityList: undefined, // Default for undefined entityList
624+
};
625+
626+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined);
627+
expect(result).toEqual(expected);
628+
});
572629
});
573630

574631
describe('parsePureAnomalies()', () => {

0 commit comments

Comments
 (0)