Skip to content

Commit e131a10

Browse files
committed
build query parameters using data_end_time
This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing. The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead. This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis. Testing done: 1. reproduced the issue and verified the fix. 2. added unit tests. 3. Confirmed that single stream detector result views remain functional post-changes. Signed-off-by: Kaituo Li <kaituo@amazon.com>
1 parent 48acb93 commit e131a10

7 files changed

+213
-101
lines changed

.github/workflows/build-and-test-workflow.yml

+6
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ jobs:
8888
run: |
8989
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
9090
yarn osd bootstrap --single-version=loose
91+
- name: Set npm to use bash for shell
92+
if: ${{ matrix.os == 'windows-latest' }}
93+
run: |
94+
# Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json build script)
95+
# commands work as intended
96+
npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
9197
- name: Build the plugin
9298
run: |
9399
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
Original file line numberDiff line numberDiff line change
@@ -1,133 +1,149 @@
1-
# Running AD integ tests stored in https://github.com/opensearch-project/opensearch-dashboards-functional-test
2-
# In the future we should pull dependencies from bundled build snapshots. Because that is not available
3-
# yet we build the cluster from source (besides core Opensearch, which is a pulled min artifact).
4-
name: Remote integ tests workflow
5-
on:
6-
push:
7-
branches:
8-
- "*"
9-
pull_request:
10-
branches:
11-
- "*"
1+
name: FTR E2E AD Workbench Test
2+
3+
on: [pull_request, push]
4+
5+
env:
6+
CI: 1
7+
# avoid warnings like "tput: No value for $TERM and no -T specified"
8+
TERM: xterm
9+
OPENSEARCH_DASHBOARDS_VERSION: 'main'
10+
OPENSEARCH_VERSION: '3.0.0'
11+
OPENSEARCH_PLUGIN_VERSION: '3.0.0.0'
12+
1213
jobs:
13-
test-without-security:
14-
name: Run integ tests without security
14+
tests:
15+
name: Run FTR E2E AD Workbench Tests
1516
strategy:
17+
fail-fast: false
1618
matrix:
17-
os: [ubuntu-latest, windows-latest]
18-
java: [11]
19-
include:
20-
- os: windows-latest
21-
cypress_cache_folder: ~/AppData/Local/Cypress/Cache
22-
- os: ubuntu-latest
23-
cypress_cache_folder: ~/.cache/Cypress
19+
os: [ ubuntu-latest ]
20+
jdk: [ 11 ]
2421
runs-on: ${{ matrix.os }}
22+
2523
steps:
26-
- name: Set up Java 11
27-
uses: actions/setup-java@v3
24+
- name: Set up JDK
25+
uses: actions/setup-java@v1
2826
with:
29-
distribution: 'corretto'
30-
java-version: '11'
31-
32-
- name: Enable longer filenames
33-
if: ${{ matrix.os == 'windows-latest' }}
34-
run: git config --system core.longpaths true
27+
java-version: ${{ matrix.jdk }}
3528

36-
- name: Checkout OpenSearch Dashboards
29+
- name: Checkout Anomaly-Detection
3730
uses: actions/checkout@v2
3831
with:
39-
repository: opensearch-project/OpenSearch-Dashboards
32+
path: anomaly-detection
33+
repository: opensearch-project/anomaly-detection
4034
ref: '${{ github.base_ref }}'
41-
path: OpenSearch-Dashboards
4235

43-
- name: Checkout Anomaly Detection OpenSearch Dashboards plugin
36+
- name: Run OpenSearch with plugin
37+
run: |
38+
cd anomaly-detection
39+
./gradlew run &
40+
timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
41+
shell: bash
42+
43+
- name: Check OpenSearch Running on Linux
44+
if: ${{ runner.os != 'Windows'}}
45+
run: curl http://localhost:9200/
46+
shell: bash
47+
48+
- name: Checkout OpenSearch Dashboards
4449
uses: actions/checkout@v2
4550
with:
46-
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
51+
path: OpenSearch-Dashboards
52+
repository: opensearch-project/OpenSearch-Dashboards
53+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
54+
fetch-depth: 0
55+
filter: |
56+
cypress
57+
test
4758
48-
- name: Setup Node
49-
uses: actions/setup-node@v3
59+
- name: Checkout AD in OpenSearch Dashboards Plugins Dir
60+
uses: actions/checkout@v2
5061
with:
51-
node-version-file: './OpenSearch-Dashboards/.nvmrc'
52-
registry-url: 'https://registry.npmjs.org'
62+
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
5363

54-
- name: Install Yarn
55-
# Need to use bash to avoid having a windows/linux specific step
56-
shell: bash
64+
- id: tool-versions
5765
run: |
58-
YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
59-
echo "Installing yarn@$YARN_VERSION"
60-
npm i -g yarn@$YARN_VERSION
61-
62-
- run: node -v
63-
- run: yarn -v
66+
echo "node_version=$(cat .node-version)" >> $GITHUB_OUTPUT
67+
echo "yarn_version=$(jq -r '.engines.yarn' package.json)" >> $GITHUB_OUTPUT
68+
working-directory: OpenSearch-Dashboards
69+
shell: bash
6470

65-
- name: Checkout Anomaly-Detection
66-
uses: actions/checkout@v2
71+
- uses: actions/setup-node@v1
6772
with:
68-
path: anomaly-detection
69-
repository: opensearch-project/anomaly-detection
70-
ref: '${{ github.base_ref }}'
73+
node-version: ${{ steps.tool-versions.outputs.node_version }}
74+
registry-url: 'https://registry.npmjs.org'
7175

72-
- name: Run OpenSearch with plugin
76+
- name: Setup Opensearch Dashboards
7377
run: |
74-
cd anomaly-detection
75-
CONFIG_PATH=../OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin/opensearch_dashboards.json
76-
OPENSEARCH_VERSION=$(node -p "require('$CONFIG_PATH').opensearchDashboardsVersion")-SNAPSHOT
77-
echo "Using OpenSearch version $OPENSEARCH_VERSION"
78-
./gradlew run -Dopensearch.version=$OPENSEARCH_VERSION &
79-
timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
78+
npm uninstall -g yarn
79+
echo "Installing yarn ${{ steps.tool-versions.outputs.yarn_version }}"
80+
npm i -g yarn@${{ steps.tool-versions.outputs.yarn_version }}
81+
yarn cache clean
82+
yarn add sha.js
83+
working-directory: OpenSearch-Dashboards
8084
shell: bash
8185

82-
- name: Bootstrap the plugin
86+
- name: Boodstrap Opensearch Dashboards
8387
run: |
84-
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
8588
yarn osd bootstrap --single-version=loose
86-
87-
- name: Run OpenSearch Dashboards server
89+
working-directory: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
90+
91+
- name: Run Opensearch Dashboards with AD Installed
8892
run: |
89-
cd OpenSearch-Dashboards
90-
yarn start --no-base-path --no-watch &
91-
shell: bash
92-
93-
# Window is slow so wait longer
94-
- name: Sleep until OSD server starts - windows
95-
if: ${{ matrix.os == 'windows-latest' }}
96-
run: Start-Sleep -s 400
97-
shell: powershell
93+
nohup yarn start --no-base-path --no-watch | tee dashboard.log &
94+
working-directory: OpenSearch-Dashboards
9895

99-
- name: Sleep until OSD server starts - non-windows
100-
if: ${{ matrix.os != 'windows-latest' }}
101-
run: sleep 300
96+
- name : Check If OpenSearch Dashboards Is Ready
97+
if: ${{ runner.os == 'Linux' }}
98+
run: |
99+
if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
100+
echo "OpenSearch Dashboards compiled successfully."
101+
else
102+
echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
103+
exit 1
104+
fi
105+
working-directory: OpenSearch-Dashboards
106+
107+
- name: Check OpenSearch Dashboards Running on Linux
108+
if: ${{ runner.os != 'Windows'}}
109+
run: curl http://localhost:5601/
102110
shell: bash
103111

104-
- name: Checkout opensearch-dashboards-functional-test
112+
- name: Checkout Dashboards Functioanl Test Repo
105113
uses: actions/checkout@v2
106114
with:
107115
path: opensearch-dashboards-functional-test
108116
repository: opensearch-project/opensearch-dashboards-functional-test
109-
ref: '${{ github.base_ref }}'
117+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
118+
fetch-depth: 0
119+
120+
- name: Install Cypress
121+
run: |
122+
npm install cypress --save-dev
123+
shell: bash
124+
working-directory: opensearch-dashboards-functional-test
110125

111126
- name: Get Cypress version
112127
id: cypress_version
113128
run: |
114-
echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')"
129+
echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')"
130+
working-directory: opensearch-dashboards-functional-test
115131

116-
- name: Cache Cypress
117-
id: cache-cypress
118-
uses: actions/cache@v1
132+
- name: Run Cypress tests
133+
run: |
134+
yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/anomaly-detection-dashboards-plugin/*.js'
135+
working-directory: opensearch-dashboards-functional-test
136+
137+
- name: Capture failure screenshots
138+
uses: actions/upload-artifact@v1
139+
if: failure()
119140
with:
120-
path: ${{ matrix.cypress_cache_folder }}
121-
key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
122-
env:
123-
CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
124-
- run: npx cypress cache list
125-
- run: npx cypress cache path
126-
127-
- name: Run AD cypress tests
128-
uses: cypress-io/github-action@v2
141+
name: cypress-screenshots-${{ matrix.os }}
142+
path: opensearch-dashboards-functional-test/cypress/screenshots
143+
144+
- name: Capture failure test video
145+
uses: actions/upload-artifact@v1
146+
if: failure()
129147
with:
130-
working-directory: opensearch-dashboards-functional-test
131-
command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
132-
env:
133-
CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}
148+
name: cypress-videos-${{ matrix.os }}
149+
path: opensearch-dashboards-functional-test/cypress/videos

package.json

+5-5
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@
44
"description": "OpenSearch Anomaly Detection Dashboards Plugin",
55
"main": "index.js",
66
"config": {
7-
"plugin_version": "3.0.0.0",
8-
"plugin_name": "anomalyDetectionDashboards",
9-
"plugin_zip_name": "anomaly-detection-dashboards"
7+
"id": "anomalyDetectionDashboards",
8+
"zip_name": "anomaly-detection-dashboards"
109
},
1110
"scripts": {
1211
"osd": "node ../../scripts/osd",
1312
"opensearch": "node ../../scripts/opensearch",
1413
"lint": "node ../../scripts/eslint .",
1514
"plugin-helpers": "node ../../scripts/plugin_helpers",
1615
"test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
17-
"build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
16+
"build": "yarn plugin-helpers build",
17+
"postbuild": "echo Renaming artifact to [$npm_package_config_zip_name-$npm_package_version.zip] && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip"
1818
},
1919
"lint-staged": {
2020
"*.{ts,tsx,js,jsx,json,css,md}": [
@@ -56,4 +56,4 @@
5656
"browserify-sign": "^4.2.2",
5757
"axios": "^1.6.1"
5858
}
59-
}
59+
}

public/pages/DetectorResults/containers/AnomalyResults.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
255255
endDate: adjustedCurrentTime.valueOf(),
256256
} as DateRange;
257257

258+
// build result search query params relative to data end time
258259
const params = buildParamsForGetAnomalyResultsWithDateRange(
259260
featureDataPointsRange.startDate,
260261
featureDataPointsRange.endDate

public/pages/utils/__tests__/anomalyResultUtils.test.ts

+57
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
getFeatureMissingDataAnnotations,
1414
getFeatureDataPointsForDetector,
1515
parsePureAnomalies,
16+
buildParamsForGetAnomalyResultsWithDateRange,
1617
} from '../anomalyResultUtils';
1718
import { getRandomDetector } from '../../../redux/reducers/__tests__/utils';
1819
import {
@@ -22,11 +23,16 @@ import {
2223
AnomalyData,
2324
} from '../../../models/interfaces';
2425
import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants';
26+
import { MAX_ANOMALIES } from '../../../utils/constants';
27+
import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants';
2528

2629
describe('anomalyResultUtils', () => {
2730
let randomDetector_20_min: Detector;
2831
let randomDetector_20_sec: Detector;
2932
let feature_id = 'deny_max';
33+
const startTime = 1609459200000; // January 1, 2021
34+
const endTime = 1609545600000; // January 2, 2021
35+
3036
beforeAll(() => {
3137
randomDetector_20_min = {
3238
...getRandomDetector(true),
@@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => {
569575
)
570576
).toEqual([]);
571577
});
578+
test('should correctly build parameters with default options', () => {
579+
const expected = {
580+
from: 0,
581+
size: MAX_ANOMALIES,
582+
sortDirection: SORT_DIRECTION.DESC,
583+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
584+
startTime: startTime,
585+
endTime: endTime,
586+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
587+
anomalyThreshold: -1,
588+
entityList: undefined, // Default as an empty array stringified
589+
};
590+
591+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime);
592+
expect(result).toEqual(expected);
593+
});
594+
595+
test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => {
596+
const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }];
597+
const expected = {
598+
from: 0,
599+
size: MAX_ANOMALIES,
600+
sortDirection: SORT_DIRECTION.DESC,
601+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
602+
startTime: startTime,
603+
endTime: endTime,
604+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
605+
anomalyThreshold: 0, // because anomalyOnly is true
606+
entityList: JSON.stringify(entities),
607+
};
608+
609+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities);
610+
expect(result).toEqual(expected);
611+
});
612+
613+
test('should handle undefined `entityList` as an empty array JSON string', () => {
614+
const expected = {
615+
from: 0,
616+
size: MAX_ANOMALIES,
617+
sortDirection: SORT_DIRECTION.DESC,
618+
sortField: AD_DOC_FIELDS.DATA_END_TIME,
619+
startTime: startTime,
620+
endTime: endTime,
621+
fieldName: AD_DOC_FIELDS.DATA_END_TIME,
622+
anomalyThreshold: -1, // default as anomalyOnly is false
623+
entityList: undefined, // Default for undefined entityList
624+
};
625+
626+
const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined);
627+
expect(result).toEqual(expected);
628+
});
572629
});
573630

574631
describe('parsePureAnomalies()', () => {

0 commit comments

Comments
 (0)