Skip to content

Commit 0a748c7

Browse files
kaituokaituo
authored
build query parameters using data_end_time (#741)
This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing. The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead. This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis. Testing done: 1. reproduced the issue and verified the fix. 2. added unit tests. 3. Confirmed that single stream detector result views remain functional post-changes. Signed-off-by: Kaituo Li <kaituo@amazon.com>
1 parent 0335282 commit 0a748c7

7 files changed

+239
-99
lines changed

‎.github/workflows/build-and-test-workflow.yml

+6
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ jobs:
8888
run: |
8989
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
9090
yarn osd bootstrap --single-version=loose
91+
- name: Set npm to use bash for shell
92+
if: ${{ matrix.os == 'windows-latest' }}
93+
run: |
94+
# Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json postbuild script)
95+
# commands work as intended
96+
npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
9197
- name: Build the plugin
9298
run: |
9399
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin

‎.github/workflows/remote-integ-tests-workflow.yml

+135-91
Original file line numberDiff line numberDiff line change
@@ -1,133 +1,177 @@
1-
# Running AD integ tests stored in https://github.com/opensearch-project/opensearch-dashboards-functional-test
2-
# In the future we should pull dependencies from bundled build snapshots. Because that is not available
3-
# yet we build the cluster from source (besides core Opensearch, which is a pulled min artifact).
4-
name: Remote integ tests workflow
5-
on:
6-
push:
7-
branches:
8-
- "*"
9-
pull_request:
10-
branches:
11-
- "*"
1+
name: FTR E2E AD Workbench Test
2+
3+
on: [pull_request, push]
4+
5+
env:
6+
CI: 1
7+
# avoid warnings like "tput: No value for $TERM and no -T specified"
8+
TERM: xterm
9+
OPENSEARCH_DASHBOARDS_VERSION: 'main'
10+
OPENSEARCH_VERSION: '3.0.0'
11+
OPENSEARCH_PLUGIN_VERSION: '3.0.0.0'
12+
1213
jobs:
13-
test-without-security:
14-
name: Run integ tests without security
14+
tests:
15+
name: Run FTR E2E AD Workbench Tests
1516
strategy:
17+
fail-fast: false
1618
matrix:
17-
os: [ubuntu-latest, windows-latest]
18-
java: [11]
19-
include:
20-
- os: windows-latest
21-
cypress_cache_folder: ~/AppData/Local/Cypress/Cache
22-
- os: ubuntu-latest
23-
cypress_cache_folder: ~/.cache/Cypress
19+
os: [ ubuntu-latest ]
20+
jdk: [ 11 ]
2421
runs-on: ${{ matrix.os }}
22+
2523
steps:
26-
- name: Set up Java 11
27-
uses: actions/setup-java@v3
24+
- name: Set up JDK
25+
uses: actions/setup-java@v1
2826
with:
29-
distribution: 'corretto'
30-
java-version: '11'
31-
32-
- name: Enable longer filenames
33-
if: ${{ matrix.os == 'windows-latest' }}
34-
run: git config --system core.longpaths true
27+
java-version: ${{ matrix.jdk }}
3528

36-
- name: Checkout OpenSearch Dashboards
29+
- name: Checkout Anomaly-Detection
3730
uses: actions/checkout@v2
3831
with:
39-
repository: opensearch-project/OpenSearch-Dashboards
32+
path: anomaly-detection
33+
repository: opensearch-project/anomaly-detection
4034
ref: '${{ github.base_ref }}'
41-
path: OpenSearch-Dashboards
4235

43-
- name: Checkout Anomaly Detection OpenSearch Dashboards plugin
36+
- name: Run OpenSearch with plugin
37+
run: |
38+
cd anomaly-detection
39+
./gradlew run &
40+
timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
41+
shell: bash
42+
43+
- name: Check OpenSearch Running on Linux
44+
if: ${{ runner.os != 'Windows'}}
45+
run: curl http://localhost:9200/
46+
shell: bash
47+
48+
- name: Checkout OpenSearch Dashboards
4449
uses: actions/checkout@v2
4550
with:
46-
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
51+
path: OpenSearch-Dashboards
52+
repository: opensearch-project/OpenSearch-Dashboards
53+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
54+
fetch-depth: 0
55+
filter: |
56+
cypress
57+
test
4758
48-
- name: Setup Node
49-
uses: actions/setup-node@v3
59+
- name: Checkout AD in OpenSearch Dashboards Plugins Dir
60+
uses: actions/checkout@v2
5061
with:
51-
node-version-file: './OpenSearch-Dashboards/.nvmrc'
52-
registry-url: 'https://registry.npmjs.org'
62+
path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
5363

54-
- name: Install Yarn
55-
# Need to use bash to avoid having a windows/linux specific step
56-
shell: bash
64+
- id: tool-versions
5765
run: |
58-
YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
59-
echo "Installing yarn@$YARN_VERSION"
60-
npm i -g yarn@$YARN_VERSION
61-
62-
- run: node -v
63-
- run: yarn -v
66+
echo "node_version=$(cat .node-version)" >> $GITHUB_OUTPUT
67+
echo "yarn_version=$(jq -r '.engines.yarn' package.json)" >> $GITHUB_OUTPUT
68+
working-directory: OpenSearch-Dashboards
69+
shell: bash
6470

65-
- name: Checkout Anomaly-Detection
66-
uses: actions/checkout@v2
71+
- uses: actions/setup-node@v1
6772
with:
68-
path: anomaly-detection
69-
repository: opensearch-project/anomaly-detection
70-
ref: '${{ github.base_ref }}'
73+
node-version: ${{ steps.tool-versions.outputs.node_version }}
74+
registry-url: 'https://registry.npmjs.org'
7175

72-
- name: Run OpenSearch with plugin
76+
- name: Setup Opensearch Dashboards
7377
run: |
74-
cd anomaly-detection
75-
CONFIG_PATH=../OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin/opensearch_dashboards.json
76-
OPENSEARCH_VERSION=$(node -p "require('$CONFIG_PATH').opensearchDashboardsVersion")-SNAPSHOT
77-
echo "Using OpenSearch version $OPENSEARCH_VERSION"
78-
./gradlew run -Dopensearch.version=$OPENSEARCH_VERSION &
79-
timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
78+
npm uninstall -g yarn
79+
echo "Installing yarn ${{ steps.tool-versions.outputs.yarn_version }}"
80+
npm i -g yarn@${{ steps.tool-versions.outputs.yarn_version }}
81+
yarn cache clean
82+
yarn add sha.js
83+
working-directory: OpenSearch-Dashboards
8084
shell: bash
8185

82-
- name: Bootstrap the plugin
86+
- name: Boodstrap Opensearch Dashboards
8387
run: |
84-
cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
8588
yarn osd bootstrap --single-version=loose
89+
working-directory: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
90+
91+
- name: Run Opensearch Dashboards with AD Installed
92+
run: |
93+
nohup yarn start --no-base-path --no-watch --server.host="0.0.0.0" | tee dashboard.log &
94+
working-directory: OpenSearch-Dashboards
8695

87-
- name: Run OpenSearch Dashboards server
96+
- name : Check If OpenSearch Dashboards Is Ready
97+
if: ${{ runner.os == 'Linux' }}
8898
run: |
89-
cd OpenSearch-Dashboards
90-
yarn start --no-base-path --no-watch &
99+
if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
100+
echo "OpenSearch Dashboards compiled successfully."
101+
else
102+
echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
103+
exit 1
104+
fi
105+
working-directory: OpenSearch-Dashboards
106+
107+
- name: Show OpenSearch Dashboards Logs
108+
if: always()
109+
run: cat dashboard.log
110+
working-directory: OpenSearch-Dashboards
111+
112+
- name: Health check
113+
run: |
114+
timeout 600 bash -c 'while [[ "$(curl -k http://localhost:5601/api/status | jq -r '.status.overall.state')" != "green" ]]; do sleep 5; done'
91115
shell: bash
92116

93-
# Window is slow so wait longer
94-
- name: Sleep until OSD server starts - windows
95-
if: ${{ matrix.os == 'windows-latest' }}
96-
run: Start-Sleep -s 400
97-
shell: powershell
98-
99-
- name: Sleep until OSD server starts - non-windows
100-
if: ${{ matrix.os != 'windows-latest' }}
101-
run: sleep 300
117+
- name: Check OpenSearch Dashboards Running on Linux
118+
if: ${{ runner.os != 'Windows'}}
119+
run: curl http://localhost:5601/api/status
102120
shell: bash
103121

104-
- name: Checkout opensearch-dashboards-functional-test
122+
- name: Checkout Dashboards Functional Test Repo
105123
uses: actions/checkout@v2
106124
with:
107125
path: opensearch-dashboards-functional-test
108126
repository: opensearch-project/opensearch-dashboards-functional-test
109-
ref: '${{ github.base_ref }}'
127+
ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
128+
fetch-depth: 0
129+
130+
- name: Install Cypress
131+
run: |
132+
npm install cypress --save-dev
133+
shell: bash
134+
working-directory: opensearch-dashboards-functional-test
110135

111136
- name: Get Cypress version
112137
id: cypress_version
113138
run: |
114-
echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')"
139+
echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')"
140+
working-directory: opensearch-dashboards-functional-test
115141

116-
- name: Cache Cypress
117-
id: cache-cypress
118-
uses: actions/cache@v1
142+
- name: Finding spec files and store to output
143+
id: finding-files
144+
run: |
145+
echo "::set-output name=FILELIST::$(find cypress/integration/plugins/anomaly-detection-dashboards-plugin -name '*.js' -print)"
146+
working-directory: opensearch-dashboards-functional-test
147+
148+
- name: Print spec files from output
149+
run: |
150+
IFS="," read -a myarray <<< ${{ steps.finding-files.outputs.FILELIST }}
151+
for i in "${myarray[@]}"; do
152+
echo "${i}"
153+
done
154+
working-directory: opensearch-dashboards-functional-test
155+
156+
- name: Run spec files from output
157+
run: |
158+
IFS="," read -a myarray <<< ${{ steps.finding-files.outputs.FILELIST }}
159+
for i in "${myarray[@]}"; do
160+
yarn cypress:run-without-security --browser electron --spec "${i}"
161+
sleep 60
162+
done
163+
working-directory: opensearch-dashboards-functional-test
164+
165+
- name: Capture failure screenshots
166+
uses: actions/upload-artifact@v1
167+
if: failure()
119168
with:
120-
path: ${{ matrix.cypress_cache_folder }}
121-
key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
122-
env:
123-
CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
124-
- run: npx cypress cache list
125-
- run: npx cypress cache path
126-
127-
- name: Run AD cypress tests
128-
uses: cypress-io/github-action@v2
169+
name: cypress-screenshots-${{ matrix.os }}
170+
path: opensearch-dashboards-functional-test/cypress/screenshots
171+
172+
- name: Capture failure test video
173+
uses: actions/upload-artifact@v1
174+
if: failure()
129175
with:
130-
working-directory: opensearch-dashboards-functional-test
131-
command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
132-
env:
133-
CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}
176+
name: cypress-videos-${{ matrix.os }}
177+
path: opensearch-dashboards-functional-test/cypress/videos

‎package.json

+5-5
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@
44
"description": "OpenSearch Anomaly Detection Dashboards Plugin",
55
"main": "index.js",
66
"config": {
7-
"plugin_version": "3.0.0.0",
8-
"plugin_name": "anomalyDetectionDashboards",
9-
"plugin_zip_name": "anomaly-detection-dashboards"
7+
"id": "anomalyDetectionDashboards",
8+
"zip_name": "anomaly-detection-dashboards"
109
},
1110
"scripts": {
1211
"osd": "node ../../scripts/osd",
1312
"opensearch": "node ../../scripts/opensearch",
1413
"lint": "node ../../scripts/eslint .",
1514
"plugin-helpers": "node ../../scripts/plugin_helpers",
1615
"test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
17-
"build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
16+
"build": "yarn plugin-helpers build",
17+
"postbuild": "echo Renaming artifact to [$npm_package_config_zip_name-$npm_package_version.zip] && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip"
1818
},
1919
"lint-staged": {
2020
"*.{ts,tsx,js,jsx,json,css,md}": [
@@ -56,4 +56,4 @@
5656
"browserify-sign": "^4.2.2",
5757
"axios": "^1.6.1"
5858
}
59-
}
59+
}

‎public/pages/DetectorResults/containers/AnomalyResults.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
255255
endDate: adjustedCurrentTime.valueOf(),
256256
} as DateRange;
257257

258+
// build result search query params relative to data end time
258259
const params = buildParamsForGetAnomalyResultsWithDateRange(
259260
featureDataPointsRange.startDate,
260261
featureDataPointsRange.endDate

0 commit comments

Comments
 (0)