build query parameters using data_end_time

kaituo · kaituo · commit e131a10ba5ce · 2024-04-29T19:00:34.000-07:00
This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing.

The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead.

This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis.

Testing done:
1. reproduced the issue and verified the fix.
2. added unit tests.
3. Confirmed that single stream detector result views remain functional post-changes.

Signed-off-by: Kaituo Li &lt;kaituo@amazon.com&gt;
diff --git a/.github/workflows/build-and-test-workflow.yml b/.github/workflows/build-and-test-workflow.yml
@@ -88,6 +88,12 @@ jobs:
         run: |
           cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
           yarn osd bootstrap --single-version=loose
+      - name: Set npm to use bash for shell
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          # Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json build script)
+          # commands work as intended
+          npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
       - name: Build the plugin
         run: |
           cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
diff --git a/.github/workflows/remote-integ-tests-workflow.yml b/.github/workflows/remote-integ-tests-workflow.yml
@@ -1,133 +1,149 @@
-# Running AD integ tests stored in https://github.com/opensearch-project/opensearch-dashboards-functional-test
-# In the future we should pull dependencies from bundled build snapshots. Because that is not available
-# yet we build the cluster from source (besides core Opensearch, which is a pulled min artifact).
-name: Remote integ tests workflow
-on:
-  push:
-    branches:
-      - "*"
-  pull_request:
-    branches:
-      - "*"
+name: FTR E2E AD Workbench Test
+
+on: [pull_request, push]
+
+env:
+  CI: 1
+  # avoid warnings like "tput: No value for $TERM and no -T specified"
+  TERM: xterm
+  OPENSEARCH_DASHBOARDS_VERSION: 'main'
+  OPENSEARCH_VERSION: '3.0.0'
+  OPENSEARCH_PLUGIN_VERSION: '3.0.0.0'
+
 jobs:
-  test-without-security:
-    name: Run integ tests without security
+  tests:
+    name: Run FTR E2E AD Workbench Tests
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest]
-        java: [11]
-        include:
-          - os: windows-latest
-            cypress_cache_folder: ~/AppData/Local/Cypress/Cache
-          - os: ubuntu-latest
-            cypress_cache_folder: ~/.cache/Cypress
+        os: [ ubuntu-latest ]
+        jdk: [ 11 ]
     runs-on: ${{ matrix.os }}
+
     steps:
-      - name: Set up Java 11
-        uses: actions/setup-java@v3
+      - name: Set up JDK
+        uses: actions/setup-java@v1
         with:
-          distribution: 'corretto'
-          java-version: '11'
-
-      - name: Enable longer filenames
-        if: ${{ matrix.os == 'windows-latest' }}
-        run: git config --system core.longpaths true
+          java-version: ${{ matrix.jdk }}
 
-      - name: Checkout OpenSearch Dashboards
+      - name: Checkout Anomaly-Detection
         uses: actions/checkout@v2
         with:
-          repository: opensearch-project/OpenSearch-Dashboards
+          path: anomaly-detection
+          repository: opensearch-project/anomaly-detection
           ref: '${{ github.base_ref }}'
-          path: OpenSearch-Dashboards
 
-      - name: Checkout Anomaly Detection OpenSearch Dashboards plugin
+      - name: Run OpenSearch with plugin
+        run: |
+          cd anomaly-detection
+          ./gradlew run &
+          timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
+        shell: bash
+    
+      - name: Check OpenSearch Running on Linux
+        if: ${{ runner.os != 'Windows'}}
+        run: curl http://localhost:9200/
+        shell: bash
+      
+      - name: Checkout OpenSearch Dashboards
         uses: actions/checkout@v2
         with:
-          path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+          path: OpenSearch-Dashboards
+          repository: opensearch-project/OpenSearch-Dashboards
+          ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
+          fetch-depth: 0
+          filter: |
+            cypress
+            test
 
-      - name: Setup Node
-        uses: actions/setup-node@v3
+      - name: Checkout AD in OpenSearch Dashboards Plugins Dir
+        uses: actions/checkout@v2
         with:
-          node-version-file: './OpenSearch-Dashboards/.nvmrc'
-          registry-url: 'https://registry.npmjs.org'
+            path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
 
-      - name: Install Yarn
-        # Need to use bash to avoid having a windows/linux specific step
-        shell: bash
+      - id: tool-versions
         run: |
-          YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
-          echo "Installing yarn@$YARN_VERSION"
-          npm i -g yarn@$YARN_VERSION
-
-      - run: node -v
-      - run: yarn -v
+          echo "node_version=$(cat .node-version)" >> $GITHUB_OUTPUT
+          echo "yarn_version=$(jq -r '.engines.yarn' package.json)" >> $GITHUB_OUTPUT
+        working-directory: OpenSearch-Dashboards
+        shell: bash
 
-      - name: Checkout Anomaly-Detection
-        uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
         with:
-          path: anomaly-detection
-          repository: opensearch-project/anomaly-detection
-          ref: '${{ github.base_ref }}'
+          node-version: ${{ steps.tool-versions.outputs.node_version }}
+          registry-url: 'https://registry.npmjs.org'
 
-      - name: Run OpenSearch with plugin
+      - name: Setup Opensearch Dashboards
         run: |
-          cd anomaly-detection
-          CONFIG_PATH=../OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin/opensearch_dashboards.json
-          OPENSEARCH_VERSION=$(node -p "require('$CONFIG_PATH').opensearchDashboardsVersion")-SNAPSHOT
-          echo "Using OpenSearch version $OPENSEARCH_VERSION"
-          ./gradlew run -Dopensearch.version=$OPENSEARCH_VERSION &
-          timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
+          npm uninstall -g yarn
+          echo "Installing yarn ${{ steps.tool-versions.outputs.yarn_version }}"
+          npm i -g yarn@${{ steps.tool-versions.outputs.yarn_version }}
+          yarn cache clean
+          yarn add sha.js
+        working-directory: OpenSearch-Dashboards
         shell: bash
 
-      - name: Bootstrap the plugin
+      - name: Boodstrap Opensearch Dashboards
         run: |
-          cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
           yarn osd bootstrap --single-version=loose
-
-      - name: Run OpenSearch Dashboards server
+        working-directory: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+      
+      - name: Run Opensearch Dashboards with AD Installed
         run: |
-          cd OpenSearch-Dashboards
-          yarn start --no-base-path --no-watch &
-        shell: bash
-
-      # Window is slow so wait longer
-      - name: Sleep until OSD server starts - windows
-        if: ${{ matrix.os == 'windows-latest' }}
-        run: Start-Sleep -s 400
-        shell: powershell
+          nohup yarn start --no-base-path --no-watch | tee dashboard.log &
+        working-directory: OpenSearch-Dashboards
 
-      - name: Sleep until OSD server starts - non-windows
-        if: ${{ matrix.os != 'windows-latest' }}
-        run: sleep 300
+      - name : Check If OpenSearch Dashboards Is Ready
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
+            echo "OpenSearch Dashboards compiled successfully."
+          else
+            echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
+            exit 1
+          fi
+        working-directory: OpenSearch-Dashboards
+
+      - name: Check OpenSearch Dashboards Running on Linux
+        if: ${{ runner.os != 'Windows'}}
+        run: curl http://localhost:5601/
         shell: bash
 
-      - name: Checkout opensearch-dashboards-functional-test
+      - name: Checkout Dashboards Functioanl Test Repo
         uses: actions/checkout@v2
         with:
           path: opensearch-dashboards-functional-test
           repository: opensearch-project/opensearch-dashboards-functional-test
-          ref: '${{ github.base_ref }}'
+          ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
+          fetch-depth: 0
+
+      - name: Install Cypress
+        run: |
+          npm install cypress --save-dev
+        shell: bash
+        working-directory: opensearch-dashboards-functional-test
 
       - name: Get Cypress version
         id: cypress_version
         run: |
-          echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')"
+          echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')"
+        working-directory: opensearch-dashboards-functional-test
 
-      - name: Cache Cypress
-        id: cache-cypress
-        uses: actions/cache@v1
+      - name: Run Cypress tests
+        run: |
+          yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/anomaly-detection-dashboards-plugin/*.js'
+        working-directory: opensearch-dashboards-functional-test
+
+      - name: Capture failure screenshots
+        uses: actions/upload-artifact@v1
+        if: failure()
         with:
-          path: ${{ matrix.cypress_cache_folder }}
-          key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
-        env:
-          CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
-      - run: npx cypress cache list
-      - run: npx cypress cache path
-
-      - name: Run AD cypress tests
-        uses: cypress-io/github-action@v2
+          name: cypress-screenshots-${{ matrix.os }}
+          path: opensearch-dashboards-functional-test/cypress/screenshots
+
+      - name: Capture failure test video
+        uses: actions/upload-artifact@v1
+        if: failure()
         with:
-          working-directory: opensearch-dashboards-functional-test
-          command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
-        env:
-          CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}   
+          name: cypress-videos-${{ matrix.os }}
+          path: opensearch-dashboards-functional-test/cypress/videos
diff --git a/package.json b/package.json
@@ -4,17 +4,17 @@
   "description": "OpenSearch Anomaly Detection Dashboards Plugin",
   "main": "index.js",
   "config": {
-    "plugin_version": "3.0.0.0",
-    "plugin_name": "anomalyDetectionDashboards",
-    "plugin_zip_name": "anomaly-detection-dashboards"
+    "id": "anomalyDetectionDashboards",
+    "zip_name": "anomaly-detection-dashboards"
   },
   "scripts": {
     "osd": "node ../../scripts/osd",
     "opensearch": "node ../../scripts/opensearch",
     "lint": "node ../../scripts/eslint .",
     "plugin-helpers": "node ../../scripts/plugin_helpers",
     "test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
-    "build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
+    "build": "yarn plugin-helpers build",
+    "postbuild": "echo Renaming artifact to [$npm_package_config_zip_name-$npm_package_version.zip] && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip"
   },
   "lint-staged": {
     "*.{ts,tsx,js,jsx,json,css,md}": [
@@ -56,4 +56,4 @@
     "browserify-sign": "^4.2.2",
     "axios": "^1.6.1"
   }
-}
+}
diff --git a/public/pages/DetectorResults/containers/AnomalyResults.tsx b/public/pages/DetectorResults/containers/AnomalyResults.tsx
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
       endDate: adjustedCurrentTime.valueOf(),
     } as DateRange;
 
+    // build result search query params relative to data end time
     const params = buildParamsForGetAnomalyResultsWithDateRange(
       featureDataPointsRange.startDate,
       featureDataPointsRange.endDate
diff --git a/public/pages/utils/__tests__/anomalyResultUtils.test.ts b/public/pages/utils/__tests__/anomalyResultUtils.test.ts
@@ -13,6 +13,7 @@ import {
   getFeatureMissingDataAnnotations,
   getFeatureDataPointsForDetector,
   parsePureAnomalies,
+  buildParamsForGetAnomalyResultsWithDateRange,
 } from '../anomalyResultUtils';
 import { getRandomDetector } from '../../../redux/reducers/__tests__/utils';
 import {
@@ -22,11 +23,16 @@ import {
   AnomalyData,
 } from '../../../models/interfaces';
 import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants';
+import { MAX_ANOMALIES } from '../../../utils/constants';
+import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants';
 
 describe('anomalyResultUtils', () => {
   let randomDetector_20_min: Detector;
   let randomDetector_20_sec: Detector;
   let feature_id = 'deny_max';
+  const startTime = 1609459200000; // January 1, 2021
+  const endTime = 1609545600000;   // January 2, 2021
+
   beforeAll(() => {
     randomDetector_20_min = {
       ...getRandomDetector(true),
@@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => {
         )
       ).toEqual([]);
     });
+    test('should correctly build parameters with default options', () => {
+      const expected = {
+          from: 0,
+          size: MAX_ANOMALIES,
+          sortDirection: SORT_DIRECTION.DESC,
+          sortField: AD_DOC_FIELDS.DATA_END_TIME,
+          startTime: startTime,
+          endTime: endTime,
+          fieldName: AD_DOC_FIELDS.DATA_END_TIME,
+          anomalyThreshold: -1,
+          entityList: undefined,  // Default as an empty array stringified
+      };
+
+      const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime);
+      expect(result).toEqual(expected);
+  });
+
+  test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => {
+      const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }];
+      const expected = {
+          from: 0,
+          size: MAX_ANOMALIES,
+          sortDirection: SORT_DIRECTION.DESC,
+          sortField: AD_DOC_FIELDS.DATA_END_TIME,
+          startTime: startTime,
+          endTime: endTime,
+          fieldName: AD_DOC_FIELDS.DATA_END_TIME,
+          anomalyThreshold: 0, // because anomalyOnly is true
+          entityList: JSON.stringify(entities),
+      };
+
+      const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities);
+      expect(result).toEqual(expected);
+  });
+
+  test('should handle undefined `entityList` as an empty array JSON string', () => {
+      const expected = {
+          from: 0,
+          size: MAX_ANOMALIES,
+          sortDirection: SORT_DIRECTION.DESC,
+          sortField: AD_DOC_FIELDS.DATA_END_TIME,
+          startTime: startTime,
+          endTime: endTime,
+          fieldName: AD_DOC_FIELDS.DATA_END_TIME,
+          anomalyThreshold: -1, // default as anomalyOnly is false
+          entityList: undefined,  // Default for undefined entityList
+      };
+
+      const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined);
+      expect(result).toEqual(expected);
+  });
   });
 
   describe('parsePureAnomalies()', () => {
diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts
diff --git a/release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md b/release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md
