backport #741 (#747)

kaituo · web-flow · commit 273dbf852c2f · 2024-04-30T13:03:14.000-07:00
Signed-off-by: Kaituo Li &lt;kaituo@amazon.com&gt;
diff --git a/.github/workflows/build-and-test-workflow.yml b/.github/workflows/build-and-test-workflow.yml
@@ -88,6 +88,12 @@ jobs:
         run: |
           cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
           yarn osd bootstrap --single-version=loose
+      - name: Set npm to use bash for shell
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          # Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json postbuild script)
+          # commands work as intended
+          npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
       - name: Build the plugin
         run: |
           cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
diff --git a/.github/workflows/remote-integ-tests-workflow.yml b/.github/workflows/remote-integ-tests-workflow.yml
@@ -1,133 +1,175 @@
-# Running AD integ tests stored in https://github.com/opensearch-project/opensearch-dashboards-functional-test
-# In the future we should pull dependencies from bundled build snapshots. Because that is not available
-# yet we build the cluster from source (besides core Opensearch, which is a pulled min artifact).
-name: Remote integ tests workflow
-on:
-  push:
-    branches:
-      - "*"
-  pull_request:
-    branches:
-      - "*"
+name: FTR E2E AD Workbench Test
+
+on: [pull_request, push]
+
+env:
+  CI: 1
+  # avoid warnings like "tput: No value for $TERM and no -T specified"
+  TERM: xterm
+  OPENSEARCH_DASHBOARDS_VERSION: '2.x'
+
 jobs:
-  test-without-security:
-    name: Run integ tests without security
+  tests:
+    name: Run FTR E2E AD Workbench Tests
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest, windows-latest]
-        java: [11]
-        include:
-          - os: windows-latest
-            cypress_cache_folder: ~/AppData/Local/Cypress/Cache
-          - os: ubuntu-latest
-            cypress_cache_folder: ~/.cache/Cypress
+        os: [ ubuntu-latest ]
+        jdk: [ 11 ]
     runs-on: ${{ matrix.os }}
+
     steps:
-      - name: Set up Java 11
-        uses: actions/setup-java@v3
+      - name: Set up JDK
+        uses: actions/setup-java@v1
         with:
-          distribution: 'corretto'
-          java-version: '11'
+          java-version: ${{ matrix.jdk }}
 
-      - name: Enable longer filenames
-        if: ${{ matrix.os == 'windows-latest' }}
-        run: git config --system core.longpaths true
+      - name: Checkout Anomaly-Detection
+        uses: actions/checkout@v2
+        with:
+          path: anomaly-detection
+          repository: opensearch-project/anomaly-detection
+          ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
+
+      - name: Run OpenSearch with plugin
+        run: |
+          cd anomaly-detection
+          ./gradlew run &
+          timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
+        shell: bash
+
+      - name: Check OpenSearch Running on Linux
+        if: ${{ runner.os != 'Windows'}}
+        run: curl http://localhost:9200/
+        shell: bash
 
       - name: Checkout OpenSearch Dashboards
         uses: actions/checkout@v2
         with:
-          repository: opensearch-project/OpenSearch-Dashboards
-          ref: '${{ github.base_ref }}'
           path: OpenSearch-Dashboards
+          repository: opensearch-project/OpenSearch-Dashboards
+          ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
+          fetch-depth: 0
+          filter: |
+            cypress
+            test
 
-      - name: Checkout Anomaly Detection OpenSearch Dashboards plugin
+      - name: Checkout AD in OpenSearch Dashboards Plugins Dir
         uses: actions/checkout@v2
         with:
-          path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
+            path: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
 
-      - name: Setup Node
-        uses: actions/setup-node@v3
-        with:
-          node-version-file: './OpenSearch-Dashboards/.nvmrc'
-          registry-url: 'https://registry.npmjs.org'
-
-      - name: Install Yarn
-        # Need to use bash to avoid having a windows/linux specific step
-        shell: bash
+      - id: tool-versions
         run: |
-          YARN_VERSION=$(node -p "require('./OpenSearch-Dashboards/package.json').engines.yarn")
-          echo "Installing yarn@$YARN_VERSION"
-          npm i -g yarn@$YARN_VERSION
-
-      - run: node -v
-      - run: yarn -v
+          echo "node_version=$(cat .node-version)" >> $GITHUB_OUTPUT
+          echo "yarn_version=$(jq -r '.engines.yarn' package.json)" >> $GITHUB_OUTPUT
+        working-directory: OpenSearch-Dashboards
+        shell: bash
 
-      - name: Checkout Anomaly-Detection
-        uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
         with:
-          path: anomaly-detection
-          repository: opensearch-project/anomaly-detection
-          ref: '${{ github.base_ref }}'
+          node-version: ${{ steps.tool-versions.outputs.node_version }}
+          registry-url: 'https://registry.npmjs.org'
 
-      - name: Run OpenSearch with plugin
+      - name: Setup Opensearch Dashboards
         run: |
-          cd anomaly-detection
-          CONFIG_PATH=../OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin/opensearch_dashboards.json
-          OPENSEARCH_VERSION=$(node -p "require('$CONFIG_PATH').opensearchDashboardsVersion")-SNAPSHOT
-          echo "Using OpenSearch version $OPENSEARCH_VERSION"
-          ./gradlew run -Dopensearch.version=$OPENSEARCH_VERSION &
-          timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
+          npm uninstall -g yarn
+          echo "Installing yarn ${{ steps.tool-versions.outputs.yarn_version }}"
+          npm i -g yarn@${{ steps.tool-versions.outputs.yarn_version }}
+          yarn cache clean
+          yarn add sha.js
+        working-directory: OpenSearch-Dashboards
         shell: bash
 
-      - name: Bootstrap the plugin
+      - name: Boodstrap Opensearch Dashboards
         run: |
-          cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
           yarn osd bootstrap --single-version=loose
+        working-directory: OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
 
-      - name: Run OpenSearch Dashboards server
+      - name: Run Opensearch Dashboards with AD Installed
         run: |
-          cd OpenSearch-Dashboards
-          yarn start --no-base-path --no-watch &
-        shell: bash
+          nohup yarn start --no-base-path --no-watch --server.host="0.0.0.0" | tee dashboard.log &
+        working-directory: OpenSearch-Dashboards
 
-      # Window is slow so wait longer
-      - name: Sleep until OSD server starts - windows
-        if: ${{ matrix.os == 'windows-latest' }}
-        run: Start-Sleep -s 400
-        shell: powershell
+      - name : Check If OpenSearch Dashboards Is Ready
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
+            echo "OpenSearch Dashboards compiled successfully."
+          else
+            echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
+            exit 1
+          fi
+        working-directory: OpenSearch-Dashboards
+
+      - name: Show OpenSearch Dashboards Logs
+        if: always()
+        run: cat dashboard.log
+        working-directory: OpenSearch-Dashboards
+
+      - name: Health check
+        run: |
+          timeout 600 bash -c 'while [[ "$(curl -k http://localhost:5601/api/status | jq -r '.status.overall.state')" != "green" ]]; do sleep 5; done'
+        shell: bash
 
-      - name: Sleep until OSD server starts - non-windows
-        if: ${{ matrix.os != 'windows-latest' }}
-        run: sleep 300
+      - name: Check OpenSearch Dashboards Running on Linux
+        if: ${{ runner.os != 'Windows'}}
+        run: curl http://localhost:5601/api/status
         shell: bash
 
-      - name: Checkout opensearch-dashboards-functional-test
+      - name: Checkout Dashboards Functional Test Repo
         uses: actions/checkout@v2
         with:
           path: opensearch-dashboards-functional-test
           repository: opensearch-project/opensearch-dashboards-functional-test
-          ref: '${{ github.base_ref }}'
+          ref: ${{ env.OPENSEARCH_DASHBOARDS_VERSION }}
+          fetch-depth: 0
+
+      - name: Install Cypress
+        run: |
+          npm install cypress --save-dev
+        shell: bash
+        working-directory: opensearch-dashboards-functional-test
 
       - name: Get Cypress version
         id: cypress_version
         run: |
-          echo "::set-output name=cypress_version::$(cat ./opensearch-dashboards-functional-test/package.json | jq '.devDependencies.cypress' | tr -d '"')"
+          echo "::set-output name=cypress_version::$(cat ./package.json | jq '.dependencies.cypress' | tr -d '"')"
+        working-directory: opensearch-dashboards-functional-test
 
-      - name: Cache Cypress
-        id: cache-cypress
-        uses: actions/cache@v1
+      - name: Finding spec files and store to output
+        id: finding-files
+        run: |
+          echo "::set-output name=FILELIST::$(find cypress/integration/plugins/anomaly-detection-dashboards-plugin -name '*.js' -print)"
+        working-directory: opensearch-dashboards-functional-test
+  
+      - name: Print spec files from output
+        run: |
+          IFS="," read -a myarray <<< ${{ steps.finding-files.outputs.FILELIST }}
+          for i in "${myarray[@]}"; do
+            echo "${i}"
+          done
+        working-directory: opensearch-dashboards-functional-test
+
+      - name: Run spec files from output
+        run: |
+          IFS="," read -a myarray <<< ${{ steps.finding-files.outputs.FILELIST }}
+          for i in "${myarray[@]}"; do
+            yarn cypress:run-without-security --browser electron --spec "${i}"
+            sleep 60
+          done
+        working-directory: opensearch-dashboards-functional-test
+
+      - name: Capture failure screenshots
+        uses: actions/upload-artifact@v1
+        if: failure()
         with:
-          path: ${{ matrix.cypress_cache_folder }}
-          key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
-        env:
-          CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
-      - run: npx cypress cache list
-      - run: npx cypress cache path
-
-      - name: Run AD cypress tests
-        uses: cypress-io/github-action@v2
+          name: cypress-screenshots-${{ matrix.os }}
+          path: opensearch-dashboards-functional-test/cypress/screenshots
+
+      - name: Capture failure test video
+        uses: actions/upload-artifact@v1
+        if: failure()
         with:
-          working-directory: opensearch-dashboards-functional-test
-          command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
-        env:
-          CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}   
+          name: cypress-videos-${{ matrix.os }}
+          path: opensearch-dashboards-functional-test/cypress/videos
diff --git a/package.json b/package.json
@@ -4,17 +4,17 @@
   "description": "OpenSearch Anomaly Detection Dashboards Plugin",
   "main": "index.js",
   "config": {
-    "plugin_version": "2.13.0.0",
-    "plugin_name": "anomalyDetectionDashboards",
-    "plugin_zip_name": "anomaly-detection-dashboards"
+    "id": "anomalyDetectionDashboards",
+    "zip_name": "anomaly-detection-dashboards"
   },
   "scripts": {
     "osd": "node ../../scripts/osd",
     "opensearch": "node ../../scripts/opensearch",
     "lint": "node ../../scripts/eslint .",
     "plugin-helpers": "node ../../scripts/plugin_helpers",
     "test:jest": "../../node_modules/.bin/jest --config ./test/jest.config.js",
-    "build": "yarn plugin-helpers build && echo Renaming artifact to $npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip && mv ./build/$npm_package_config_plugin_name*.zip ./build/$npm_package_config_plugin_zip_name-$npm_package_config_plugin_version.zip"
+    "build": "yarn plugin-helpers build",
+    "postbuild": "echo Renaming artifact to [$npm_package_config_zip_name-$npm_package_version.zip] && mv build/$npm_package_config_id*.zip build/$npm_package_config_zip_name-$npm_package_version.zip"
   },
   "lint-staged": {
     "*.{ts,tsx,js,jsx,json,css,md}": [
diff --git a/public/pages/DetectorResults/containers/AnomalyResults.tsx b/public/pages/DetectorResults/containers/AnomalyResults.tsx
@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
       endDate: adjustedCurrentTime.valueOf(),
     } as DateRange;
 
+    // build result search query params relative to data end time
     const params = buildParamsForGetAnomalyResultsWithDateRange(
       featureDataPointsRange.startDate,
       featureDataPointsRange.endDate
diff --git a/public/pages/utils/__tests__/anomalyResultUtils.test.ts b/public/pages/utils/__tests__/anomalyResultUtils.test.ts
@@ -13,6 +13,7 @@ import {
   getFeatureMissingDataAnnotations,
   getFeatureDataPointsForDetector,
   parsePureAnomalies,
+  buildParamsForGetAnomalyResultsWithDateRange,
 } from '../anomalyResultUtils';
 import { getRandomDetector } from '../../../redux/reducers/__tests__/utils';
 import {
@@ -22,11 +23,16 @@ import {
   AnomalyData,
 } from '../../../models/interfaces';
 import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants';
+import { MAX_ANOMALIES } from '../../../utils/constants';
+import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants';
 
 describe('anomalyResultUtils', () => {
   let randomDetector_20_min: Detector;
   let randomDetector_20_sec: Detector;
   let feature_id = 'deny_max';
+  const startTime = 1609459200000; // January 1, 2021
+  const endTime = 1609545600000;   // January 2, 2021
+
   beforeAll(() => {
     randomDetector_20_min = {
       ...getRandomDetector(true),
@@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => {
         )
       ).toEqual([]);
     });
+    test('should correctly build parameters with default options', () => {
+      const expected = {
+          from: 0,
+          size: MAX_ANOMALIES,
+          sortDirection: SORT_DIRECTION.DESC,
+          sortField: AD_DOC_FIELDS.DATA_END_TIME,
+          startTime: startTime,
+          endTime: endTime,
+          fieldName: AD_DOC_FIELDS.DATA_END_TIME,
+          anomalyThreshold: -1,
+          entityList: undefined,  // Default as an empty array stringified
+      };
+
+      const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime);
+      expect(result).toEqual(expected);
+  });
+
+  test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => {
+      const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }];
+      const expected = {
+          from: 0,
+          size: MAX_ANOMALIES,
+          sortDirection: SORT_DIRECTION.DESC,
+          sortField: AD_DOC_FIELDS.DATA_END_TIME,
+          startTime: startTime,
+          endTime: endTime,
+          fieldName: AD_DOC_FIELDS.DATA_END_TIME,
+          anomalyThreshold: 0, // because anomalyOnly is true
+          entityList: JSON.stringify(entities),
+      };
+
+      const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities);
+      expect(result).toEqual(expected);
+  });
+
+  test('should handle undefined `entityList` as an empty array JSON string', () => {
+      const expected = {
+          from: 0,
+          size: MAX_ANOMALIES,
+          sortDirection: SORT_DIRECTION.DESC,
+          sortField: AD_DOC_FIELDS.DATA_END_TIME,
+          startTime: startTime,
+          endTime: endTime,
+          fieldName: AD_DOC_FIELDS.DATA_END_TIME,
+          anomalyThreshold: -1, // default as anomalyOnly is false
+          entityList: undefined,  // Default for undefined entityList
+      };
+
+      const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined);
+      expect(result).toEqual(expected);
+  });
   });
 
   describe('parsePureAnomalies()', () => {
diff --git a/public/pages/utils/anomalyResultUtils.ts b/public/pages/utils/anomalyResultUtils.ts
diff --git a/release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md b/release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md
