build query parameters using data_end_time

This PR addresses a data non-population issue observed in HC detectors. When setting the time horizon in the anomaly overview to the past hour, two boxes appeared in the heatmap. However, clicking on both resulted in no data being populated. Extending the time horizon to three hours increased the number of boxes to six, but similarly, clicking on these boxes also resulted in no data appearing.

The root cause of the issue is a mismatch in time references: the time displayed in the HC heatmap cells is calculated based on the anomaly plot time, which corresponds to data_end_time. However, when querying data within the HC heatmap cell's time range, data_start_time was used instead.

This PR updates sorting and querying fields from `DATA_START_TIME` to `DATA_END_TIME` to align with the data displayed in HC heatmap cells and ensure accuracy in temporal data analysis.

Testing done:
1. reproduced the issue and verified the fix.
2. added unit tests.
3. Confirmed that single stream detector result views remain functional post-changes.

Signed-off-by: Kaituo Li <kaituo@amazon.com>

Author: kaituo

.github/workflows/build-and-test-workflow.yml

@@ -88,6 +88,12 @@ jobs:
         run: |
           cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
           yarn osd bootstrap --single-version=loose
+      - name: Set npm to use bash for shell
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          # Sets Windows to use bash for npm shell so the script (e.g., environment variable resolution in package.json build script)
+          # commands work as intended
+          npm config set script-shell "C:\\Program Files\\git\\bin\\bash.exe"
       - name: Build the plugin
         run: |
           cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin

.github/workflows/remote-integ-tests-workflow.yml

@@ -79,6 +79,11 @@ jobs:
           timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9200)" != "200" ]]; do sleep 5; done'
         shell: bash
 
+      - name: Check OpenSearch Running on Linux
+        if: ${{ matrix.os != 'windows-latest' }}
+        run: curl http://localhost:9200/
+        shell: bash
+
       - name: Bootstrap the plugin
         run: |
           cd OpenSearch-Dashboards/plugins/anomaly-detection-dashboards-plugin
@@ -87,9 +92,20 @@ jobs:
       - name: Run OpenSearch Dashboards server
         run: |
           cd OpenSearch-Dashboards
-          yarn start --no-base-path --no-watch &
+          nohup yarn start --no-base-path --no-watch | tee dashboard.log &
         shell: bash
 
+      - name : Check If OpenSearch Dashboards Is Ready
+        if: ${{ matrix.os != 'windows-latest' }}
+        run: |
+          if timeout 600 grep -q "bundles compiled successfully after" <(tail -n0 -f dashboard.log); then
+            echo "OpenSearch Dashboards compiled successfully."
+          else
+            echo "Timeout for 600 seconds reached. OpenSearch Dashboards did not finish compiling."
+            exit 1
+          fi
+        working-directory: OpenSearch-Dashboards
+
       # Window is slow so wait longer
       - name: Sleep until OSD server starts - windows
         if: ${{ matrix.os == 'windows-latest' }}
@@ -128,6 +144,6 @@ jobs:
         uses: cypress-io/github-action@v2
         with:
           working-directory: opensearch-dashboards-functional-test
-          command: yarn run cypress run --env SECURITY_ENABLED=false --spec cypress/integration/plugins/anomaly-detection-dashboards-plugin/**/*.js
+          command: yarn cypress:run-without-security --browser chromium --spec 'cypress/integration/plugins/anomaly-detection-dashboards-plugin/*.js'
         env:
           CYPRESS_CACHE_FOLDER: ${{ matrix.cypress_cache_folder }}   

public/pages/DetectorResults/containers/AnomalyResults.tsx

@@ -255,6 +255,7 @@ export function AnomalyResults(props: AnomalyResultsProps) {
       endDate: adjustedCurrentTime.valueOf(),
     } as DateRange;
 
+    // build result search query params relative to data end time
     const params = buildParamsForGetAnomalyResultsWithDateRange(
       featureDataPointsRange.startDate,
       featureDataPointsRange.endDate

public/pages/utils/__tests__/anomalyResultUtils.test.ts

@@ -13,6 +13,7 @@ import {
   getFeatureMissingDataAnnotations,
   getFeatureDataPointsForDetector,
   parsePureAnomalies,
+  buildParamsForGetAnomalyResultsWithDateRange,
 } from '../anomalyResultUtils';
 import { getRandomDetector } from '../../../redux/reducers/__tests__/utils';
 import {
@@ -22,11 +23,16 @@ import {
   AnomalyData,
 } from '../../../models/interfaces';
 import { ANOMALY_RESULT_SUMMARY, PARSED_ANOMALIES } from './constants';
+import { MAX_ANOMALIES } from '../../../utils/constants';
+import { SORT_DIRECTION, AD_DOC_FIELDS } from '../../../../server/utils/constants';
 
 describe('anomalyResultUtils', () => {
   let randomDetector_20_min: Detector;
   let randomDetector_20_sec: Detector;
   let feature_id = 'deny_max';
+  const startTime = 1609459200000; // January 1, 2021
+  const endTime = 1609545600000;   // January 2, 2021
+
   beforeAll(() => {
     randomDetector_20_min = {
       ...getRandomDetector(true),
@@ -569,6 +575,57 @@ describe('anomalyResultUtils', () => {
         )
       ).toEqual([]);
     });
+    test('should correctly build parameters with default options', () => {
+      const expected = {
+          from: 0,
+          size: MAX_ANOMALIES,
+          sortDirection: SORT_DIRECTION.DESC,
+          sortField: AD_DOC_FIELDS.DATA_END_TIME,
+          startTime: startTime,
+          endTime: endTime,
+          fieldName: AD_DOC_FIELDS.DATA_END_TIME,
+          anomalyThreshold: -1,
+          entityList: undefined,  // Default as an empty array stringified
+      };
+
+      const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime);
+      expect(result).toEqual(expected);
+  });
+
+  test('should correctly handle `anomalyOnly` and non-empty `entityList`', () => {
+      const entities = [{ id: '1', name: 'Entity1' }, { id: '2', name: 'Entity2' }];
+      const expected = {
+          from: 0,
+          size: MAX_ANOMALIES,
+          sortDirection: SORT_DIRECTION.DESC,
+          sortField: AD_DOC_FIELDS.DATA_END_TIME,
+          startTime: startTime,
+          endTime: endTime,
+          fieldName: AD_DOC_FIELDS.DATA_END_TIME,
+          anomalyThreshold: 0, // because anomalyOnly is true
+          entityList: JSON.stringify(entities),
+      };
+
+      const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, true, entities);
+      expect(result).toEqual(expected);
+  });
+
+  test('should handle undefined `entityList` as an empty array JSON string', () => {
+      const expected = {
+          from: 0,
+          size: MAX_ANOMALIES,
+          sortDirection: SORT_DIRECTION.DESC,
+          sortField: AD_DOC_FIELDS.DATA_END_TIME,
+          startTime: startTime,
+          endTime: endTime,
+          fieldName: AD_DOC_FIELDS.DATA_END_TIME,
+          anomalyThreshold: -1, // default as anomalyOnly is false
+          entityList: undefined,  // Default for undefined entityList
+      };
+
+      const result = buildParamsForGetAnomalyResultsWithDateRange(startTime, endTime, false, undefined);
+      expect(result).toEqual(expected);
+  });
   });
 
   describe('parsePureAnomalies()', () => {

public/pages/utils/anomalyResultUtils.ts

@@ -118,6 +118,38 @@ export const getLiveAnomalyResults = (
   );
 };
 
+/**
+ * Builds search query parameters for retrieving anomaly results within a specified date range.
+ *
+ * This function constructs a parameter object for querying an anomaly detection system, filtering results
+ * by a given start and end time. It supports filtering anomalies based on a threshold and can limit results to
+ * specific entities if provided.
+ *
+ * In the context of anomaly results, the startTime and endTime parameters are used to compare against the data_end_time.
+ * Using data_end_time instead of data_start_time is crucial because, within HC heatmap cells, the startTime and
+ * endTime are derived from each cell's start and end times, which are determined based on the plotTime—coinciding
+ * with the data_end_time. This alignment ensures that the temporal data within each heatmap cell accurately
+ * reflects the intervals intended for analysis.
+ *
+ * @param startTime - The epoch time (in milliseconds) marking the start of the date range for the query.
+ * @param endTime - The epoch time (in milliseconds) marking the end of the date range for the query.
+ * @param anomalyOnly - Optional. If true, the query will return only results where anomalies are detected
+ *                      (anomaly threshold is set to 0). If false or omitted, it will include all results
+ *                      (anomaly threshold is set to -1). Default is `false`.
+ * @param entityList - Optional. An array of entities to filter the results. If omitted, results are not filtered
+ *                     by entities. Default is `undefined`.
+ *
+ * @returns An object containing the necessary parameters for the anomaly results search query. This object includes:
+ *          - `from`: The starting index for fetching results (always set to 0).
+ *          - `size`: The maximum number of anomalies to return (`MAX_ANOMALIES`).
+ *          - `sortDirection`: The sorting order of results, set to descending (`SORT_DIRECTION.DESC`).
+ *          - `sortField`: The field used to sort the data, set to data end time (`AD_DOC_FIELDS.DATA_END_TIME`).
+ *          - `startTime`: Passed start time for the search range.
+ *          - `endTime`: Passed end time for the search range.
+ *          - `fieldName`: Field used to query the data, set to data end time (`AD_DOC_FIELDS.DATA_END_TIME`).
+ *          - `anomalyThreshold`: The minimum score threshold for anomalies, dependent on `anomalyOnly` parameter.
+ *          - `entityList`: A JSON string representing the list of entities to filter the results by.
+ */
 export const buildParamsForGetAnomalyResultsWithDateRange = (
   startTime: number,
   endTime: number,
@@ -128,10 +160,10 @@ export const buildParamsForGetAnomalyResultsWithDateRange = (
     from: 0,
     size: MAX_ANOMALIES,
     sortDirection: SORT_DIRECTION.DESC,
-    sortField: AD_DOC_FIELDS.DATA_START_TIME,
+    sortField: AD_DOC_FIELDS.DATA_END_TIME,
     startTime: startTime,
     endTime: endTime,
-    fieldName: AD_DOC_FIELDS.DATA_START_TIME,
+    fieldName: AD_DOC_FIELDS.DATA_END_TIME,
     anomalyThreshold: anomalyOnly ? 0 : -1,
     entityList: JSON.stringify(entityList),
   };

release-notes/opendistro-for-elasticsearch.anomaly-detection-kibana-plugin.release-notes-1.7.0.0.md

@@ -61,7 +61,7 @@ You can use the plugin with the same version of the [Open Distro for Elasticsear
 - Tune AD result charts [PR #102](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/102)
 - Use annotation for live chart [PR #119](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/119)
 - Set fixed height for anomalies live chart [PR #123](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/123)
-- Use scientific notation when number less than 0.01 on live chart [PR #124](https://github.com/opendistro-for-elasticsearchanomaly-detection-kibana-plugin/pull/124)
+- Use scientific notation when number less than 0.01 on live chart [PR #124](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/124)
 - Use bucket aggregation for anomaly distribution [PR #126](https://github.com/opendistro-for-elasticsearch/anomaly-detection-kibana-plugin/pull/126)
 
 ## Bug Fixes