Enable shingle in HCAD (opensearch-project#71)

Previously, we hardcoded HCAD (including real-time and historical) shingle size to be one due to time constraints when releasing HCAD. This PR enabled shingling in HCAD on the frontend (backend PR: opensearch-project/anomaly-detection#187). Specifically, after this PR
* both single-stream and HCAD detectors have a default shingle size of 8.
* no special handling of HCAD detectors' shingle size.

Testing done:
1. unit tests and cypress tests pass.
2. Manual testing to verify HCAD real-time and historical tasks use shingle eight by default and can use customized size.
3. Manual testing to verify preview uses shingle eight by default and can use the customized size.

Author: kaituo

public/pages/ConfigureModel/components/CategoryField/CategoryField.tsx

@@ -39,10 +39,7 @@ import {
 } from '@elastic/eui';
 import { Field, FieldProps, FormikProps } from 'formik';
 import { get, isEmpty } from 'lodash';
-import {
-  MULTI_ENTITY_SHINGLE_SIZE,
-  BASE_DOCS_LINK,
-} from '../../../../utils/constants';
+import { BASE_DOCS_LINK } from '../../../../utils/constants';
 import React, { useState, useEffect } from 'react';
 import ContentPanel from '../../../../components/ContentPanel/ContentPanel';
 import {
@@ -58,7 +55,6 @@ interface CategoryFieldProps {
   categoryFieldOptions: string[];
   setIsHCDetector(isHCDetector: boolean): void;
   isLoading: boolean;
-  originalShingleSize: number;
   formikProps: FormikProps<ModelConfigurationFormikValues>;
 }
 
@@ -129,10 +125,6 @@ export function CategoryField(props: CategoryFieldProps) {
                   if (enabled) {
                     props.setIsHCDetector(false);
                     form.setFieldValue('categoryField', []);
-                    form.setFieldValue(
-                      'shingleSize',
-                      props.originalShingleSize
-                    );
                   }
                   setEnabled(!enabled);
                 }}
@@ -159,18 +151,9 @@ export function CategoryField(props: CategoryFieldProps) {
                       if (!isEmpty(selection)) {
                         if (selection.length <= 2) {
                           form.setFieldValue('categoryField', selection);
-                          form.setFieldValue(
-                            'shingleSize',
-                            MULTI_ENTITY_SHINGLE_SIZE
-                          );
                         }
                       } else {
                         form.setFieldValue('categoryField', []);
-
-                        form.setFieldValue(
-                          'shingleSize',
-                          props.originalShingleSize
-                        );
                       }
                     }}
                     selectedOptions={

public/pages/ConfigureModel/components/CategoryField/__tests__/CategoryField.test.tsx

@@ -49,7 +49,6 @@ describe('<CategoryField /> spec', () => {
                   return;
                 }}
                 isLoading={false}
-                originalShingleSize={1}
                 formikProps={{
                   values: {
                     categoryFieldEnabled: false,
@@ -91,7 +90,6 @@ describe('<CategoryField /> spec', () => {
                   return;
                 }}
                 isLoading={false}
-                originalShingleSize={1}
                 formikProps={{
                   values: {
                     categoryFieldEnabled: true,
@@ -130,7 +128,6 @@ describe('<CategoryField /> spec', () => {
                   return;
                 }}
                 isLoading={false}
-                originalShingleSize={1}
                 formikProps={{
                   values: {
                     categoryFieldEnabled: true,
@@ -166,7 +163,6 @@ describe('<CategoryField /> spec', () => {
                   return;
                 }}
                 isLoading={true}
-                originalShingleSize={1}
                 formikProps={{
                   values: {
                     categoryFieldEnabled: true,
@@ -201,7 +197,6 @@ describe('<CategoryField /> spec', () => {
                   return;
                 }}
                 isLoading={false}
-                originalShingleSize={1}
                 formikProps={{
                   values: {
                     categoryFieldEnabled: true,

public/pages/ConfigureModel/containers/ConfigureModel.tsx

@@ -102,7 +102,6 @@ export function ConfigureModel(props: ConfigureModelProps) {
   const isLoading = useSelector(
     (state: AppState) => state.opensearch.requesting
   );
-  const originalShingleSize = getShingleSizeFromObject(detector, isHCDetector);
 
   // Jump to top of page on first load
   useEffect(() => {
@@ -277,7 +276,6 @@ export function ConfigureModel(props: ConfigureModelProps) {
                 categoryFieldOptions={getCategoryFields(indexDataTypes)}
                 setIsHCDetector={setIsHCDetector}
                 isLoading={isLoading}
-                originalShingleSize={originalShingleSize}
                 formikProps={formikProps}
               />
               <EuiSpacer />

public/pages/ConfigureModel/utils/constants.tsx

@@ -29,12 +29,13 @@ import {
   ModelConfigurationFormikValues,
   FeaturesFormikValues,
 } from '../../ConfigureModel/models/interfaces';
+import { DEFAULT_SHINGLE_SIZE } from '../../../utils/constants';
 
 export const INITIAL_MODEL_CONFIGURATION_VALUES: ModelConfigurationFormikValues = {
   featureList: [],
   categoryFieldEnabled: false,
   categoryField: [],
-  shingleSize: 4,
+  shingleSize: DEFAULT_SHINGLE_SIZE,
 };
 
 export const INITIAL_FEATURE_VALUES: FeaturesFormikValues = {

public/pages/ConfigureModel/utils/helpers.ts

@@ -24,11 +24,7 @@
  * permissions and limitations under the License.
  */
 
-import {
-  DATA_TYPES,
-  MULTI_ENTITY_SHINGLE_SIZE,
-  SINGLE_ENTITY_SHINGLE_SIZE,
-} from '../../../utils/constants';
+import { DATA_TYPES, DEFAULT_SHINGLE_SIZE } from '../../../utils/constants';
 import {
   FEATURE_TYPE,
   FeatureAttributes,
@@ -222,15 +218,8 @@ export const getCategoryFields = (dataTypes: DataTypes) => {
   return keywordFields.concat(ipFields);
 };
 
-export const getShingleSizeFromObject = (
-  obj: object,
-  isHCDetector: boolean
-) => {
-  return get(
-    obj,
-    'shingleSize',
-    isHCDetector ? MULTI_ENTITY_SHINGLE_SIZE : SINGLE_ENTITY_SHINGLE_SIZE
-  );
+export const getShingleSizeFromObject = (obj: object) => {
+  return get(obj, 'shingleSize', DEFAULT_SHINGLE_SIZE);
 };
 
 export function clearModelConfiguration(ad: Detector): Detector {
@@ -242,7 +231,7 @@ export function clearModelConfiguration(ad: Detector): Detector {
       features: {},
     },
     categoryField: undefined,
-    shingleSize: SINGLE_ENTITY_SHINGLE_SIZE,
+    shingleSize: DEFAULT_SHINGLE_SIZE,
   };
 }
 
@@ -258,7 +247,7 @@ export function modelConfigurationToFormik(
     featureList: featuresToFormik(detector),
     categoryFieldEnabled: !isEmpty(get(detector, 'categoryField', [])),
     categoryField: get(detector, 'categoryField', []),
-    shingleSize: get(detector, 'shingleSize', 4),
+    shingleSize: get(detector, 'shingleSize', DEFAULT_SHINGLE_SIZE),
   };
 }
 

public/pages/DefineDetector/utils/helpers.ts

@@ -37,7 +37,7 @@ import {
   formikToIndices,
   formikToFilterQuery,
 } from '../../ReviewAndCreate/utils/helpers';
-import { SINGLE_ENTITY_SHINGLE_SIZE } from '../../../utils/constants';
+import { DEFAULT_SHINGLE_SIZE } from '../../../utils/constants';
 
 export function detectorDefinitionToFormik(
   ad: Detector
@@ -149,6 +149,6 @@ export function clearModelConfiguration(ad: Detector): Detector {
       features: {},
     },
     categoryField: undefined,
-    shingleSize: SINGLE_ENTITY_SHINGLE_SIZE,
+    shingleSize: DEFAULT_SHINGLE_SIZE,
   };
 }

public/pages/DetectorConfig/containers/Features.tsx

@@ -105,8 +105,7 @@ export const Features = (props: FeaturesProps) => {
     return sorted;
   };
   const featureAttributes = get(props.detector, 'featureAttributes', []);
-  const isHCDetector = !isEmpty(get(props.detector, 'categoryField', []));
-  const shingleSize = getShingleSizeFromObject(props.detector, isHCDetector);
+  const shingleSize = getShingleSizeFromObject(props.detector);
 
   const sorting = {
     sort: {

public/pages/DetectorResults/utils/utils.ts

@@ -29,7 +29,7 @@ import {
   NO_DATA_IN_WINDOW_ERROR_MESSAGE,
   NO_RCF_MODEL_ERROR_MESSAGE,
 } from './constants';
-import { SINGLE_ENTITY_SHINGLE_SIZE } from '../../../utils/constants';
+import { DEFAULT_SHINGLE_SIZE } from '../../../utils/constants';
 import { DETECTOR_STATE } from '../../../../server/utils/constants';
 import moment, { Moment } from 'moment';
 import { get } from 'lodash';
@@ -63,7 +63,7 @@ const isDetectorInitOverTime = (currentTime: Moment, detector: Detector) => {
     //@ts-ignore
     currentTime
       .subtract(
-        get(detector, 'shingleSize', SINGLE_ENTITY_SHINGLE_SIZE) *
+        get(detector, 'shingleSize', DEFAULT_SHINGLE_SIZE) *
           detector.detectionInterval.period.interval,
         detector.detectionInterval.period.unit.toLowerCase()
       )

public/pages/ReviewAndCreate/components/ModelConfigurationFields/ModelConfigurationFields.tsx

@@ -30,7 +30,7 @@ import {
   FEATURE_TYPE,
   FeatureAttributes,
 } from '../../../../models/interfaces';
-import { get, isEmpty, sortBy } from 'lodash';
+import { get, sortBy } from 'lodash';
 import ContentPanel from '../../../../components/ContentPanel/ContentPanel';
 import { CodeModal } from '../../../../components/CodeModal/CodeModal';
 import { AdditionalSettings } from '../AdditionalSettings/AdditionalSettings';
@@ -100,8 +100,7 @@ export const ModelConfigurationFields = (
     return sorted;
   };
   const featureAttributes = get(props.detector, 'featureAttributes', []);
-  const isHCDetector = !isEmpty(get(props.detector, 'categoryField', []));
-  const shingleSize = getShingleSizeFromObject(props.detector, isHCDetector);
+  const shingleSize = getShingleSizeFromObject(props.detector);
 
   const sorting = {
     sort: {

public/pages/ReviewAndCreate/containers/__tests__/__snapshots__/ReviewAndCreate.test.tsx.snap

@@ -486,7 +486,7 @@ exports[`<ReviewAndCreate /> spec renders the component 1`] = `
                           <span
                             class="euiTableCellContent__text"
                           >
-                            4
+                            8
                           </span>
                         </div>
                       </td>

public/pages/ReviewAndCreate/utils/__tests__/helpers.test.tsx

@@ -25,7 +25,7 @@
  */
 
 import { INITIAL_DETECTOR_DEFINITION_VALUES } from '../../../DefineDetector/utils/constants';
-import { SINGLE_ENTITY_SHINGLE_SIZE } from '../../../../utils/constants';
+import { DEFAULT_SHINGLE_SIZE } from '../../../../utils/constants';
 import { getRandomDetector } from '../../../../redux/reducers/__tests__/utils';
 import { formikToDetector, formikToFilterQuery } from '../../utils/helpers';
 import {
@@ -56,7 +56,7 @@ describe('formikToDetector', () => {
       featureList: [],
       categoryFieldEnabled: false,
       categoryField: [],
-      shingleSize: SINGLE_ENTITY_SHINGLE_SIZE,
+      shingleSize: DEFAULT_SHINGLE_SIZE,
       realTime: false,
       historical: false,
       startTime: undefined,
@@ -101,7 +101,7 @@ describe('formikToDetector', () => {
           unit: UNITS.MINUTES,
         },
       },
-      shingleSize: SINGLE_ENTITY_SHINGLE_SIZE,
+      shingleSize: DEFAULT_SHINGLE_SIZE,
       categoryField: undefined,
     });
   });

public/redux/reducers/__tests__/utils.ts

@@ -22,7 +22,7 @@ import {
 } from '../../../models/interfaces';
 import moment from 'moment';
 import { DETECTOR_STATE } from '../../../../server/utils/constants';
-import { SINGLE_ENTITY_SHINGLE_SIZE } from '../../../utils/constants';
+import { DEFAULT_SHINGLE_SIZE } from '../../../utils/constants';
 
 const detectorFaker = new chance('seed');
 
@@ -115,7 +115,7 @@ export const getRandomDetector = (isCreate: boolean = true): Detector => {
     disabledTime: moment(1586823218000).subtract(1, 'days').valueOf(),
     curState: DETECTOR_STATE.INIT,
     stateError: '',
-    shingleSize: SINGLE_ENTITY_SHINGLE_SIZE,
+    shingleSize: DEFAULT_SHINGLE_SIZE,
   };
 };
 

public/utils/constants.ts

@@ -84,9 +84,7 @@ export const MAX_CATEGORY_FIELD_NUM = 2;
 export const NAME_REGEX = RegExp('^[a-zA-Z0-9._-]+$');
 
 //https://github.com/opensearch-project/anomaly-detection/blob/main/src/main/java/com/amazon/opendistroforelasticsearch/ad/settings/AnomalyDetectorSettings.java
-export const SINGLE_ENTITY_SHINGLE_SIZE = 8;
-
-export const MULTI_ENTITY_SHINGLE_SIZE = 4;
+export const DEFAULT_SHINGLE_SIZE = 8;
 
 export const FEATURE_DATA_POINTS_WINDOW = 3;