Skip to content

Commit 38308fd

Browse files
gaobinlonggaobinlong
committed
Merge remote-tracking branch 'upstream/main' into discover_action
2 parents eb55c35 + 3ed0058 commit 38308fd

File tree

23 files changed

+1081
-76
lines changed

23 files changed

+1081
-76
lines changed

‎public/models/interfaces.ts

+2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import { DETECTOR_STATE } from '../../server/utils/constants';
1515
import { Duration } from 'moment';
1616
import moment from 'moment';
1717
import { MDSQueryParams } from '../../server/models/types';
18+
import { ImputationOption } from './types';
1819

1920
export type FieldInfo = {
2021
label: string;
@@ -210,6 +211,7 @@ export type Detector = {
210211
taskState?: DETECTOR_STATE;
211212
taskProgress?: number;
212213
taskError?: string;
214+
imputationOption?: ImputationOption;
213215
};
214216

215217
export type DetectorListItem = {

‎public/models/types.ts

+21
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,24 @@
1212
export type AggregationOption = {
1313
label: string;
1414
};
15+
16+
export type ImputationOption = {
17+
method: ImputationMethod;
18+
defaultFill?: Array<{ featureName: string; data: number }>;
19+
};
20+
21+
export enum ImputationMethod {
22+
/**
23+
* This method replaces all missing values with 0's. It's a simple approach, but it may introduce bias if the data is not centered around zero.
24+
*/
25+
ZERO = 'ZERO',
26+
/**
27+
* This method replaces missing values with a predefined set of values. The values are the same for each input dimension, and they need to be specified by the user.
28+
*/
29+
FIXED_VALUES = 'FIXED_VALUES',
30+
/**
31+
* This method replaces missing values with the last known value in the respective input dimension. It's a commonly used method for time series data, where temporal continuity is expected.
32+
*/
33+
PREVIOUS = 'PREVIOUS',
34+
}
35+

‎public/pages/ConfigureModel/components/AdvancedSettings/AdvancedSettings.tsx

+151-32
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,12 @@ import {
1717
EuiTitle,
1818
EuiCompressedFieldNumber,
1919
EuiSpacer,
20+
EuiCompressedSelect,
21+
EuiButtonIcon,
22+
EuiCompressedFieldText,
2023
} from '@elastic/eui';
21-
import { Field, FieldProps } from 'formik';
22-
import React, { useState } from 'react';
24+
import { Field, FieldProps, FieldArray, } from 'formik';
25+
import React, { useEffect, useState } from 'react';
2326
import ContentPanel from '../../../../components/ContentPanel/ContentPanel';
2427
import { BASE_DOCS_LINK } from '../../../../utils/constants';
2528
import {
@@ -28,13 +31,22 @@ import {
2831
validatePositiveInteger,
2932
} from '../../../../utils/utils';
3033
import { FormattedFormRow } from '../../../../components/FormattedFormRow/FormattedFormRow';
34+
import { SparseDataOptionValue } from '../../utils/constants';
3135

3236
interface AdvancedSettingsProps {}
3337

3438
export function AdvancedSettings(props: AdvancedSettingsProps) {
3539
const [showAdvancedSettings, setShowAdvancedSettings] =
3640
useState<boolean>(false);
3741

42+
// Options for the sparse data handling dropdown
43+
const sparseDataOptions = [
44+
{ value: SparseDataOptionValue.IGNORE, text: 'Ignore missing value' },
45+
{ value: SparseDataOptionValue.PREVIOUS_VALUE, text: 'Previous value' },
46+
{ value: SparseDataOptionValue.SET_TO_ZERO, text: 'Set to zero' },
47+
{ value: SparseDataOptionValue.CUSTOM_VALUE, text: 'Custom value' },
48+
];
49+
3850
return (
3951
<ContentPanel
4052
title={
@@ -58,41 +70,148 @@ export function AdvancedSettings(props: AdvancedSettingsProps) {
5870
>
5971
{showAdvancedSettings ? <EuiSpacer size="m" /> : null}
6072
{showAdvancedSettings ? (
61-
<Field name="shingleSize" validate={validatePositiveInteger}>
62-
{({ field, form }: FieldProps) => (
63-
<FormattedFormRow
64-
title="Shingle size"
65-
hint={[
66-
`Set the number of intervals to consider in a detection
73+
<>
74+
<Field name="shingleSize" validate={validatePositiveInteger}>
75+
{({ field, form }: FieldProps) => (
76+
<FormattedFormRow
77+
title="Shingle size"
78+
hint={[
79+
`Set the number of intervals to consider in a detection
6780
window for your model. The anomaly detector expects the
68-
shingle size to be in the range of 1 and 60. The default
81+
shingle size to be in the range of 1 and 128. The default
6982
shingle size is 8. We recommend that you don’t choose 1
7083
unless you have two or more features. Smaller values might
7184
increase recall but also false positives. Larger values
7285
might be useful for ignoring noise in a signal.`,
73-
]}
74-
hintLink={`${BASE_DOCS_LINK}/ad`}
75-
isInvalid={isInvalid(field.name, form)}
76-
error={getError(field.name, form)}
77-
>
78-
<EuiFlexGroup gutterSize="s" alignItems="center">
79-
<EuiFlexItem grow={false}>
80-
<EuiCompressedFieldNumber
81-
id="shingleSize"
82-
placeholder="Shingle size"
83-
data-test-subj="shingleSize"
84-
{...field}
85-
/>
86-
</EuiFlexItem>
87-
<EuiFlexItem>
88-
<EuiText>
89-
<p className="minutes">intervals</p>
90-
</EuiText>
91-
</EuiFlexItem>
92-
</EuiFlexGroup>
93-
</FormattedFormRow>
94-
)}
95-
</Field>
86+
]}
87+
hintLink={`${BASE_DOCS_LINK}/ad`}
88+
isInvalid={isInvalid(field.name, form)}
89+
error={getError(field.name, form)}
90+
>
91+
<EuiFlexGroup gutterSize="s" alignItems="center">
92+
<EuiFlexItem grow={false}>
93+
<EuiCompressedFieldNumber
94+
id="shingleSize"
95+
placeholder="Shingle size"
96+
data-test-subj="shingleSize"
97+
{...field}
98+
/>
99+
</EuiFlexItem>
100+
<EuiFlexItem>
101+
<EuiText>
102+
<p className="minutes">intervals</p>
103+
</EuiText>
104+
</EuiFlexItem>
105+
</EuiFlexGroup>
106+
</FormattedFormRow>
107+
)}
108+
</Field>
109+
110+
<Field
111+
name="imputationOption.imputationMethod"
112+
id="imputationOption.imputationMethod"
113+
>
114+
{({ field, form }: FieldProps) => {
115+
// Add an empty row if CUSTOM_VALUE is selected and no rows exist
116+
useEffect(() => {
117+
if (
118+
field.value === SparseDataOptionValue.CUSTOM_VALUE &&
119+
(!form.values.imputationOption?.custom_value ||
120+
form.values.imputationOption.custom_value.length === 0)
121+
) {
122+
form.setFieldValue('imputationOption.custom_value', [
123+
{ featureName: '', value: undefined },
124+
]);
125+
}
126+
}, [field.value, form]);
127+
128+
return (
129+
<>
130+
<FormattedFormRow
131+
title="Sparse data handling"
132+
hint={[`Choose how to handle missing data points.`]}
133+
hintLink={`${BASE_DOCS_LINK}/ad`}
134+
isInvalid={isInvalid(field.name, form)}
135+
error={getError(field.name, form)}
136+
>
137+
<EuiCompressedSelect {...field} options={sparseDataOptions}/>
138+
</FormattedFormRow>
139+
140+
{/* Conditionally render the "Custom value" title and the input fields when 'Custom value' is selected */}
141+
{field.value === SparseDataOptionValue.CUSTOM_VALUE && (
142+
<>
143+
<EuiSpacer size="m" />
144+
<EuiText size="xs">
145+
<h5>Custom value</h5>
146+
</EuiText>
147+
<EuiSpacer size="s" />
148+
<FieldArray name="imputationOption.custom_value">
149+
{(arrayHelpers) => (
150+
<>
151+
{form.values.imputationOption.custom_value?.map((_, index) => (
152+
<EuiFlexGroup
153+
key={index}
154+
gutterSize="s"
155+
alignItems="center"
156+
>
157+
<EuiFlexItem grow={false}>
158+
<Field
159+
name={`imputationOption.custom_value.${index}.featureName`}
160+
id={`imputationOption.custom_value.${index}.featureName`}
161+
>
162+
{({ field }: FieldProps) => (
163+
<EuiCompressedFieldText
164+
placeholder="Feature name"
165+
{...field}
166+
/>
167+
)}
168+
</Field>
169+
</EuiFlexItem>
170+
<EuiFlexItem grow={false}>
171+
<Field
172+
name={`imputationOption.custom_value.${index}.data`}
173+
id={`imputationOption.custom_value.${index}.data`}
174+
>
175+
{/* the value is set to field.value || '' to avoid displaying 0 as a default value. */ }
176+
{({ field, form }: FieldProps) => (
177+
<EuiCompressedFieldNumber
178+
fullWidth
179+
placeholder="Custom value"
180+
{...field}
181+
value={field.value || ''}
182+
/>
183+
)}
184+
</Field>
185+
</EuiFlexItem>
186+
<EuiFlexItem grow={false}>
187+
<EuiButtonIcon
188+
iconType="trash"
189+
color="danger"
190+
aria-label="Delete row"
191+
onClick={() => arrayHelpers.remove(index)}
192+
/>
193+
</EuiFlexItem>
194+
</EuiFlexGroup>
195+
))}
196+
<EuiSpacer size="s" />
197+
{ /* add new rows with empty values when the add button is clicked. */}
198+
<EuiButtonIcon
199+
iconType="plusInCircle"
200+
onClick={() =>
201+
arrayHelpers.push({ featureName: '', value: 0 })
202+
}
203+
aria-label="Add row"
204+
/>
205+
</>
206+
)}
207+
</FieldArray>
208+
</>
209+
)}
210+
</>
211+
);
212+
}}
213+
</Field>
214+
</>
96215
) : null}
97216
</ContentPanel>
98217
);

‎public/pages/ConfigureModel/containers/ConfigureModel.tsx

+62-2
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ import {
3939
focusOnFirstWrongFeature,
4040
getCategoryFields,
4141
focusOnCategoryField,
42-
getShingleSizeFromObject,
4342
modelConfigurationToFormik,
43+
focusOnImputationOption,
4444
} from '../utils/helpers';
4545
import { formikToDetector } from '../../ReviewAndCreate/utils/helpers';
4646
import { formikToModelConfiguration } from '../utils/helpers';
@@ -53,7 +53,7 @@ import { CoreServicesContext } from '../../../components/CoreServices/CoreServic
5353
import { Detector } from '../../../models/interfaces';
5454
import { prettifyErrorMessage } from '../../../../server/utils/helpers';
5555
import { DetectorDefinitionFormikValues } from '../../DefineDetector/models/interfaces';
56-
import { ModelConfigurationFormikValues } from '../models/interfaces';
56+
import { ModelConfigurationFormikValues, FeaturesFormikValues } from '../models/interfaces';
5757
import { CreateDetectorFormikValues } from '../../CreateDetectorSteps/models/interfaces';
5858
import { DETECTOR_STATE } from '../../../../server/utils/constants';
5959
import { getErrorMessage } from '../../../utils/utils';
@@ -68,6 +68,7 @@ import {
6868
getSavedObjectsClient,
6969
} from '../../../services';
7070
import { DataSourceViewConfig } from '../../../../../../src/plugins/data_source_management/public';
71+
import { SparseDataOptionValue } from '../utils/constants';
7172

7273
interface ConfigureModelRouterProps {
7374
detectorId?: string;
@@ -173,6 +174,49 @@ export function ConfigureModel(props: ConfigureModelProps) {
173174
}
174175
}, [hasError]);
175176

177+
const validateImputationOption = (
178+
formikValues: ModelConfigurationFormikValues,
179+
errors: any
180+
) => {
181+
const imputationOption = get(formikValues, 'imputationOption', null);
182+
183+
// Initialize an array to hold individual error messages
184+
const customValueErrors: string[] = [];
185+
186+
// Validate imputationOption when method is CUSTOM_VALUE
187+
if (imputationOption && imputationOption.imputationMethod === SparseDataOptionValue.CUSTOM_VALUE) {
188+
const enabledFeatures = formikValues.featureList.filter(
189+
(feature: FeaturesFormikValues) => feature.featureEnabled
190+
);
191+
192+
// Validate that the number of custom values matches the number of enabled features
193+
if ((imputationOption.custom_value || []).length !== enabledFeatures.length) {
194+
customValueErrors.push(
195+
`The number of custom values (${(imputationOption.custom_value || []).length}) does not match the number of enabled features (${enabledFeatures.length}).`
196+
);
197+
}
198+
199+
// Validate that each enabled feature has a corresponding custom value
200+
const missingFeatures = enabledFeatures
201+
.map((feature: FeaturesFormikValues) => feature.featureName)
202+
.filter(
203+
(name: string | undefined) =>
204+
!imputationOption.custom_value?.some((cv) => cv.featureName === name)
205+
);
206+
207+
if (missingFeatures.length > 0) {
208+
customValueErrors.push(
209+
`The following enabled features are missing in custom values: ${missingFeatures.join(', ')}.`
210+
);
211+
}
212+
213+
// If there are any custom value errors, join them into a single string with proper formatting
214+
if (customValueErrors.length > 0) {
215+
errors.custom_value = customValueErrors.join(' ');
216+
}
217+
}
218+
};
219+
176220
const handleFormValidation = async (
177221
formikProps: FormikProps<ModelConfigurationFormikValues>
178222
) => {
@@ -185,7 +229,12 @@ export function ConfigureModel(props: ConfigureModelProps) {
185229
formikProps.setFieldTouched('featureList');
186230
formikProps.setFieldTouched('categoryField', isHCDetector);
187231
formikProps.setFieldTouched('shingleSize');
232+
formikProps.setFieldTouched('imputationOption');
233+
188234
formikProps.validateForm().then((errors) => {
235+
// Call the extracted validation method
236+
validateImputationOption(formikProps.values, errors);
237+
189238
if (isEmpty(errors)) {
190239
if (props.isEdit) {
191240
// TODO: possibly add logic to also start RT and/or historical from here. Need to think
@@ -204,11 +253,22 @@ export function ConfigureModel(props: ConfigureModelProps) {
204253
props.setStep(3);
205254
}
206255
} else {
256+
const customValueError = get(errors, 'custom_value')
257+
if (customValueError) {
258+
core.notifications.toasts.addDanger(
259+
customValueError
260+
);
261+
focusOnImputationOption();
262+
return;
263+
}
264+
207265
// TODO: can add focus to all components or possibly customize error message too
208266
if (get(errors, 'featureList')) {
209267
focusOnFirstWrongFeature(errors, formikProps.setFieldTouched);
210268
} else if (get(errors, 'categoryField')) {
211269
focusOnCategoryField();
270+
} else {
271+
console.log(`unexpected error ${JSON.stringify(errors)}`);
212272
}
213273

214274
core.notifications.toasts.addDanger(

‎public/pages/ConfigureModel/models/interfaces.ts

+11
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ export interface ModelConfigurationFormikValues {
1818
categoryFieldEnabled: boolean;
1919
categoryField: string[];
2020
shingleSize: number;
21+
imputationOption?: ImputationFormikValues;
2122
}
2223

2324
export interface FeaturesFormikValues {
@@ -30,3 +31,13 @@ export interface FeaturesFormikValues {
3031
aggregationOf?: AggregationOption[];
3132
newFeature?: boolean;
3233
}
34+
35+
export interface ImputationFormikValues {
36+
imputationMethod?: string;
37+
custom_value?: CustomValueFormikValues[];
38+
}
39+
40+
export interface CustomValueFormikValues {
41+
featureName: string;
42+
data: number;
43+
}

0 commit comments

Comments
 (0)