Skip to content

Commit e913e41

Browse files
ohltylerohltyler
authored
onboard normalization processor (#311)
Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>
1 parent 8f276e3 commit e913e41

File tree

8 files changed

+199
-3
lines changed

8 files changed

+199
-3
lines changed

‎common/constants.ts

+3
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ export enum PROCESSOR_TYPE {
7373
SPLIT = 'split',
7474
SORT = 'sort',
7575
TEXT_CHUNKING = 'text_chunking',
76+
NORMALIZATION = 'normalization-processor',
7677
}
7778

7879
export enum MODEL_TYPE {
@@ -129,6 +130,8 @@ export const TEXT_CHUNKING_PROCESSOR_LINK =
129130
'https://opensearch.org/docs/latest/ingest-pipelines/processors/text-chunking/';
130131
export const CREATE_WORKFLOW_LINK =
131132
'https://opensearch.org/docs/latest/automating-configurations/api/create-workflow/';
133+
export const NORMALIZATION_PROCESSOR_LINK =
134+
'https://opensearch.org/docs/latest/search-plugins/search-pipelines/normalization-processor/';
132135

133136
/**
134137
* Text chunking algorithm constants

‎public/configs/search_response_processors/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@
66
export * from './ml_search_response_processor';
77
export * from './split_search_response_processor';
88
export * from './sort_search_response_processor';
9+
export * from './normalization_processor';

‎public/configs/search_response_processors/normalization_processor.ts

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
import { PROCESSOR_TYPE } from '../../../common';
7+
import { Processor } from '../processor';
8+
9+
/**
10+
* The normalization processor config. Used in search flows.
11+
*/
12+
export class NormalizationProcessor extends Processor {
13+
constructor() {
14+
super();
15+
this.type = PROCESSOR_TYPE.NORMALIZATION;
16+
this.name = 'Normalization Processor';
17+
this.fields = [];
18+
this.optionalFields = [
19+
{
20+
id: 'weights',
21+
type: 'string',
22+
},
23+
{
24+
id: 'normalization_technique',
25+
type: 'select',
26+
selectOptions: ['min_max', 'l2'],
27+
},
28+
{
29+
id: 'combination_technique',
30+
type: 'select',
31+
selectOptions: ['arithmetic_mean', 'geometric_mean', 'harmonic_mean'],
32+
},
33+
{
34+
id: 'description',
35+
type: 'string',
36+
},
37+
{
38+
id: 'tag',
39+
type: 'string',
40+
},
41+
];
42+
}
43+
}

‎public/pages/workflow_detail/workflow_inputs/processor_inputs/normalization_processor_inputs.tsx

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
import React from 'react';
7+
import { EuiAccordion, EuiFlexItem, EuiSpacer } from '@elastic/eui';
8+
import {
9+
IProcessorConfig,
10+
PROCESSOR_CONTEXT,
11+
WorkflowConfig,
12+
NORMALIZATION_PROCESSOR_LINK,
13+
} from '../../../../../common';
14+
import { TextField } from '../input_fields';
15+
import { ConfigFieldList } from '../config_field_list';
16+
17+
interface NormalizationProcessorInputsProps {
18+
uiConfig: WorkflowConfig;
19+
config: IProcessorConfig;
20+
baseConfigPath: string; // the base path of the nested config, if applicable. e.g., 'ingest.enrich'
21+
context: PROCESSOR_CONTEXT;
22+
}
23+
24+
/**
25+
* Specialized component to render the normalization processor. Adds some helper text around weights field.
26+
* In the future, may have a more customizable / guided way for specifying the array of weights.
27+
* For example, could have some visual way of linking it to the underlying sub-queries in the query field,
28+
* enforce its length = the number of queries, etc.
29+
*/
30+
export function NormalizationProcessorInputs(
31+
props: NormalizationProcessorInputsProps
32+
) {
33+
// extracting field info from the config
34+
const optionalFields = props.config.optionalFields || [];
35+
const weightsFieldPath = `${props.baseConfigPath}.${props.config.id}.weights`;
36+
const optionalFieldsWithoutWeights = optionalFields.filter(
37+
(field) => field.id !== 'weights'
38+
);
39+
40+
return (
41+
// We only have optional fields for this processor, so everything is nested under the accordion
42+
<EuiAccordion
43+
id={`advancedSettings${props.config.id}`}
44+
buttonContent="Advanced settings"
45+
paddingSize="none"
46+
>
47+
<EuiSpacer size="s" />
48+
<EuiFlexItem>
49+
<TextField
50+
label={'Weights'}
51+
helpText={`A comma-separated array of floating-point values specifying the weight for each query. For example: '0.8, 0.2'`}
52+
helpLink={NORMALIZATION_PROCESSOR_LINK}
53+
fieldPath={weightsFieldPath}
54+
showError={true}
55+
/>
56+
</EuiFlexItem>
57+
<EuiSpacer size="s" />
58+
<ConfigFieldList
59+
configId={props.config.id}
60+
configFields={optionalFieldsWithoutWeights}
61+
baseConfigPath={props.baseConfigPath}
62+
/>
63+
</EuiAccordion>
64+
);
65+
}

‎public/pages/workflow_detail/workflow_inputs/processor_inputs/processor_inputs.tsx

+15
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
import { MLProcessorInputs } from './ml_processor_inputs';
1616
import { ConfigFieldList } from '../config_field_list';
1717
import { TextChunkingProcessorInputs } from './text_chunking_processor_inputs';
18+
import { NormalizationProcessorInputs } from './normalization_processor_inputs';
1819

1920
/**
2021
* Base component for rendering processor form inputs based on the processor type
@@ -69,6 +70,20 @@ export function ProcessorInputs(props: ProcessorInputsProps) {
6970
);
7071
break;
7172
}
73+
case PROCESSOR_TYPE.NORMALIZATION: {
74+
el = (
75+
<EuiFlexItem>
76+
<NormalizationProcessorInputs
77+
uiConfig={props.uiConfig}
78+
config={props.config}
79+
baseConfigPath={props.baseConfigPath}
80+
context={props.context}
81+
/>
82+
<EuiSpacer size={PROCESSOR_INPUTS_SPACER_SIZE} />
83+
</EuiFlexItem>
84+
);
85+
break;
86+
}
7287
default: {
7388
el = (
7489
<EuiFlexItem>

‎public/pages/workflow_detail/workflow_inputs/processors_list.tsx

+10
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import {
2828
MLIngestProcessor,
2929
MLSearchRequestProcessor,
3030
MLSearchResponseProcessor,
31+
NormalizationProcessor,
3132
SortIngestProcessor,
3233
SortSearchResponseProcessor,
3334
SplitIngestProcessor,
@@ -272,6 +273,15 @@ export function ProcessorsList(props: ProcessorsListProps) {
272273
);
273274
},
274275
},
276+
{
277+
name: 'Normalization Processor',
278+
onClick: () => {
279+
closePopover();
280+
addProcessor(
281+
new NormalizationProcessor().toObj()
282+
);
283+
},
284+
},
275285
],
276286
},
277287
]}

‎public/pages/workflows/new_workflow/quick_configure_inputs.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ export function QuickConfigureInputs(props: QuickConfigureInputsProps) {
6565
<EuiAccordion
6666
id="optionalConfiguration"
6767
buttonContent="Optional configuration"
68-
initialIsOpen={true}
68+
initialIsOpen={false}
6969
>
7070
<EuiSpacer size="m" />
7171
<EuiCompressedFormRow

‎public/utils/config_to_template_utils.ts

+61-2
Original file line numberDiff line numberDiff line change
@@ -116,11 +116,26 @@ function searchConfigToTemplateNodes(
116116
const searchRequestProcessors = processorConfigsToTemplateProcessors(
117117
searchConfig.enrichRequest.processors
118118
);
119+
// For the configured response processors, we don't maintain separate UI / config
120+
// between response processors and phase results processors. So, we parse
121+
// out those different processor types here when configuring the final search pipeline.
122+
// Currently, the only special phase results processor supported is the normalization processor,
123+
// so we filter & partition on that type.
124+
const normalizationProcessor = searchConfig.enrichResponse.processors.find(
125+
(processor) => processor.type === PROCESSOR_TYPE.NORMALIZATION
126+
);
127+
const phaseResultsProcessors = processorConfigsToTemplateProcessors(
128+
normalizationProcessor ? [normalizationProcessor] : []
129+
);
119130
const searchResponseProcessors = processorConfigsToTemplateProcessors(
120-
searchConfig.enrichResponse.processors
131+
searchConfig.enrichResponse.processors.filter(
132+
(processor) => processor.type !== PROCESSOR_TYPE.NORMALIZATION
133+
)
121134
);
122135
const hasProcessors =
123-
searchRequestProcessors.length > 0 || searchResponseProcessors.length > 0;
136+
searchRequestProcessors.length > 0 ||
137+
searchResponseProcessors.length > 0 ||
138+
phaseResultsProcessors.length > 0;
124139

125140
return hasProcessors
126141
? [
@@ -133,6 +148,7 @@ function searchConfigToTemplateNodes(
133148
configurations: JSON.stringify({
134149
request_processors: searchRequestProcessors,
135150
response_processors: searchResponseProcessors,
151+
phase_results_processors: phaseResultsProcessors,
136152
} as SearchPipelineConfig),
137153
},
138154
} as CreateSearchPipelineNode,
@@ -254,6 +270,49 @@ export function processorConfigsToTemplateProcessors(
254270
});
255271
break;
256272
}
273+
// optionally add any parameters specified, in the expected nested format
274+
// for the normalization processor
275+
case PROCESSOR_TYPE.NORMALIZATION: {
276+
const {
277+
normalization_technique,
278+
combination_technique,
279+
weights,
280+
} = processorConfigToFormik(processorConfig);
281+
282+
let finalConfig = {} as any;
283+
if (!isEmpty(normalization_technique)) {
284+
finalConfig = {
285+
...finalConfig,
286+
normalization: {
287+
technique: normalization_technique,
288+
},
289+
};
290+
}
291+
if (!isEmpty(combination_technique)) {
292+
finalConfig = {
293+
...finalConfig,
294+
combination: {
295+
...(finalConfig?.combination || {}),
296+
technique: combination_technique,
297+
},
298+
};
299+
}
300+
if (!isEmpty(weights) && weights.split(',').length > 0) {
301+
finalConfig = {
302+
...finalConfig,
303+
combination: {
304+
...(finalConfig?.combination || {}),
305+
parameters: {
306+
weights: weights.split(',').map(Number),
307+
},
308+
},
309+
};
310+
}
311+
processorsList.push({
312+
[processorConfig.type]: finalConfig,
313+
});
314+
break;
315+
}
257316
case PROCESSOR_TYPE.SPLIT:
258317
case PROCESSOR_TYPE.SORT:
259318
default: {

0 commit comments

Comments
 (0)