Skip to content

Commit d490d15

Browse files
ohltylerohltyler
authored
Integrate with JSONPath; complete input transform (ingest) (opensearch-project#229)
Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>
1 parent 6dc64bb commit d490d15

File tree

9 files changed

+337
-100
lines changed

9 files changed

+337
-100
lines changed

‎common/constants.ts

+1
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ export const FETCH_ALL_QUERY_BODY = {
129129
size: 1000,
130130
};
131131
export const INDEX_NOT_FOUND_EXCEPTION = 'index_not_found_exception';
132+
export const JSONPATH_ROOT_SELECTOR = '$.';
132133

133134
export enum PROCESSOR_CONTEXT {
134135
INGEST = 'ingest',

‎package.json

+3-1
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,13 @@
2727
]
2828
},
2929
"dependencies": {
30+
"@types/jsonpath": "^0.2.4",
3031
"formik": "2.4.2",
3132
"js-yaml": "^4.1.0",
33+
"jsonpath": "^1.1.1",
3234
"reactflow": "^11.8.3",
3335
"yup": "^1.3.2"
3436
},
3537
"devDependencies": {},
3638
"resolutions": {}
37-
}
39+
}

‎public/pages/workflow_detail/workflow_inputs/config_field_list.tsx

-25
Original file line numberDiff line numberDiff line change
@@ -55,31 +55,6 @@ export function ConfigFieldList(props: ConfigFieldListProps) {
5555
);
5656
break;
5757
}
58-
// case 'map': {
59-
// el = (
60-
// <EuiFlexItem key={idx}>
61-
// <MapField
62-
// field={field}
63-
// fieldPath={`${props.baseConfigPath}.${configId}.${field.id}`}
64-
// onFormChange={props.onFormChange}
65-
// />
66-
// <EuiSpacer size={CONFIG_FIELD_SPACER_SIZE} />
67-
// </EuiFlexItem>
68-
// );
69-
// break;
70-
// }
71-
// case 'json': {
72-
// el = (
73-
// <EuiFlexItem key={idx}>
74-
// <JsonField
75-
// label={field.label}
76-
// placeholder={field.placeholder || ''}
77-
// />
78-
// <EuiSpacer size={INPUT_FIELD_SPACER_SIZE} />
79-
// </EuiFlexItem>
80-
// );
81-
// break;
82-
// }
8358
}
8459
return el;
8560
})}

‎public/pages/workflow_detail/workflow_inputs/input_fields/map_field.tsx

+7-6
Original file line numberDiff line numberDiff line change
@@ -28,21 +28,24 @@ interface MapFieldProps {
2828
label: string;
2929
helpLink?: string;
3030
helpText?: string;
31+
keyPlaceholder?: string;
32+
valuePlaceholder?: string;
3133
onFormChange: () => void;
3234
}
3335

3436
/**
3537
* Input component for configuring field mappings
3638
*/
3739
export function MapField(props: MapFieldProps) {
38-
const { setFieldValue, errors, touched } = useFormikContext<
40+
const { setFieldValue, setFieldTouched, errors, touched } = useFormikContext<
3941
WorkflowFormValues
4042
>();
4143

4244
// Adding a map entry to the end of the existing arr
4345
function addMapEntry(curEntries: MapFormValue): void {
4446
const updatedEntries = [...curEntries, { key: '', value: '' } as MapEntry];
4547
setFieldValue(props.fieldPath, updatedEntries);
48+
setFieldTouched(props.fieldPath, true);
4649
props.onFormChange();
4750
}
4851

@@ -54,6 +57,7 @@ export function MapField(props: MapFieldProps) {
5457
const updatedEntries = [...curEntries];
5558
updatedEntries.splice(entryIndexToDelete, 1);
5659
setFieldValue(props.fieldPath, updatedEntries);
60+
setFieldTouched(props.fieldPath, true);
5761
props.onFormChange();
5862
}
5963

@@ -97,10 +101,7 @@ export function MapField(props: MapFieldProps) {
97101
startControl={
98102
<input
99103
type="string"
100-
// TODO: find a way to config/title the placeholder text.
101-
// For example, K/V values have different meanings if input
102-
// map or output map for ML inference processors.
103-
placeholder="Input"
104+
placeholder={props.keyPlaceholder || 'Input'}
104105
className="euiFieldText"
105106
value={mapping.key}
106107
onChange={(e) => {
@@ -119,7 +120,7 @@ export function MapField(props: MapFieldProps) {
119120
endControl={
120121
<input
121122
type="string"
122-
placeholder="Output"
123+
placeholder={props.valuePlaceholder || 'Output'}
123124
className="euiFieldText"
124125
value={mapping.value}
125126
onChange={(e) => {

‎public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx

+108-20
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
*/
55

66
import React, { useState } from 'react';
7-
import { useFormikContext } from 'formik';
7+
import { useFormikContext, getIn } from 'formik';
8+
import { isEmpty, get } from 'lodash';
9+
import jsonpath from 'jsonpath';
810
import {
911
EuiButton,
10-
EuiButtonEmpty,
1112
EuiCodeBlock,
12-
EuiCodeEditor,
1313
EuiFlexGroup,
1414
EuiFlexItem,
1515
EuiModal,
@@ -21,8 +21,10 @@ import {
2121
EuiText,
2222
} from '@elastic/eui';
2323
import {
24+
IConfigField,
2425
IProcessorConfig,
2526
IngestPipelineConfig,
27+
JSONPATH_ROOT_SELECTOR,
2628
PROCESSOR_CONTEXT,
2729
SimulateIngestPipelineDoc,
2830
SimulateIngestPipelineResponse,
@@ -32,13 +34,16 @@ import {
3234
import { formikToIngestPipeline, generateId } from '../../../../utils';
3335
import { simulatePipeline, useAppDispatch } from '../../../../store';
3436
import { getCore } from '../../../../services';
37+
import { MapField } from '../input_fields';
3538

3639
interface InputTransformModalProps {
3740
uiConfig: WorkflowConfig;
3841
config: IProcessorConfig;
3942
context: PROCESSOR_CONTEXT;
43+
inputMapField: IConfigField;
44+
inputMapFieldPath: string;
4045
onClose: () => void;
41-
onConfirm: () => void;
46+
onFormChange: () => void;
4247
}
4348

4449
/**
@@ -50,13 +55,16 @@ export function InputTransformModal(props: InputTransformModalProps) {
5055

5156
// source input / transformed output state
5257
const [sourceInput, setSourceInput] = useState<string>('[]');
53-
const [transformedOutput, setTransformedOutput] = useState<string>('TODO');
58+
const [transformedOutput, setTransformedOutput] = useState<string>('[]');
59+
60+
// parse out the values and determine if there are none/some/all valid jsonpaths
61+
const mapValues = getIn(values, `ingest.enrich.${props.config.id}.inputMap`);
5462

5563
return (
5664
<EuiModal onClose={props.onClose} style={{ width: '70vw' }}>
5765
<EuiModalHeader>
5866
<EuiModalHeaderTitle>
59-
<p>{`Configure input transform`}</p>
67+
<p>{`Configure input`}</p>
6068
</EuiModalHeaderTitle>
6169
</EuiModalHeader>
6270
<EuiModalBody>
@@ -116,24 +124,105 @@ export function InputTransformModal(props: InputTransformModalProps) {
116124
</EuiFlexItem>
117125
<EuiFlexItem>
118126
<>
119-
<EuiText>Define transform with JSONPath</EuiText>
127+
<EuiText>Define transform</EuiText>
128+
<EuiText size="s" color="subdued">
129+
{`Dot notation is used by default. To explicitly use JSONPath, please ensure to prepend with the
130+
root object selector "${JSONPATH_ROOT_SELECTOR}"`}
131+
</EuiText>
120132
<EuiSpacer size="s" />
121-
<EuiCodeEditor
122-
mode="json"
123-
theme="textmate"
124-
value={`TODO`}
125-
readOnly={false}
126-
setOptions={{
127-
fontSize: '12px',
128-
autoScrollEditorIntoView: true,
129-
}}
130-
tabSize={2}
133+
<MapField
134+
field={props.inputMapField}
135+
fieldPath={props.inputMapFieldPath}
136+
label="Input map"
137+
helpText={`An array specifying how to map fields from the ingested document to the model’s input.`}
138+
helpLink={
139+
'https://opensearch.org/docs/latest/ingest-pipelines/processors/ml-inference/#configuration-parameters'
140+
}
141+
keyPlaceholder="Model input field"
142+
valuePlaceholder="Document field"
143+
onFormChange={props.onFormChange}
131144
/>
132145
</>
133146
</EuiFlexItem>
134147
<EuiFlexItem>
135148
<>
136149
<EuiText>Expected output</EuiText>
150+
<EuiButton
151+
style={{ width: '100px' }}
152+
disabled={
153+
isEmpty(mapValues) || isEmpty(JSON.parse(sourceInput))
154+
}
155+
onClick={async () => {
156+
switch (props.context) {
157+
case PROCESSOR_CONTEXT.INGEST: {
158+
if (
159+
!isEmpty(mapValues) &&
160+
!isEmpty(JSON.parse(sourceInput))
161+
) {
162+
let output = {};
163+
let sampleSourceInput = {};
164+
try {
165+
sampleSourceInput = JSON.parse(sourceInput)[0];
166+
} catch {}
167+
168+
mapValues.forEach(
169+
(mapValue: { key: string; value: string }) => {
170+
const path = mapValue.value;
171+
try {
172+
let transformedResult = undefined;
173+
// ML inference processors will use standard dot notation or JSONPath depending on the input.
174+
// We follow the same logic here to generate consistent results.
175+
if (
176+
mapValue.value.startsWith(
177+
JSONPATH_ROOT_SELECTOR
178+
)
179+
) {
180+
// JSONPath transform
181+
transformedResult = jsonpath.query(
182+
sampleSourceInput,
183+
path
184+
);
185+
// Bracket notation not supported - throw an error
186+
} else if (
187+
mapValue.value.includes(']') ||
188+
mapValue.value.includes(']')
189+
) {
190+
throw new Error();
191+
// Standard dot notation
192+
} else {
193+
transformedResult = get(
194+
sampleSourceInput,
195+
path
196+
);
197+
}
198+
199+
output = {
200+
...output,
201+
[mapValue.key]: transformedResult || '',
202+
};
203+
204+
setTransformedOutput(
205+
JSON.stringify(output, undefined, 2)
206+
);
207+
} catch (e: any) {
208+
console.error(e);
209+
getCore().notifications.toasts.addDanger(
210+
'Error generating expected output. Ensure your inputs are valid JSONPath or dot notation syntax.',
211+
e
212+
);
213+
}
214+
}
215+
);
216+
}
217+
218+
break;
219+
}
220+
// TODO: complete for search request / search response contexts
221+
}
222+
}}
223+
>
224+
Generate
225+
</EuiButton>
137226
<EuiSpacer size="s" />
138227
<EuiCodeBlock fontSize="m" isCopyable={false}>
139228
{transformedOutput}
@@ -143,9 +232,8 @@ export function InputTransformModal(props: InputTransformModalProps) {
143232
</EuiFlexGroup>
144233
</EuiModalBody>
145234
<EuiModalFooter>
146-
<EuiButtonEmpty onClick={props.onClose}>Cancel</EuiButtonEmpty>
147-
<EuiButton onClick={props.onConfirm} fill={true} color="primary">
148-
Save
235+
<EuiButton onClick={props.onClose} fill={false} color="primary">
236+
Close
149237
</EuiButton>
150238
</EuiModalFooter>
151239
</EuiModal>

0 commit comments

Comments
 (0)