Skip to content

Commit c9dd036

Browse files
ohltylerohltyler
authored
Add fine-grained error handling; misc bug fixes (#598)
1 parent 2a61016 commit c9dd036

19 files changed

+504
-82
lines changed

‎common/interfaces.ts

+45
Original file line numberDiff line numberDiff line change
@@ -568,6 +568,10 @@ export type CachedFormikState = {
568568
touched?: {};
569569
};
570570

571+
export type IngestPipelineErrors = {
572+
[idx: number]: { processorType: string; errorMsg: string };
573+
};
574+
571575
/**
572576
********** OPENSEARCH TYPES/INTERFACES ************
573577
*/
@@ -596,6 +600,24 @@ export type SimulateIngestPipelineResponse = {
596600
docs: SimulateIngestPipelineDocResponse[];
597601
};
598602

603+
// verbose mode
604+
// from https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest/#query-parameters
605+
export type SimulateIngestPipelineDocResponseVerbose = SimulateIngestPipelineDocResponse & {
606+
processor_type: string;
607+
status: 'success' | 'error';
608+
description?: string;
609+
};
610+
611+
// verbose mode
612+
// from https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest/#query-parameters
613+
export type SimulateIngestPipelineResponseVerbose = {
614+
docs: [
615+
{
616+
processor_results: SimulateIngestPipelineDocResponseVerbose[];
617+
}
618+
];
619+
};
620+
599621
export type SearchHit = SimulateIngestPipelineDoc;
600622

601623
export type SearchResponse = {
@@ -619,6 +641,29 @@ export type SearchResponse = {
619641
ext?: {};
620642
};
621643

644+
export type SearchProcessorInputData = {
645+
_index: string;
646+
_id: string;
647+
_score: number;
648+
_source: {};
649+
};
650+
export type SearchProcessorOutputData = SearchProcessorInputData;
651+
652+
export type SearchProcessorResult = {
653+
processor_name: string;
654+
duration_millis: number;
655+
status: 'success' | 'fail';
656+
error?: string;
657+
input_data: SearchProcessorInputData[] | null;
658+
output_data: SearchProcessorOutputData[] | null;
659+
};
660+
661+
export type SearchResponseVerbose = SearchResponse & {
662+
processor_results: SearchProcessorResult[];
663+
};
664+
665+
export type SearchPipelineErrors = IngestPipelineErrors;
666+
622667
export type IndexResponse = {
623668
indexName: string;
624669
indexDetails: IndexConfiguration;

‎public/configs/ingest_processors/split_ingest_processor.ts

+8
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,13 @@ export class SplitIngestProcessor extends SplitProcessor {
1313
constructor() {
1414
super();
1515
this.id = generateId('split_processor_ingest');
16+
this.optionalFields = [
17+
...(this.optionalFields || []),
18+
{
19+
id: 'ignore_missing',
20+
type: 'boolean',
21+
value: false,
22+
},
23+
];
1624
}
1725
}

‎public/configs/ingest_processors/text_chunking_ingest_processor.ts

+5
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ export class TextChunkingIngestProcessor extends Processor {
7070
id: 'tag',
7171
type: 'string',
7272
},
73+
{
74+
id: 'ignore_missing',
75+
type: 'boolean',
76+
value: false,
77+
},
7378
];
7479
}
7580
}

‎public/configs/sort_processor.ts

+5
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@ export abstract class SortProcessor extends Processor {
4040
id: 'tag',
4141
type: 'string',
4242
},
43+
{
44+
id: 'ignore_failure',
45+
type: 'boolean',
46+
value: false,
47+
},
4348
];
4449
}
4550
}

‎public/configs/split_processor.ts

+5
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ export abstract class SplitProcessor extends Processor {
4545
id: 'tag',
4646
type: 'string',
4747
},
48+
{
49+
id: 'ignore_failure',
50+
type: 'boolean',
51+
value: false,
52+
},
4853
];
4954
}
5055
}

‎public/pages/workflow_detail/tools/query/query.tsx

+47-14
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,23 @@ import {
2323
FETCH_ALL_QUERY,
2424
QueryParam,
2525
SearchResponse,
26+
SearchResponseVerbose,
2627
WorkflowFormValues,
2728
} from '../../../../../common';
28-
import { AppState, searchIndex, useAppDispatch } from '../../../../store';
29+
import {
30+
AppState,
31+
searchIndex,
32+
setOpenSearchError,
33+
setSearchPipelineErrors,
34+
useAppDispatch,
35+
} from '../../../../store';
2936
import {
3037
containsEmptyValues,
3138
containsSameValues,
39+
formatSearchPipelineErrors,
3240
getDataSourceId,
3341
getPlaceholdersFromQuery,
42+
getSearchPipelineErrors,
3443
injectParameters,
3544
} from '../../../../utils';
3645
import { QueryParamsList, Results } from '../../../../general_components';
@@ -166,9 +175,7 @@ export function Query(props: QueryProps) {
166175
: [SEARCH_OPTIONS[1]]
167176
}
168177
selectedOptions={
169-
props.hasSearchPipeline &&
170-
includePipeline &&
171-
props.selectedStep === CONFIG_STEP.SEARCH
178+
includePipeline
172179
? [SEARCH_OPTIONS[0]]
173180
: [SEARCH_OPTIONS[1]]
174181
}
@@ -194,23 +201,49 @@ export function Query(props: QueryProps) {
194201
apiBody: {
195202
index: indexToSearch,
196203
body: injectParameters(queryParams, tempRequest),
197-
searchPipeline:
198-
props.hasSearchPipeline &&
199-
includePipeline &&
200-
props.selectedStep === CONFIG_STEP.SEARCH &&
201-
!isEmpty(values?.search?.pipelineName)
202-
? values?.search?.pipelineName
203-
: '_none',
204+
searchPipeline: includePipeline
205+
? values?.search?.pipelineName
206+
: '_none',
204207
},
205208
dataSourceId,
209+
verbose: includePipeline,
206210
})
207211
)
208212
.unwrap()
209-
.then(async (resp: SearchResponse) => {
210-
setQueryResponse(resp);
211-
})
213+
.then(
214+
async (
215+
resp: SearchResponse | SearchResponseVerbose
216+
) => {
217+
if (includePipeline) {
218+
const searchPipelineErrors = getSearchPipelineErrors(
219+
resp as SearchResponseVerbose
220+
);
221+
// The errors map may be empty; in which case, this dispatch will clear
222+
// any older errors.
223+
dispatch(
224+
setSearchPipelineErrors({
225+
errors: searchPipelineErrors,
226+
})
227+
);
228+
if (!isEmpty(searchPipelineErrors)) {
229+
dispatch(
230+
setOpenSearchError({
231+
error: `Error running search pipeline. ${formatSearchPipelineErrors(
232+
searchPipelineErrors
233+
)}`,
234+
})
235+
);
236+
}
237+
} else {
238+
setSearchPipelineErrors({ errors: {} });
239+
}
240+
241+
setQueryResponse(resp);
242+
}
243+
)
212244
.catch((error: any) => {
213245
setQueryResponse(undefined);
246+
setSearchPipelineErrors({ errors: {} });
214247
console.error('Error running query: ', error);
215248
});
216249
}}

‎public/pages/workflow_detail/workflow_inputs/processors_list.tsx

+43-4
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
*/
55

66
import React, { useEffect, useState } from 'react';
7+
import { useSelector } from 'react-redux';
78
import semver from 'semver';
8-
import { getEffectiveVersion } from '../../../pages/workflows/new_workflow/new_workflow';
99
import {
1010
EuiSmallButtonEmpty,
1111
EuiSmallButtonIcon,
@@ -29,7 +29,7 @@ import {
2929
WorkflowFormValues,
3030
} from '../../../../common';
3131
import { formikToUiConfig, getDataSourceFromURL } from '../../../utils';
32-
32+
import { getEffectiveVersion } from '../../../pages/workflows/new_workflow/new_workflow';
3333
import {
3434
CollapseProcessor,
3535
CopyIngestProcessor,
@@ -53,6 +53,12 @@ import {
5353
MIN_SUPPORTED_VERSION,
5454
MINIMUM_FULL_SUPPORTED_VERSION,
5555
} from '../../../../common';
56+
import {
57+
AppState,
58+
setIngestPipelineErrors,
59+
setSearchPipelineErrors,
60+
useAppDispatch,
61+
} from '../../../store';
5662

5763
interface ProcessorsListProps {
5864
uiConfig: WorkflowConfig;
@@ -67,13 +73,26 @@ const PANEL_ID = 0;
6773
* General component for configuring pipeline processors (ingest / search request / search response)
6874
*/
6975
export function ProcessorsList(props: ProcessorsListProps) {
76+
const dispatch = useAppDispatch();
77+
const {
78+
ingestPipeline: ingestPipelineErrors,
79+
searchPipeline: searchPipelineErrors,
80+
} = useSelector((state: AppState) => state.errors);
7081
const { values, errors, touched } = useFormikContext<WorkflowFormValues>();
7182
const [version, setVersion] = useState<string>('');
7283
const location = useLocation();
7384
const [processorAdded, setProcessorAdded] = useState<boolean>(false);
7485
const [isPopoverOpen, setPopover] = useState(false);
7586
const [processors, setProcessors] = useState<IProcessorConfig[]>([]);
7687

88+
function clearProcessorErrors(): void {
89+
if (props.context === PROCESSOR_CONTEXT.INGEST) {
90+
dispatch(setIngestPipelineErrors({ errors: {} }));
91+
} else {
92+
dispatch(setSearchPipelineErrors({ errors: {} }));
93+
}
94+
}
95+
7796
const closePopover = () => {
7897
setPopover(false);
7998
};
@@ -258,6 +277,7 @@ export function ProcessorsList(props: ProcessorsListProps) {
258277
// the list of processors. Additionally, persist any current form state
259278
// (touched, errors) so they are re-initialized when the form is reset.
260279
function addProcessor(processor: IProcessorConfig): void {
280+
clearProcessorErrors();
261281
props.setCachedFormikState({
262282
errors,
263283
touched,
@@ -295,6 +315,7 @@ export function ProcessorsList(props: ProcessorsListProps) {
295315
// (getting any updated/interim values along the way) delete
296316
// the specified processor from the list of processors
297317
function deleteProcessor(processorIdToDelete: string): void {
318+
clearProcessorErrors();
298319
const existingConfig = cloneDeep(props.uiConfig as WorkflowConfig);
299320
let newConfig = formikToUiConfig(values, existingConfig);
300321
switch (props.context) {
@@ -341,6 +362,23 @@ export function ProcessorsList(props: ProcessorsListProps) {
341362
}
342363
} catch (e) {}
343364

365+
const processorFormError =
366+
hasErrors && allTouched
367+
? 'Invalid or missing fields detected'
368+
: undefined;
369+
const processorRuntimeError =
370+
props.context === PROCESSOR_CONTEXT.INGEST
371+
? getIn(
372+
ingestPipelineErrors,
373+
`${processorIndex}.errorMsg`,
374+
undefined
375+
)
376+
: getIn(
377+
searchPipelineErrors,
378+
`${processorIndex}.errorMsg`,
379+
undefined
380+
);
381+
344382
return (
345383
<EuiFlexItem key={processorIndex}>
346384
<EuiPanel paddingSize="s">
@@ -354,14 +392,15 @@ export function ProcessorsList(props: ProcessorsListProps) {
354392
<EuiFlexItem grow={false}>
355393
<EuiText>{`${processor.name || 'Processor'}`}</EuiText>
356394
</EuiFlexItem>
357-
{hasErrors && allTouched && (
395+
{(processorFormError !== undefined ||
396+
processorRuntimeError !== undefined) && (
358397
<EuiFlexItem grow={false}>
359398
<EuiIconTip
360399
aria-label="Warning"
361400
size="m"
362401
type="alert"
363402
color="danger"
364-
content="Invalid or missing fields detected"
403+
content={processorFormError || processorRuntimeError}
365404
position="right"
366405
/>
367406
</EuiFlexItem>

0 commit comments

Comments
 (0)