diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index 5da7fc64..284c802e 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -27,6 +27,7 @@ import { } from '../../../../../common'; import { SourceDataModal } from './source_data_modal'; import { BulkPopoverContent } from './bulk_popover_content'; +import { getObjsFromJSONLines } from '../../../../utils'; interface SourceDataProps { workflow: Workflow | undefined; @@ -42,11 +43,7 @@ export function SourceData(props: SourceDataProps) { const { values, setFieldValue } = useFormikContext(); // empty/populated docs state - let docs = []; - try { - const lines = getIn(values, 'ingest.docs', '').split('\n') as string[]; - lines.forEach((line) => docs.push(JSON.parse(line))); - } catch {} + const docs = getObjsFromJSONLines(getIn(values, 'ingest.docs', '')); const docsPopulated = docs.length > 0; // selected option state diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_expression_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_expression_modal.tsx index 6561e9e5..ff412e79 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_expression_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_expression_modal.tsx @@ -49,6 +49,7 @@ import { generateTransform, getDataSourceId, getInitialValue, + getObjsFromJSONLines, getPlaceholdersFromQuery, injectParameters, prepareDocsForSimulate, @@ -128,11 +129,7 @@ export function ConfigureExpressionModal(props: ConfigureExpressionModalProps) { `${props.baseConfigPath}.${props.config.id}.one_to_one` ); const docs = getIn(values, 'ingest.docs'); - let docObjs = [] as {}[] | undefined; - try { - const lines = docs?.split('\n') as string[]; - lines.forEach((line) => docObjs?.push(JSON.parse(line))); - } catch {} + const docObjs = getObjsFromJSONLines(docs); const query = getIn(values, 'search.request'); let queryObj = {} as {} | undefined; try { @@ -475,12 +472,8 @@ export function ConfigureExpressionModal(props: ConfigureExpressionModalProps) { }); } else { try { - const docObjs = [] as {}[]; - const lines = values?.ingest?.docs?.split( - '\n' - ) as string[]; - lines.forEach((line) => - docObjs?.push(JSON.parse(line)) + const docObjs = getObjsFromJSONLines( + values?.ingest?.docs ); if (docObjs.length > 0) { setSourceInput( diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_multi_expression_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_multi_expression_modal.tsx index 45b336a4..318d9d59 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_multi_expression_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_multi_expression_modal.tsx @@ -47,6 +47,7 @@ import { formikToPartialPipeline, generateTransform, getDataSourceId, + getObjsFromJSONLines, getPlaceholdersFromQuery, injectParameters, prepareDocsForSimulate, @@ -134,11 +135,7 @@ export function ConfigureMultiExpressionModal( // get some current form values const docs = getIn(values, 'ingest.docs'); - let docObjs = [] as {}[] | undefined; - try { - const lines = docs?.split('\n') as string[]; - lines.forEach((line) => docObjs?.push(JSON.parse(line))); - } catch {} + const docObjs = getObjsFromJSONLines(docs); const query = getIn(values, 'search.request'); let queryObj = {} as {} | undefined; try { diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_template_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_template_modal.tsx index 680cfb37..eea72901 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_template_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_template_modal.tsx @@ -53,6 +53,7 @@ import { generateTransform, getDataSourceId, getInitialValue, + getObjsFromJSONLines, getPlaceholdersFromQuery, injectParameters, prepareDocsForSimulate, @@ -154,11 +155,7 @@ export function ConfigureTemplateModal(props: ConfigureTemplateModalProps) { `${props.baseConfigPath}.${props.config.id}.one_to_one` ); const docs = getIn(values, 'ingest.docs'); - let docObjs = [] as {}[] | undefined; - try { - const lines = docs?.split('\n') as string[]; - lines.forEach((line) => docObjs?.push(JSON.parse(line))); - } catch {} + const docObjs = getObjsFromJSONLines(docs); const query = getIn(values, 'search.request'); let queryObj = {} as {} | undefined; try { @@ -706,9 +703,9 @@ export function ConfigureTemplateModal(props: ConfigureTemplateModalProps) { }); } else { try { - const docObjs = JSON.parse( - values.ingest.docs - ) as {}[]; + const docObjs = getObjsFromJSONLines( + values?.ingest?.docs + ); if (docObjs.length > 0) { setSourceInput( customStringify(docObjs[0]) diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx index 563b519d..58b761d0 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/model_inputs.tsx @@ -46,6 +46,7 @@ import { import { AppState, getMappings, useAppDispatch } from '../../../../../store'; import { getDataSourceId, + getObjsFromJSONLines, parseModelInputs, sanitizeJSONPath, } from '../../../../../utils'; @@ -126,9 +127,10 @@ export function ModelInputs(props: ModelInputsProps) { >([]); useEffect(() => { try { - const docObjKeys = Object.keys( - flattie((JSON.parse(values.ingest.docs) as {}[])[0]) + const ingestDocsObjs = getObjsFromJSONLines( + getIn(values, 'ingest.docs', '') ); + const docObjKeys = Object.keys(flattie(ingestDocsObjs[0])); if (docObjKeys.length > 0) { setDocFields( docObjKeys.map((key) => { diff --git a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx index 16be17e2..e99f1f0e 100644 --- a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx @@ -61,6 +61,7 @@ import { useDataSourceVersion, getIsPreV219, useMissingDataSourceVersion, + getObjsFromJSONLines, } from '../../../utils'; import { BooleanField } from './input_fields'; import '../workspace/workspace-styles.scss'; @@ -271,11 +272,7 @@ export function WorkflowInputs(props: WorkflowInputsProps) { // populated ingest docs state const [docsPopulated, setDocsPopulated] = useState(false); useEffect(() => { - let parsedDocsObjs = [] as {}[]; - try { - const lines = props.ingestDocs?.split('\n') as string[]; - lines.forEach((line) => parsedDocsObjs.push(JSON.parse(line))); - } catch {} + const parsedDocsObjs = getObjsFromJSONLines(props.ingestDocs); setDocsPopulated(parsedDocsObjs.length > 0 && !isEmpty(parsedDocsObjs[0])); }, [props.ingestDocs]); @@ -604,11 +601,7 @@ export function WorkflowInputs(props: WorkflowInputsProps) { props.setIsRunningIngest(true); let success = false; try { - let ingestDocsObjs = [] as {}[]; - try { - const lines = props.ingestDocs?.split('\n') as string[]; - lines.forEach((line) => ingestDocsObjs.push(JSON.parse(line))); - } catch (e) {} + const ingestDocsObjs = getObjsFromJSONLines(props.ingestDocs); if (ingestDocsObjs.length > 0 && !isEmpty(ingestDocsObjs[0])) { success = await validateAndUpdateWorkflow(false, true, false); if (success) { diff --git a/public/utils/utils.tsx b/public/utils/utils.tsx index f176e422..22c17389 100644 --- a/public/utils/utils.tsx +++ b/public/utils/utils.tsx @@ -209,11 +209,7 @@ export function prepareDocsForSimulate( indexName: string ): SimulateIngestPipelineDoc[] { const preparedDocs = [] as SimulateIngestPipelineDoc[]; - let docObjs = [] as {}[]; - try { - const lines = docs?.split('\n') as string[]; - lines.forEach((line) => docObjs.push(JSON.parse(line))); - } catch {} + const docObjs = getObjsFromJSONLines(docs); docObjs?.forEach((doc) => { preparedDocs.push({ _index: indexName, @@ -224,6 +220,17 @@ export function prepareDocsForSimulate( return preparedDocs; } +// Utility fn to transform a raw JSON Lines string into an arr of JSON objs +// for easier downstream parsing +export function getObjsFromJSONLines(jsonLines: string | undefined): {}[] { + let objs = [] as {}[]; + try { + const lines = jsonLines?.split('\n') as string[]; + lines.forEach((line) => objs.push(JSON.parse(line))); + } catch {} + return objs; +} + // Docs are returned in a certain format from the simulate ingest pipeline API. We want // to format them into a more readable string to display export function unwrapTransformedDocs(