Switch to jsonlines in all components and parsing

ohltyler · ohltyler · commit d52b16bab2ef · 2025-02-21T10:47:43.000-08:00
Signed-off-by: Tyler Ohlsen &lt;ohltyler@amazon.com&gt;
diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx
@@ -44,7 +44,8 @@ export function SourceData(props: SourceDataProps) {
   // empty/populated docs state
   let docs = [];
   try {
-    docs = JSON.parse(getIn(values, 'ingest.docs', []));
+    const lines = getIn(values, 'ingest.docs', '').split('\n') as string[];
+    lines.forEach((line) => docs.push(JSON.parse(line)));
   } catch {}
   const docsPopulated = docs.length > 0;
 
diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx
@@ -22,9 +22,10 @@ import {
   EuiButtonGroup,
   EuiCompressedComboBox,
 } from '@elastic/eui';
-import { JsonField } from '../input_fields';
+import { JsonLinesField } from '../input_fields';
 import {
   customStringify,
+  customStringifySingleLine,
   FETCH_ALL_QUERY_LARGE,
   IConfigField,
   IndexMappings,
@@ -177,8 +178,14 @@ export function SourceDataModal(props: SourceDataProps) {
               .then((resp) => {
                 const docObjs = resp?.hits?.hits
                   ?.slice(0, MAX_DOCS_TO_IMPORT)
-                  ?.map((hit: SearchHit) => hit?._source);
-                formikProps.setFieldValue('docs', customStringify(docObjs));
+                  ?.map((hit: SearchHit) => hit?._source) as {}[];
+                let jsonLinesStr = '';
+                try {
+                  docObjs.forEach((docObj) => {
+                    jsonLinesStr += customStringifySingleLine(docObj) + '\n';
+                  });
+                } catch {}
+                formikProps.setFieldValue('docs', jsonLinesStr);
               });
           }
         }, [selectedIndex]);
@@ -287,14 +294,13 @@ export function SourceDataModal(props: SourceDataProps) {
                     <EuiSpacer size="xs" />
                   </>
                 )}
-                <JsonField
+                <JsonLinesField
                   label="Documents to be imported"
                   fieldPath={'docs'}
                   helpText="Documents must be in JSON lines format."
                   editorHeight="40vh"
                   readOnly={false}
                   validate={true}
-                  validateInline={true}
                 />
               </>
             </EuiModalBody>
diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/json_lines_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/json_lines_field.tsx
@@ -20,7 +20,6 @@ import { camelCaseToTitleString } from '../../../../utils';
 interface JsonLinesFieldProps {
   fieldPath: string; // the full path in string-form to the field (e.g., 'ingest.enrich.processors.text_embedding_processor.inputField')
   validate?: boolean;
-  validateInline?: boolean;
   label?: string;
   helpLink?: string;
   helpText?: string;
@@ -34,12 +33,14 @@ interface JsonLinesFieldProps {
  */
 export function JsonLinesField(props: JsonLinesFieldProps) {
   const validate = props.validate ?? true;
-  const validateInline = props.validateInline ?? true;
 
   const { errors, touched, values } = useFormikContext<WorkflowFormValues>();
 
   // temp input state. only format when users click out of the code editor
   const [jsonStr, setJsonStr] = useState<string>('{}');
+  const [customErrMsg, setCustomErrMsg] = useState<string | undefined>(
+    undefined
+  );
 
   // initializing the text to be the stringified form value
   useEffect(() => {
@@ -54,6 +55,16 @@ export function JsonLinesField(props: JsonLinesFieldProps) {
   return (
     <Field name={props.fieldPath}>
       {({ field, form }: FieldProps) => {
+        let errMsgToDisplay = undefined as string | undefined;
+        if (validate) {
+          if (customErrMsg) {
+            errMsgToDisplay = customErrMsg;
+          } else if (getIn(errors, field.name)) {
+            errMsgToDisplay = getIn(errors, field.name);
+          } else {
+            errMsgToDisplay = undefined;
+          }
+        }
         return (
           <EuiCompressedFormRow
             fullWidth={true}
@@ -69,75 +80,62 @@ export function JsonLinesField(props: JsonLinesFieldProps) {
               ) : undefined
             }
             helpText={props.helpText || undefined}
-            error={validate ? getIn(errors, field.name) : undefined}
+            error={errMsgToDisplay}
             isInvalid={
               validate
                 ? getIn(errors, field.name) && getIn(touched, field.name)
                 : false
             }
           >
             <EuiCodeEditor
-              mode="json"
+              mode="hjson"
               theme="textmate"
               width="100%"
               height={props.editorHeight || '15vh'}
               value={jsonStr}
               onChange={(input) => {
                 setJsonStr(input);
                 form.setFieldValue(field.name, input);
+                setCustomErrMsg(undefined);
               }}
               onBlur={() => {
+                form.setFieldTouched(field.name);
                 let finalJsonStr = '';
+                let curIdx = -1;
                 try {
-                  jsonStr?.split('\n').forEach((line: string) => {
+                  const lines = jsonStr?.split('\n');
+                  lines.forEach((line: string, idx) => {
+                    curIdx = idx;
                     if (line.trim() !== '') {
-                      try {
-                        finalJsonStr +=
-                          customStringifySingleLine(JSON.parse(line)) + '\n';
-                      } catch (e) {}
+                      finalJsonStr +=
+                        customStringifySingleLine(JSON.parse(line)) + '\n';
                     }
                   });
-                } catch (error) {}
-
-                console.log('final json str: ', finalJsonStr);
-                form.setFieldValue(field.name, finalJsonStr);
-                form.setFieldTouched(field.name);
-
-                // TODO: format under regular scenario, leave alone if JSONLines scenario
-                // try {
-                //   form.setFieldValue(
-                //     field.name,
-                //     customStringify(JSON.parse(jsonStr))
-                //   );
-                // } catch (error) {
-                //   form.setFieldValue(field.name, jsonStr);
-                // } finally {
-                //   form.setFieldTouched(field.name);
-                // }
+                  // remove trailing newline
+                  if (finalJsonStr !== '') {
+                    finalJsonStr = finalJsonStr.slice(0, -1);
+                  }
+                  form.setFieldValue(field.name, finalJsonStr);
+                } catch (error) {
+                  setCustomErrMsg(
+                    `Error on line ${curIdx + 1}: ${getIn(
+                      error,
+                      'message',
+                      'Invalid JSON'
+                    ).replace(/^(.*?)\s+in JSON.*/, '$1')}`
+                  );
+                }
               }}
               readOnly={props.readOnly || false}
               setOptions={{
                 fontSize: '14px',
-                useWorker: validateInline,
+                useWorker: false,
                 highlightActiveLine: !props.readOnly,
                 highlightSelectedWord: !props.readOnly,
                 highlightGutterLine: !props.readOnly,
                 wrap: true,
               }}
               aria-label="Code Editor"
-              tabSize={2}
-              // onValidate={(props) => {
-              //   console.log('validate props: ', props);
-              //   return [];
-              // }}
-              // annotations={[
-              //   {
-              //     column: 0,
-              //     row: 1,
-              //     text: 'Invalid JSON',
-              //     type: 'error',
-              //   },
-              // ]}
             />
           </EuiCompressedFormRow>
         );
diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_expression_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_expression_modal.tsx
@@ -123,7 +123,8 @@ export function ConfigureExpressionModal(props: ConfigureExpressionModalProps) {
   const docs = getIn(values, 'ingest.docs');
   let docObjs = [] as {}[] | undefined;
   try {
-    docObjs = JSON.parse(docs);
+    const lines = docs?.split('\n') as string[];
+    lines.forEach((line) => docObjs?.push(JSON.parse(line)));
   } catch {}
   const query = getIn(values, 'search.request');
   let queryObj = {} as {} | undefined;
diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_multi_expression_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_multi_expression_modal.tsx
@@ -129,7 +129,8 @@ export function ConfigureMultiExpressionModal(
   const docs = getIn(values, 'ingest.docs');
   let docObjs = [] as {}[] | undefined;
   try {
-    docObjs = JSON.parse(docs);
+    const lines = docs?.split('\n') as string[];
+    lines.forEach((line) => docObjs?.push(JSON.parse(line)));
   } catch {}
   const query = getIn(values, 'search.request');
   let queryObj = {} as {} | undefined;
diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_template_modal.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/modals/configure_template_modal.tsx
@@ -149,7 +149,8 @@ export function ConfigureTemplateModal(props: ConfigureTemplateModalProps) {
   const docs = getIn(values, 'ingest.docs');
   let docObjs = [] as {}[] | undefined;
   try {
-    docObjs = JSON.parse(docs);
+    const lines = docs?.split('\n') as string[];
+    lines.forEach((line) => docObjs?.push(JSON.parse(line)));
   } catch {}
   const query = getIn(values, 'search.request');
   let queryObj = {} as {} | undefined;
diff --git a/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/workflow_inputs.tsx
@@ -275,8 +275,9 @@ export function WorkflowInputs(props: WorkflowInputsProps) {
   useEffect(() => {
     let parsedDocsObjs = [] as {}[];
     try {
-      parsedDocsObjs = JSON.parse(props.ingestDocs);
-    } catch (e) {}
+      const lines = props.ingestDocs?.split('\n') as string[];
+      lines.forEach((line) => parsedDocsObjs.push(JSON.parse(line)));
+    } catch {}
     setDocsPopulated(parsedDocsObjs.length > 0 && !isEmpty(parsedDocsObjs[0]));
   }, [props.ingestDocs]);
 
@@ -607,7 +608,8 @@ export function WorkflowInputs(props: WorkflowInputsProps) {
     try {
       let ingestDocsObjs = [] as {}[];
       try {
-        ingestDocsObjs = JSON.parse(props.ingestDocs);
+        const lines = props.ingestDocs?.split('\n') as string[];
+        lines.forEach((line) => ingestDocsObjs.push(JSON.parse(line)));
       } catch (e) {}
       if (ingestDocsObjs.length > 0 && !isEmpty(ingestDocsObjs[0])) {
         success = await validateAndUpdateWorkflow(false, true, false);
diff --git a/public/utils/utils.tsx b/public/utils/utils.tsx
@@ -176,6 +176,8 @@ export function prepareDocsForSimulate(
   let docObjs = [] as {}[];
   try {
     docObjs = JSON.parse(docs) as {}[];
+    const lines = docs?.split('\n') as string[];
+    lines.forEach((line) => docObjs.push(JSON.parse(line)));
   } catch {}
   docObjs.forEach((doc) => {
     preparedDocs.push({
