Add new RAG + hybrid search preset

common/constants.ts

@@ -166,8 +166,8 @@ export enum WORKFLOW_TYPE {
   SEMANTIC_SEARCH = 'Semantic Search',
   MULTIMODAL_SEARCH = 'Multimodal Search',
   HYBRID_SEARCH = 'Hybrid Search',
-  RAG = 'RAG with Lexical Retrieval',
   VECTOR_SEARCH_WITH_RAG = 'RAG with Vector Retrieval',
+  HYBRID_SEARCH_WITH_RAG = 'RAG with Hybrid Search',
   CUSTOM = 'Custom Search',
   UNKNOWN = 'Unknown',
 }

common/utils.ts

@@ -4,7 +4,7 @@
  */
 
 import moment from 'moment';
-import { DATE_FORMAT_PATTERN, WORKFLOW_TYPE, Workflow } from './';
+import { DATE_FORMAT_PATTERN, WORKFLOW_TYPE } from './';
 import { isEmpty } from 'lodash';
 
 export function toFormattedDate(timestampMillis: number): String {
@@ -44,14 +44,25 @@ export function customStringifySingleLine(jsonObj: {}): string {
   return JSON.stringify(jsonObj, undefined, 0);
 }
 
-export function isVectorSearchUseCase(workflow: Workflow | undefined): boolean {
+export function isVectorSearchUseCase(workflowType?: WORKFLOW_TYPE): boolean {
   return (
-    workflow?.ui_metadata?.type !== undefined &&
+    workflowType !== undefined &&
     [
-      WORKFLOW_TYPE.HYBRID_SEARCH,
-      WORKFLOW_TYPE.MULTIMODAL_SEARCH,
       WORKFLOW_TYPE.SEMANTIC_SEARCH,
+      WORKFLOW_TYPE.MULTIMODAL_SEARCH,
+      WORKFLOW_TYPE.HYBRID_SEARCH,
+      WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG,
+      WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG,
+    ].includes(workflowType)
+  );
+}
+
+export function isRAGUseCase(workflowType?: WORKFLOW_TYPE): boolean {
+  return (
+    workflowType !== undefined &&
+    [
       WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG,
-    ].includes(workflow?.ui_metadata?.type)
+      WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG,
+    ].includes(workflowType)
   );
 }

public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx

@@ -68,7 +68,7 @@ export function SourceData(props: SourceDataProps) {
 
     // try to clear out any default values for the ML ingest processor, if applicable
     if (
-      isVectorSearchUseCase(props.workflow) &&
+      isVectorSearchUseCase(props.workflow?.ui_metadata?.type) &&
       isEditModalOpen &&
       selectedOption !== SOURCE_OPTIONS.EXISTING_INDEX
     ) {

public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data_modal.tsx

@@ -112,7 +112,7 @@ export function SourceDataModal(props: SourceDataProps) {
 
     // 2. Update several form values if an index is selected (and if vector search)
     if (selectedIndex !== undefined) {
-      if (isVectorSearchUseCase(props.workflow)) {
+      if (isVectorSearchUseCase(props.workflow?.ui_metadata?.type)) {
         dispatch(getMappings({ index: selectedIndex, dataSourceId }))
           .unwrap()
           .then((resp: IndexMappings) => {

public/pages/workflows/new_workflow/quick_configure_modal.tsx

@@ -59,6 +59,7 @@ import {
   MODEL_STATE,
   ML_REMOTE_MODEL_LINK,
   MODEL_CATEGORY,
+  isRAGUseCase,
 } from '../../../../common';
 import { APP_PATH, getInitialValue } from '../../../utils';
 import { AppState, createWorkflow, useAppDispatch } from '../../../store';
@@ -140,10 +141,7 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) {
     // If not custom/blank, we will have more req'd form fields for the users to supply
     if (workflowType !== WORKFLOW_TYPE.CUSTOM) {
       // if a RAG workflow, require an LLM
-      if (
-        workflowType === WORKFLOW_TYPE.RAG ||
-        workflowType === WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG
-      ) {
+      if (isRAGUseCase(workflowType)) {
         tempFormValues = {
           ...tempFormValues,
           llm: getInitialValue('model'),
@@ -160,24 +158,21 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) {
           }),
         };
       }
-      // all workflows besides custom and vanilla RAG require an embedding model
-      if (workflowType !== WORKFLOW_TYPE.RAG) {
-        tempFormValues = {
-          ...tempFormValues,
-          embeddingModel: getInitialValue('model'),
-        };
-        tempFormSchemaObj = {
-          ...tempFormSchemaObj,
-          embeddingModel: yup.object({
-            id: yup
-              .string()
-              .trim()
-              .min(1, 'Too short')
-              .max(MAX_STRING_LENGTH, 'Too long')
-              .required('Required'),
-          }),
-        };
-      }
+      tempFormValues = {
+        ...tempFormValues,
+        embeddingModel: getInitialValue('model'),
+      };
+      tempFormSchemaObj = {
+        ...tempFormSchemaObj,
+        embeddingModel: yup.object({
+          id: yup
+            .string()
+            .trim()
+            .min(1, 'Too short')
+            .max(MAX_STRING_LENGTH, 'Too long')
+            .required('Required'),
+        }),
+      };
     }
     setFormValues(tempFormValues);
     setFormSchemaObj(tempFormSchemaObj);
@@ -297,30 +292,26 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) {
                       />
                     </EuiFlexItem>
                   )}
-                {(props.workflow?.ui_metadata?.type === WORKFLOW_TYPE.RAG ||
-                  props.workflow?.ui_metadata?.type ===
-                    WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG) &&
-                  !isEmpty(deployedModels) && (
-                    <EuiFlexItem>
-                      <ModelField
-                        modelCategory={MODEL_CATEGORY.LLM}
-                        fieldPath="llm"
-                        showMissingInterfaceCallout={false}
-                        label="Large language model"
-                        helpText="The large language model to generate user-friendly responses."
-                        fullWidth={true}
-                        showError={true}
-                        onModelChange={(modelId) =>
-                          setQuickConfigureFields({
-                            ...quickConfigureFields,
-                            llmId: modelId,
-                          })
-                        }
-                      />
-                    </EuiFlexItem>
-                  )}
+                {isRAGUseCase(props.workflow?.ui_metadata?.type) && (
+                  <EuiFlexItem>
+                    <ModelField
+                      modelCategory={MODEL_CATEGORY.LLM}
+                      fieldPath="llm"
+                      showMissingInterfaceCallout={false}
+                      label="Large language model"
+                      helpText="The large language model to generate user-friendly responses."
+                      fullWidth={true}
+                      showError={true}
+                      onModelChange={(modelId) =>
+                        setQuickConfigureFields({
+                          ...quickConfigureFields,
+                          llmId: modelId,
+                        })
+                      }
+                    />
+                  </EuiFlexItem>
+                )}
                 {props.workflow?.ui_metadata?.type !== WORKFLOW_TYPE.CUSTOM &&
-                  props.workflow?.ui_metadata?.type !== WORKFLOW_TYPE.RAG &&
                   !isEmpty(deployedModels) && (
                     <EuiFlexItem>
                       <>
@@ -449,7 +440,7 @@ function injectQuickConfigureFields(
             workflow.ui_metadata.config,
             quickConfigureFields,
             embeddingModelInterface,
-            isVectorSearchUseCase(workflow)
+            isVectorSearchUseCase(workflow?.ui_metadata?.type)
           );
           workflow.ui_metadata.config = updateIndexConfig(
             workflow.ui_metadata.config,
@@ -463,32 +454,19 @@ function injectQuickConfigureFields(
             workflow.ui_metadata.config,
             quickConfigureFields,
             embeddingModelInterface,
-            isVectorSearchUseCase(workflow)
-          );
-        }
-        break;
-      }
-      case WORKFLOW_TYPE.RAG: {
-        if (!isEmpty(quickConfigureFields) && workflow.ui_metadata?.config) {
-          workflow.ui_metadata.config = updateIndexConfig(
-            workflow.ui_metadata.config,
-            quickConfigureFields
-          );
-          workflow.ui_metadata.config = updateRAGSearchResponseProcessors(
-            workflow.ui_metadata.config,
-            quickConfigureFields,
-            llmInterface
+            isVectorSearchUseCase(workflow?.ui_metadata?.type)
           );
         }
         break;
       }
-      case WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG: {
+      case WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG:
+      case WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG: {
         if (!isEmpty(quickConfigureFields) && workflow.ui_metadata?.config) {
           workflow.ui_metadata.config = updateIngestProcessors(
             workflow.ui_metadata.config,
             quickConfigureFields,
             embeddingModelInterface,
-            isVectorSearchUseCase(workflow)
+            isVectorSearchUseCase(workflow?.ui_metadata?.type)
           );
           workflow.ui_metadata.config = updateIndexConfig(
             workflow.ui_metadata.config,
@@ -502,7 +480,7 @@ function injectQuickConfigureFields(
             workflow.ui_metadata.config,
             quickConfigureFields,
             embeddingModelInterface,
-            isVectorSearchUseCase(workflow)
+            isVectorSearchUseCase(workflow?.ui_metadata?.type)
           );
           workflow.ui_metadata.config = updateRAGSearchResponseProcessors(
             workflow.ui_metadata.config,

public/pages/workflows/new_workflow/quick_configure_optional_fields.tsx

@@ -26,6 +26,8 @@ import {
   ModelInterface,
   QuickConfigureFields,
   WORKFLOW_TYPE,
+  isRAGUseCase,
+  isVectorSearchUseCase,
 } from '../../../../common';
 import { AppState } from '../../../store';
 import { getEmbeddingModelDimensions, parseModelInputs } from '../../../utils';
@@ -80,15 +82,8 @@ export function QuickConfigureOptionalFields(
         };
         break;
       }
-      case WORKFLOW_TYPE.RAG: {
-        defaultFieldValues = {
-          textField: DEFAULT_TEXT_FIELD,
-          promptField: '',
-          llmResponseField: DEFAULT_LLM_RESPONSE_FIELD,
-        };
-        break;
-      }
-      case WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG: {
+      case WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG:
+      case WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG: {
         defaultFieldValues = {
           textField: DEFAULT_TEXT_FIELD,
           vectorField: DEFAULT_VECTOR_FIELD,
@@ -157,11 +152,7 @@ export function QuickConfigureOptionalFields(
           fullWidth={true}
           label={'Text field'}
           isInvalid={false}
-          helpText={`The name of the text document field to be ${
-            props.workflowType === WORKFLOW_TYPE.RAG
-              ? 'used as context to the large language model (LLM).'
-              : 'embedded.'
-          }`}
+          helpText={`The name of the text document field to be embedded`}
         >
           <EuiCompressedFieldText
             data-testid="textFieldQuickConfigure"
@@ -198,10 +189,7 @@ export function QuickConfigureOptionalFields(
             <EuiSpacer size="s" />
           </>
         )}
-        {(props.workflowType === WORKFLOW_TYPE.SEMANTIC_SEARCH ||
-          props.workflowType === WORKFLOW_TYPE.MULTIMODAL_SEARCH ||
-          props.workflowType === WORKFLOW_TYPE.HYBRID_SEARCH ||
-          props.workflowType === WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG) && (
+        {isVectorSearchUseCase(props.workflowType) && (
           <>
             <EuiCompressedFormRow
               fullWidth={true}
@@ -244,8 +232,7 @@ export function QuickConfigureOptionalFields(
             )}
           </>
         )}
-        {(props.workflowType === WORKFLOW_TYPE.RAG ||
-          props.workflowType === WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG) && (
+        {isRAGUseCase(props.workflowType) && (
           <>
             <EuiCompressedFormRow
               fullWidth={true}

public/pages/workflows/new_workflow/utils.ts

@@ -56,14 +56,14 @@ export function enrichPresetWorkflowWithUiMetadata(
       uiMetadata = fetchHybridSearchMetadata(workflowVersion);
       break;
     }
-    case WORKFLOW_TYPE.RAG: {
-      uiMetadata = fetchRAGMetadata(workflowVersion);
-      break;
-    }
     case WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG: {
       uiMetadata = fetchVectorSearchWithRAGMetadata(workflowVersion);
       break;
     }
+    case WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG: {
+      uiMetadata = fetchHybridSearchWithRAGMetadata(workflowVersion);
+      break;
+    }
     default: {
       uiMetadata = fetchEmptyMetadata();
       break;
@@ -243,36 +243,50 @@ export function fetchHybridSearchMetadata(version: string): UIState {
   return baseState;
 }
 
-export function fetchRAGMetadata(version: string): UIState {
+export function fetchVectorSearchWithRAGMetadata(version: string): UIState {
   let baseState = fetchEmptyMetadata();
-  baseState.type = WORKFLOW_TYPE.RAG;
-  baseState.config.ingest.index.name.value = generateId('my_index', 6);
-  baseState.config.search.request.value = customStringify(FETCH_ALL_QUERY);
+  baseState.type = WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG;
+  // Ingest config: knn index w/ an ML inference processor
+  baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()];
+  baseState.config.ingest.index.name.value = generateId('knn_index', 6);
+  baseState.config.ingest.index.settings.value = customStringify({
+    [`index.knn`]: true,
+  });
+  // Search config: match query => ML inference processor for generating embeddings =>
+  // ML inference processor for returning LLM-generated response of results
+  baseState.config.search.request.value = customStringify(MATCH_QUERY_TEXT);
+  baseState.config.search.enrichRequest.processors = [
+    injectQueryTemplateInProcessor(
+      new MLSearchRequestProcessor().toObj(),
+      KNN_QUERY
+    ),
+  ];
   baseState.config.search.enrichResponse.processors = [
     new MLSearchResponseProcessor().toObj(),
   ];
   return baseState;
 }
 
-export function fetchVectorSearchWithRAGMetadata(version: string): UIState {
+export function fetchHybridSearchWithRAGMetadata(version: string): UIState {
   let baseState = fetchEmptyMetadata();
-  baseState.type = WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG;
+  baseState.type = WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG;
   // Ingest config: knn index w/ an ML inference processor
   baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()];
   baseState.config.ingest.index.name.value = generateId('knn_index', 6);
   baseState.config.ingest.index.settings.value = customStringify({
     [`index.knn`]: true,
   });
-  // Search config: match query => ML inference processor for generating embeddings =>
-  // ML inference processor for returning LLM-generated response of results
+  // Search config: match query => ML inference processor for generating embeddings
+  // with hybrid search => ML inference processor for returning LLM-generated response of results
   baseState.config.search.request.value = customStringify(MATCH_QUERY_TEXT);
   baseState.config.search.enrichRequest.processors = [
     injectQueryTemplateInProcessor(
       new MLSearchRequestProcessor().toObj(),
-      KNN_QUERY
+      HYBRID_SEARCH_QUERY_MATCH_KNN
     ),
   ];
   baseState.config.search.enrichResponse.processors = [
+    new NormalizationProcessor().toObj(),
     new MLSearchResponseProcessor().toObj(),
   ];
   return baseState;