Add new RAG + hybrid search preset

common/constants.ts

@@ -168,6 +168,7 @@ export enum WORKFLOW_TYPE {
   HYBRID_SEARCH = 'Hybrid Search',
   RAG = 'RAG with Lexical Retrieval',
   VECTOR_SEARCH_WITH_RAG = 'RAG with Vector Retrieval',
+  HYBRID_SEARCH_WITH_RAG = 'RAG with Hybrid Search',
   CUSTOM = 'Custom Search',
   UNKNOWN = 'Unknown',
 }

common/utils.ts

@@ -52,6 +52,7 @@ export function isVectorSearchUseCase(workflow: Workflow | undefined): boolean {
       WORKFLOW_TYPE.MULTIMODAL_SEARCH,
       WORKFLOW_TYPE.SEMANTIC_SEARCH,
       WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG,
+      WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG,
     ].includes(workflow?.ui_metadata?.type)
   );
 }

public/pages/workflows/new_workflow/quick_configure_modal.tsx

@@ -142,7 +142,8 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) {
       // if a RAG workflow, require an LLM
       if (
         workflowType === WORKFLOW_TYPE.RAG ||
-        workflowType === WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG
+        workflowType === WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG ||
+        workflowType === WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG
       ) {
         tempFormValues = {
           ...tempFormValues,
@@ -299,7 +300,9 @@ export function QuickConfigureModal(props: QuickConfigureModalProps) {
                   )}
                 {(props.workflow?.ui_metadata?.type === WORKFLOW_TYPE.RAG ||
                   props.workflow?.ui_metadata?.type ===
-                    WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG) &&
+                    WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG ||
+                  props.workflow?.ui_metadata?.type ===
+                    WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG) &&
                   !isEmpty(deployedModels) && (
                     <EuiFlexItem>
                       <ModelField
@@ -482,7 +485,8 @@ function injectQuickConfigureFields(
         }
         break;
       }
-      case WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG: {
+      case WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG:
+      case WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG: {
         if (!isEmpty(quickConfigureFields) && workflow.ui_metadata?.config) {
           workflow.ui_metadata.config = updateIngestProcessors(
             workflow.ui_metadata.config,

public/pages/workflows/new_workflow/quick_configure_optional_fields.tsx

@@ -88,7 +88,8 @@ export function QuickConfigureOptionalFields(
         };
         break;
       }
-      case WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG: {
+      case WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG:
+      case WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG: {
         defaultFieldValues = {
           textField: DEFAULT_TEXT_FIELD,
           vectorField: DEFAULT_VECTOR_FIELD,
@@ -201,7 +202,8 @@ export function QuickConfigureOptionalFields(
         {(props.workflowType === WORKFLOW_TYPE.SEMANTIC_SEARCH ||
           props.workflowType === WORKFLOW_TYPE.MULTIMODAL_SEARCH ||
           props.workflowType === WORKFLOW_TYPE.HYBRID_SEARCH ||
-          props.workflowType === WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG) && (
+          props.workflowType === WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG ||
+          props.workflowType === WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG) && (
           <>
             <EuiCompressedFormRow
               fullWidth={true}
@@ -245,7 +247,8 @@ export function QuickConfigureOptionalFields(
           </>
         )}
         {(props.workflowType === WORKFLOW_TYPE.RAG ||
-          props.workflowType === WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG) && (
+          props.workflowType === WORKFLOW_TYPE.VECTOR_SEARCH_WITH_RAG ||
+          props.workflowType === WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG) && (
           <>
             <EuiCompressedFormRow
               fullWidth={true}

public/pages/workflows/new_workflow/utils.ts

@@ -64,6 +64,10 @@ export function enrichPresetWorkflowWithUiMetadata(
       uiMetadata = fetchVectorSearchWithRAGMetadata(workflowVersion);
       break;
     }
+    case WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG: {
+      uiMetadata = fetchHybridSearchWithRAGMetadata(workflowVersion);
+      break;
+    }
     default: {
       uiMetadata = fetchEmptyMetadata();
       break;
@@ -278,6 +282,31 @@ export function fetchVectorSearchWithRAGMetadata(version: string): UIState {
   return baseState;
 }
 
+export function fetchHybridSearchWithRAGMetadata(version: string): UIState {
+  let baseState = fetchEmptyMetadata();
+  baseState.type = WORKFLOW_TYPE.HYBRID_SEARCH_WITH_RAG;
+  // Ingest config: knn index w/ an ML inference processor
+  baseState.config.ingest.enrich.processors = [new MLIngestProcessor().toObj()];
+  baseState.config.ingest.index.name.value = generateId('knn_index', 6);
+  baseState.config.ingest.index.settings.value = customStringify({
+    [`index.knn`]: true,
+  });
+  // Search config: match query => ML inference processor for generating embeddings
+  // with hybrid search => ML inference processor for returning LLM-generated response of results
+  baseState.config.search.request.value = customStringify(MATCH_QUERY_TEXT);
+  baseState.config.search.enrichRequest.processors = [
+    injectQueryTemplateInProcessor(
+      new MLSearchRequestProcessor().toObj(),
+      HYBRID_SEARCH_QUERY_MATCH_KNN
+    ),
+  ];
+  baseState.config.search.enrichResponse.processors = [
+    new NormalizationProcessor().toObj(),
+    new MLSearchResponseProcessor().toObj(),
+  ];
+  return baseState;
+}
+
 // populate the `query_template` config value with a given query template
 // by default, we replace any vector pattern ("{{vector}}") with the unquoted
 // vector template placeholder (${vector}) so it becomes a proper template

public/utils/form_to_pipeline_utils.ts

@@ -8,6 +8,7 @@ import {
   IProcessorConfig,
   IngestPipelineConfig,
   PROCESSOR_CONTEXT,
+  PROCESSOR_TYPE,
   SearchPipelineConfig,
   WorkflowConfig,
   WorkflowFormValues,
@@ -83,8 +84,17 @@ export function formikToPartialPipeline(
                 requestProcessors,
                 context
               ),
+              // for search response, we need to explicitly separate out any phase results processors
+              phase_results_processors: processorConfigsToTemplateProcessors(
+                precedingProcessors.filter((processor) =>
+                  isPhaseResultsProcessor(processor)
+                ),
+                context
+              ),
               response_processors: processorConfigsToTemplateProcessors(
-                precedingProcessors,
+                precedingProcessors.filter(
+                  (processor) => !isPhaseResultsProcessor(processor)
+                ),
                 context
               ),
             } as SearchPipelineConfig)
@@ -113,3 +123,8 @@ function getPrecedingProcessors(
   });
   return precedingProcessors;
 }
+
+// currently the only phase results processor supported is the normalization processor
+function isPhaseResultsProcessor(processor: IProcessorConfig): boolean {
+  return processor.type === PROCESSOR_TYPE.NORMALIZATION;
+}

server/resources/templates/hybrid_search_with_rag.json

@@ -0,0 +1,14 @@
+{
+    "name": "RAG with Hybrid Search",
+    "description": "Build a search application that uses retrieval-augmented generation (RAG) to retrieve relevant documents using hybrid search, pass them to large language models, and synthesize answers.",
+    "version": {
+        "template": "1.0.0",
+        "compatibility": [
+            "2.19.0",
+            "3.0.0"
+        ]
+    },
+    "ui_metadata": {
+        "type": "RAG with Hybrid Search"
+    }
+}