From bdbb9b89ddaba77e478893da087062a4d7713ce2 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 18 Apr 2024 18:11:25 -0700 Subject: [PATCH 1/7] Add prototype page skeleton; add semantic search query generation Signed-off-by: Tyler Ohlsen --- .../pages/workflow_detail/prototype/index.ts | 6 + .../workflow_detail/prototype/prototype.tsx | 57 +++++ .../prototype/query_executor.tsx | 196 ++++++++++++++++++ .../utils/data_extractor_utils.ts | 87 ++++++++ public/pages/workflow_detail/utils/index.ts | 1 + .../utils/workflow_to_template_utils.ts | 2 +- .../pages/workflow_detail/workflow_detail.tsx | 17 ++ 7 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 public/pages/workflow_detail/prototype/index.ts create mode 100644 public/pages/workflow_detail/prototype/prototype.tsx create mode 100644 public/pages/workflow_detail/prototype/query_executor.tsx create mode 100644 public/pages/workflow_detail/utils/data_extractor_utils.ts diff --git a/public/pages/workflow_detail/prototype/index.ts b/public/pages/workflow_detail/prototype/index.ts new file mode 100644 index 00000000..1021e1a2 --- /dev/null +++ b/public/pages/workflow_detail/prototype/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export * from './prototype'; diff --git a/public/pages/workflow_detail/prototype/prototype.tsx b/public/pages/workflow_detail/prototype/prototype.tsx new file mode 100644 index 00000000..40e73af8 --- /dev/null +++ b/public/pages/workflow_detail/prototype/prototype.tsx @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { + EuiEmptyPrompt, + EuiFlexGroup, + EuiFlexItem, + EuiPageContent, + EuiSpacer, + EuiText, + EuiTitle, +} from '@elastic/eui'; +import { Workflow } from '../../../../common'; +import { QueryExecutor } from './query_executor'; + +interface PrototypeProps { + workflow?: Workflow; +} + +/** + * A simple prototyping page to perform ingest and search. + */ +export function Prototype(props: PrototypeProps) { + return ( + + +

Prototype

+ + + {props.workflow?.resourcesCreated && + props.workflow?.resourcesCreated.length > 0 ? ( + + + + + + ) : ( + No resources available} + titleSize="s" + body={ + <> + + Provision the workflow to generate resources in order to start + prototyping. + + + } + /> + )} + + ); +} diff --git a/public/pages/workflow_detail/prototype/query_executor.tsx b/public/pages/workflow_detail/prototype/query_executor.tsx new file mode 100644 index 00000000..d7079f2f --- /dev/null +++ b/public/pages/workflow_detail/prototype/query_executor.tsx @@ -0,0 +1,196 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useState, useEffect } from 'react'; +import { + EuiButton, + EuiCodeEditor, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiText, +} from '@elastic/eui'; +import { + USE_CASE, + Workflow, + getSemanticSearchValues, +} from '../../../../common'; + +interface QueryExecutorProps { + workflow: Workflow; +} + +type WorkflowValues = { + modelId: string; +}; + +type SemanticSearchValues = WorkflowValues & { + inputField: string; + vectorField: string; +}; + +type QueryGeneratorFn = ( + queryText: string, + workflowValues: SemanticSearchValues +) => {}; + +/** + * A basic and flexible UI for executing queries against an index. Sets up guardrails to limit + * what is customized in the query, and setting readonly values based on the workflow's use case + * and details. + * + * For example, given a semantic search workflow configured on index A, with model B, input field C, and vector field D, + * the UI will enforce a semantic search neural query configured with B,C,D, and run it against A. + */ +export function QueryExecutor(props: QueryExecutorProps) { + // query state + const [workflowValues, setWorkflowValues] = useState(); + const [queryGeneratorFn, setQueryGeneratorFn] = useState(); + const [queryObj, setQueryObj] = useState<{}>({}); + const [userInput, setUserInput] = useState(''); + + // results state + const [resultsObj, setResultsObj] = useState<{}>({}); + + // hook to set the appropriate values and query generator fn + useEffect(() => { + setWorkflowValues(getWorkflowValues(props.workflow)); + setQueryGeneratorFn(getQueryGeneratorFn(props.workflow)); + }, [props.workflow]); + + // hook to generate the query once all dependent input vars are available + useEffect(() => { + if (queryGeneratorFn && workflowValues) { + setQueryObj(queryGeneratorFn(userInput, workflowValues)); + } + }, [userInput, queryGeneratorFn, workflowValues]); + + function onExecute() { + console.log('executing...'); + } + + return ( + + + + + Query + + + + + { + setUserInput(e.target.value); + }} + /> + + + + Run! + + + + + + {}} + readOnly={true} + setOptions={{ + fontSize: '14px', + // enableBasicAutocompletion: true, + // enableLiveAutocompletion: true, + }} + aria-label="Code Editor" + tabSize={2} + /> + + + + + + + + Results + + + {}} + readOnly={true} + setOptions={{ + fontSize: '14px', + }} + aria-label="Code Editor" + tabSize={2} + /> + + + + + ); +} + +// utility fn to get the displayable JSON string +function getFormattedJSONString(obj: {}): string { + return Object.values(obj).length > 0 ? JSON.stringify(obj, null, '\t') : ''; +} + +// getting the appropriate query generator function based on the use case +function getQueryGeneratorFn(workflow: Workflow): QueryGeneratorFn { + let fn; + switch (workflow.use_case) { + case USE_CASE.SEMANTIC_SEARCH: + default: { + fn = () => generateSemanticSearchQuery; + } + } + return fn; +} + +// getting the appropriate static values from the workflow based on the use case +function getWorkflowValues(workflow: Workflow): WorkflowValues { + let values; + switch (workflow.use_case) { + case USE_CASE.SEMANTIC_SEARCH: + default: { + values = getSemanticSearchValues(workflow); + } + } + return values; +} + +// utility fn to generate a semantic search query +function generateSemanticSearchQuery( + queryText: string, + workflowValues: SemanticSearchValues +): {} { + return { + _source: { + excludes: [`${workflowValues.vectorField}`], + }, + query: { + neural: { + [workflowValues.vectorField]: { + query_text: queryText, + model_id: workflowValues.modelId, + k: 5, + }, + }, + }, + }; +} diff --git a/public/pages/workflow_detail/utils/data_extractor_utils.ts b/public/pages/workflow_detail/utils/data_extractor_utils.ts new file mode 100644 index 00000000..c886cfcd --- /dev/null +++ b/public/pages/workflow_detail/utils/data_extractor_utils.ts @@ -0,0 +1,87 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + ReactFlowComponent, + COMPONENT_CLASS, + componentDataToFormik, + ModelFormValue, + MODEL_CATEGORY, + WorkspaceFormValues, + Workflow, + WORKFLOW_RESOURCE_TYPE, + WorkflowResource, +} from '../../../../common'; +import { getIngestNodesAndEdges } from './workflow_to_template_utils'; + +/** + * Collection of utility fns to extract + * data fields from a Workflow + */ +export function getSemanticSearchValues( + workflow: Workflow +): { modelId: string; inputField: string; vectorField: string } { + const formValues = getFormValues(workflow) as WorkspaceFormValues; + const modelId = getModelId(workflow, formValues) as string; + const transformerComponent = getTransformerComponent( + workflow, + formValues + ) as ReactFlowComponent; + const { inputField, vectorField } = componentDataToFormik( + transformerComponent.data + ) as { inputField: string; vectorField: string }; + return { modelId, inputField, vectorField }; +} + +function getFormValues(workflow: Workflow): WorkspaceFormValues | undefined { + if (workflow?.ui_metadata?.workspace_flow) { + const formValues = {} as WorkspaceFormValues; + workflow.ui_metadata.workspace_flow.nodes.forEach((node) => { + formValues[node.id] = componentDataToFormik(node.data); + }); + return formValues; + } +} + +function getModelId( + workflow: Workflow, + formValues: WorkspaceFormValues +): string | undefined { + if (workflow?.ui_metadata?.workspace_flow) { + const transformerComponent = getTransformerComponent(workflow, formValues); + if (transformerComponent) { + const { model } = componentDataToFormik(transformerComponent.data) as { + model: ModelFormValue; + inputField: string; + vectorField: string; + }; + + // if it's a pretrained model, we created a new model ID, parse from resources + if (model.category === MODEL_CATEGORY.PRETRAINED) { + const modelResource = workflow.resourcesCreated?.find( + (resource) => resource.type === WORKFLOW_RESOURCE_TYPE.MODEL_ID + ) as WorkflowResource; + return modelResource.id; + } else { + return model.id; + } + } + } +} + +function getTransformerComponent( + workflow: Workflow, + formValues: WorkspaceFormValues +): ReactFlowComponent | undefined { + if (workflow?.ui_metadata?.workspace_flow) { + const { ingestNodes } = getIngestNodesAndEdges( + workflow?.ui_metadata?.workspace_flow?.nodes, + workflow?.ui_metadata?.workspace_flow?.edges + ); + return ingestNodes.find((ingestNode) => + ingestNode.data.baseClasses?.includes(COMPONENT_CLASS.ML_TRANSFORMER) + ); + } +} diff --git a/public/pages/workflow_detail/utils/index.ts b/public/pages/workflow_detail/utils/index.ts index 91b6465b..94f7e2e4 100644 --- a/public/pages/workflow_detail/utils/index.ts +++ b/public/pages/workflow_detail/utils/index.ts @@ -5,3 +5,4 @@ export * from './utils'; export * from './workflow_to_template_utils'; +export * from './data_extractor_utils'; diff --git a/public/pages/workflow_detail/utils/workflow_to_template_utils.ts b/public/pages/workflow_detail/utils/workflow_to_template_utils.ts index 399129d8..4db87cc4 100644 --- a/public/pages/workflow_detail/utils/workflow_to_template_utils.ts +++ b/public/pages/workflow_detail/utils/workflow_to_template_utils.ts @@ -51,7 +51,7 @@ export function toTemplateFlows( }; } -function getIngestNodesAndEdges( +export function getIngestNodesAndEdges( allNodes: ReactFlowComponent[], allEdges: ReactFlowEdge[] ): { ingestNodes: ReactFlowComponent[]; ingestEdges: ReactFlowEdge[] } { diff --git a/public/pages/workflow_detail/workflow_detail.tsx b/public/pages/workflow_detail/workflow_detail.tsx index 8c397b22..8406e34d 100644 --- a/public/pages/workflow_detail/workflow_detail.tsx +++ b/public/pages/workflow_detail/workflow_detail.tsx @@ -28,6 +28,7 @@ import { Resources } from './resources'; // styling import './workflow-detail-styles.scss'; +import { Prototype } from './prototype'; export interface WorkflowDetailRouterProps { workflowId: string; @@ -42,6 +43,10 @@ enum WORKFLOW_DETAILS_TAB { // This gives clarity into what has been done on the cluster on behalf // of the frontend provisioning workflows. RESOURCES = 'resources', + // TODO: temporarily adding a prototype tab until UX is finalized. + // This allows simple UI for executing ingest and search against + // created workflow resources + PROTOTYPE = 'prototype', } const ACTIVE_TAB_PARAM = 'tab'; @@ -142,6 +147,15 @@ export function WorkflowDetail(props: WorkflowDetailProps) { replaceActiveTab(WORKFLOW_DETAILS_TAB.RESOURCES, props); }, }, + { + id: WORKFLOW_DETAILS_TAB.PROTOTYPE, + label: 'Prototype', + isSelected: selectedTabId === WORKFLOW_DETAILS_TAB.PROTOTYPE, + onClick: () => { + setSelectedTabId(WORKFLOW_DETAILS_TAB.PROTOTYPE); + replaceActiveTab(WORKFLOW_DETAILS_TAB.PROTOTYPE, props); + }, + }, ]; return ( @@ -164,6 +178,9 @@ export function WorkflowDetail(props: WorkflowDetailProps) { {selectedTabId === WORKFLOW_DETAILS_TAB.RESOURCES && ( )} + {selectedTabId === WORKFLOW_DETAILS_TAB.PROTOTYPE && ( + + )} From 9669a0a4a4d4665527dabc638ad4f4fb77512662 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 18 Apr 2024 18:22:24 -0700 Subject: [PATCH 2/7] Fetch configured index Signed-off-by: Tyler Ohlsen --- .../prototype/query_executor.tsx | 9 ++-- .../utils/data_extractor_utils.ts | 43 ++++++++++++++----- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/public/pages/workflow_detail/prototype/query_executor.tsx b/public/pages/workflow_detail/prototype/query_executor.tsx index d7079f2f..f9bab365 100644 --- a/public/pages/workflow_detail/prototype/query_executor.tsx +++ b/public/pages/workflow_detail/prototype/query_executor.tsx @@ -15,6 +15,7 @@ import { import { USE_CASE, Workflow, + getIndexName, getSemanticSearchValues, } from '../../../../common'; @@ -48,16 +49,18 @@ export function QueryExecutor(props: QueryExecutorProps) { // query state const [workflowValues, setWorkflowValues] = useState(); const [queryGeneratorFn, setQueryGeneratorFn] = useState(); + const [indexName, setIndexName] = useState(); const [queryObj, setQueryObj] = useState<{}>({}); const [userInput, setUserInput] = useState(''); // results state const [resultsObj, setResultsObj] = useState<{}>({}); - // hook to set the appropriate values and query generator fn + // hook to set all of the workflow-related fields based on the use case useEffect(() => { setWorkflowValues(getWorkflowValues(props.workflow)); setQueryGeneratorFn(getQueryGeneratorFn(props.workflow)); + setIndexName(getIndexName(props.workflow)); }, [props.workflow]); // hook to generate the query once all dependent input vars are available @@ -67,9 +70,7 @@ export function QueryExecutor(props: QueryExecutorProps) { } }, [userInput, queryGeneratorFn, workflowValues]); - function onExecute() { - console.log('executing...'); - } + function onExecute() {} return ( diff --git a/public/pages/workflow_detail/utils/data_extractor_utils.ts b/public/pages/workflow_detail/utils/data_extractor_utils.ts index c886cfcd..f4fd01bb 100644 --- a/public/pages/workflow_detail/utils/data_extractor_utils.ts +++ b/public/pages/workflow_detail/utils/data_extractor_utils.ts @@ -20,14 +20,25 @@ import { getIngestNodesAndEdges } from './workflow_to_template_utils'; * Collection of utility fns to extract * data fields from a Workflow */ + +export function getIndexName(workflow: Workflow): string | undefined { + if (workflow?.ui_metadata?.workspace_flow) { + const indexerComponent = getIndexerComponent(workflow); + if (indexerComponent) { + const { indexName } = componentDataToFormik(indexerComponent.data) as { + indexName: string; + }; + return indexName; + } + } +} + export function getSemanticSearchValues( workflow: Workflow ): { modelId: string; inputField: string; vectorField: string } { - const formValues = getFormValues(workflow) as WorkspaceFormValues; - const modelId = getModelId(workflow, formValues) as string; + const modelId = getModelId(workflow) as string; const transformerComponent = getTransformerComponent( - workflow, - formValues + workflow ) as ReactFlowComponent; const { inputField, vectorField } = componentDataToFormik( transformerComponent.data @@ -45,12 +56,9 @@ function getFormValues(workflow: Workflow): WorkspaceFormValues | undefined { } } -function getModelId( - workflow: Workflow, - formValues: WorkspaceFormValues -): string | undefined { +function getModelId(workflow: Workflow): string | undefined { if (workflow?.ui_metadata?.workspace_flow) { - const transformerComponent = getTransformerComponent(workflow, formValues); + const transformerComponent = getTransformerComponent(workflow); if (transformerComponent) { const { model } = componentDataToFormik(transformerComponent.data) as { model: ModelFormValue; @@ -72,8 +80,7 @@ function getModelId( } function getTransformerComponent( - workflow: Workflow, - formValues: WorkspaceFormValues + workflow: Workflow ): ReactFlowComponent | undefined { if (workflow?.ui_metadata?.workspace_flow) { const { ingestNodes } = getIngestNodesAndEdges( @@ -85,3 +92,17 @@ function getTransformerComponent( ); } } + +function getIndexerComponent( + workflow: Workflow +): ReactFlowComponent | undefined { + if (workflow?.ui_metadata?.workspace_flow) { + const { ingestNodes } = getIngestNodesAndEdges( + workflow?.ui_metadata?.workspace_flow?.nodes, + workflow?.ui_metadata?.workspace_flow?.edges + ); + return ingestNodes.find((ingestNode) => + ingestNode.data.baseClasses?.includes(COMPONENT_CLASS.INDEXER) + ); + } +} From b37be0da590f124fce6746d341daf99ef5bc3255 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 18 Apr 2024 18:30:24 -0700 Subject: [PATCH 3/7] add formatting on index side Signed-off-by: Tyler Ohlsen --- .../prototype/query_executor.tsx | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/public/pages/workflow_detail/prototype/query_executor.tsx b/public/pages/workflow_detail/prototype/query_executor.tsx index f9bab365..046b0fe2 100644 --- a/public/pages/workflow_detail/prototype/query_executor.tsx +++ b/public/pages/workflow_detail/prototype/query_executor.tsx @@ -70,7 +70,9 @@ export function QueryExecutor(props: QueryExecutorProps) { } }, [userInput, queryGeneratorFn, workflowValues]); - function onExecute() {} + function onExecute() { + console.log('executing...'); + } return ( @@ -120,11 +122,24 @@ export function QueryExecutor(props: QueryExecutorProps) { - + Results - + + + + + + + + Date: Thu, 18 Apr 2024 19:22:26 -0700 Subject: [PATCH 4/7] Onboard search API; onboard query execution on UI Signed-off-by: Tyler Ohlsen --- common/constants.ts | 1 + .../prototype/query_executor.tsx | 45 +++++++++++++++---- public/route_service.ts | 15 +++++++ public/store/reducers/opensearch_reducer.ts | 29 ++++++++++++ server/routes/opensearch_routes_service.ts | 40 ++++++++++++++++- 5 files changed, 119 insertions(+), 11 deletions(-) diff --git a/common/constants.ts b/common/constants.ts index 6cdbbd63..7e835875 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -35,6 +35,7 @@ export const BASE_NODE_API_PATH = '/api/flow_framework'; // OpenSearch node APIs export const BASE_OPENSEARCH_NODE_API_PATH = `${BASE_NODE_API_PATH}/opensearch`; export const CAT_INDICES_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/catIndices`; +export const SEARCH_INDEX_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/search`; // Flow Framework node APIs export const BASE_WORKFLOW_NODE_API_PATH = `${BASE_NODE_API_PATH}/workflow`; diff --git a/public/pages/workflow_detail/prototype/query_executor.tsx b/public/pages/workflow_detail/prototype/query_executor.tsx index 046b0fe2..525b504a 100644 --- a/public/pages/workflow_detail/prototype/query_executor.tsx +++ b/public/pages/workflow_detail/prototype/query_executor.tsx @@ -18,6 +18,8 @@ import { getIndexName, getSemanticSearchValues, } from '../../../../common'; +import { searchIndex, useAppDispatch } from '../../../store'; +import { getCore } from '../../../services'; interface QueryExecutorProps { workflow: Workflow; @@ -46,15 +48,18 @@ type QueryGeneratorFn = ( * the UI will enforce a semantic search neural query configured with B,C,D, and run it against A. */ export function QueryExecutor(props: QueryExecutorProps) { + const dispatch = useAppDispatch(); // query state const [workflowValues, setWorkflowValues] = useState(); const [queryGeneratorFn, setQueryGeneratorFn] = useState(); - const [indexName, setIndexName] = useState(); + const [indexName, setIndexName] = useState(''); const [queryObj, setQueryObj] = useState<{}>({}); + const [formattedQuery, setFormattedQuery] = useState(''); const [userInput, setUserInput] = useState(''); // results state - const [resultsObj, setResultsObj] = useState<{}>({}); + const [resultHits, setResultHits] = useState<{}[]>([]); + const [formattedHits, setFormattedHits] = useState(''); // hook to set all of the workflow-related fields based on the use case useEffect(() => { @@ -70,8 +75,26 @@ export function QueryExecutor(props: QueryExecutorProps) { } }, [userInput, queryGeneratorFn, workflowValues]); - function onExecute() { - console.log('executing...'); + // hooks to persist the formatted data. this is so we don't + // re-execute the JSON formatting unless necessary + useEffect(() => { + setFormattedHits(getFormattedJSONString(processHits(resultHits))); + }, [resultHits]); + useEffect(() => { + setFormattedQuery(getFormattedJSONString(queryObj)); + }, [queryObj]); + + // + function onExecuteSearch() { + dispatch(searchIndex({ index: indexName, body: queryObj })) + .unwrap() + .then(async (result) => { + setResultHits(result.hits.hits); + }) + .catch((error: any) => { + getCore().notifications.toasts.addDanger(error); + setResultHits([]); + }); } return ( @@ -94,8 +117,8 @@ export function QueryExecutor(props: QueryExecutorProps) { /> - - Run! + + Search! @@ -106,7 +129,7 @@ export function QueryExecutor(props: QueryExecutorProps) { theme="textmate" width="100%" height="50vh" - value={getFormattedJSONString(queryObj)} + value={formattedQuery} onChange={() => {}} readOnly={true} setOptions={{ @@ -133,7 +156,7 @@ export function QueryExecutor(props: QueryExecutorProps) { placeholder={indexName} prepend="Index:" compressed={false} - disabled={false} + disabled={true} readOnly={true} /> @@ -145,7 +168,7 @@ export function QueryExecutor(props: QueryExecutorProps) { theme="textmate" width="100%" height="50vh" - value={getFormattedJSONString(resultsObj)} + value={formattedHits} onChange={() => {}} readOnly={true} setOptions={{ @@ -210,3 +233,7 @@ function generateSemanticSearchQuery( }, }; } + +function processHits(hits: any[]): {}[] { + return hits.map((hit) => hit._source); +} diff --git a/public/route_service.ts b/public/route_service.ts index 1ff6290e..8eda5d01 100644 --- a/public/route_service.ts +++ b/public/route_service.ts @@ -17,6 +17,7 @@ import { DEPROVISION_WORKFLOW_NODE_API_PATH, UPDATE_WORKFLOW_NODE_API_PATH, WorkflowTemplate, + SEARCH_INDEX_NODE_API_PATH, } from '../common'; /** @@ -40,6 +41,7 @@ export interface RouteService { deleteWorkflow: (workflowId: string) => Promise; getWorkflowPresets: () => Promise; catIndices: (pattern: string) => Promise; + searchIndex: (index: string, body: {}) => Promise; searchModels: (body: {}) => Promise; } @@ -157,6 +159,19 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, + searchIndex: async (index: string, body: {}) => { + try { + const response = await core.http.post<{ respString: string }>( + `${SEARCH_INDEX_NODE_API_PATH}/${index}`, + { + body: JSON.stringify(body), + } + ); + return response; + } catch (e: any) { + return e as HttpFetchError; + } + }, searchModels: async (body: {}) => { try { const response = await core.http.post<{ respString: string }>( diff --git a/public/store/reducers/opensearch_reducer.ts b/public/store/reducers/opensearch_reducer.ts index d9a68e10..8e4f1ec9 100644 --- a/public/store/reducers/opensearch_reducer.ts +++ b/public/store/reducers/opensearch_reducer.ts @@ -16,6 +16,7 @@ const initialState = { const OPENSEARCH_PREFIX = 'opensearch'; const CAT_INDICES_ACTION = `${OPENSEARCH_PREFIX}/catIndices`; +const SEARCH_INDEX_ACTION = `${OPENSEARCH_PREFIX}/search`; export const catIndices = createAsyncThunk( CAT_INDICES_ACTION, @@ -35,6 +36,22 @@ export const catIndices = createAsyncThunk( } ); +export const searchIndex = createAsyncThunk( + SEARCH_INDEX_ACTION, + async (searchIndexInfo: { index: string; body: {} }, { rejectWithValue }) => { + const { index, body } = searchIndexInfo; + const response: any | HttpFetchError = await getRouteService().searchIndex( + index, + body + ); + if (response instanceof HttpFetchError) { + return rejectWithValue('Error searching index: ' + response.body.message); + } else { + return response; + } + } +); + const opensearchSlice = createSlice({ name: OPENSEARCH_PREFIX, initialState, @@ -45,6 +62,10 @@ const opensearchSlice = createSlice({ state.loading = true; state.errorMessage = ''; }) + .addCase(searchIndex.pending, (state, action) => { + state.loading = true; + state.errorMessage = ''; + }) .addCase(catIndices.fulfilled, (state, action) => { const indicesMap = new Map(); action.payload.forEach((index: Index) => { @@ -54,9 +75,17 @@ const opensearchSlice = createSlice({ state.loading = false; state.errorMessage = ''; }) + .addCase(searchIndex.fulfilled, (state, action) => { + state.loading = false; + state.errorMessage = ''; + }) .addCase(catIndices.rejected, (state, action) => { state.errorMessage = action.payload as string; state.loading = false; + }) + .addCase(searchIndex.rejected, (state, action) => { + state.errorMessage = action.payload as string; + state.loading = false; }); }, }); diff --git a/server/routes/opensearch_routes_service.ts b/server/routes/opensearch_routes_service.ts index d162e3ce..f87a8bfe 100644 --- a/server/routes/opensearch_routes_service.ts +++ b/server/routes/opensearch_routes_service.ts @@ -4,7 +4,6 @@ */ import { schema } from '@osd/config-schema'; -import { SearchRequest } from '@opensearch-project/opensearch/api/types'; import { IRouter, IOpenSearchDashboardsResponse, @@ -12,7 +11,11 @@ import { OpenSearchDashboardsRequest, OpenSearchDashboardsResponseFactory, } from '../../../../src/core/server'; -import { CAT_INDICES_NODE_API_PATH, Index } from '../../common'; +import { + CAT_INDICES_NODE_API_PATH, + Index, + SEARCH_INDEX_NODE_API_PATH, +} from '../../common'; import { generateCustomError } from './helpers'; /** @@ -34,6 +37,18 @@ export function registerOpenSearchRoutes( }, opensearchRoutesService.catIndices ); + router.post( + { + path: `${SEARCH_INDEX_NODE_API_PATH}/{index}`, + validate: { + params: schema.object({ + index: schema.string(), + }), + body: schema.any(), + }, + }, + opensearchRoutesService.searchIndex + ); } export class OpenSearchRoutesService { @@ -69,4 +84,25 @@ export class OpenSearchRoutesService { return generateCustomError(res, err); } }; + + searchIndex = async ( + context: RequestHandlerContext, + req: OpenSearchDashboardsRequest, + res: OpenSearchDashboardsResponseFactory + ): Promise> => { + const { index } = req.params as { index: string }; + const body = req.body; + try { + const response = await this.client + .asScoped(req) + .callAsCurrentUser('search', { + index, + body, + }); + + return res.ok({ body: response }); + } catch (err: any) { + return generateCustomError(res, err); + } + }; } From 41e3d6178bc3d64baa3cc6016caa5b03e4215606 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 18 Apr 2024 20:02:10 -0700 Subject: [PATCH 5/7] Add tabs; support same functionality for ingest; Signed-off-by: Tyler Ohlsen --- common/constants.ts | 1 + .../workflow_detail/prototype/ingestor.tsx | 218 ++++++++++++++++++ .../workflow_detail/prototype/prototype.tsx | 58 ++++- .../prototype/query_executor.tsx | 10 +- .../pages/workflow_detail/prototype/utils.ts | 12 + public/route_service.ts | 15 ++ public/store/reducers/opensearch_reducer.ts | 19 ++ server/routes/opensearch_routes_service.ts | 34 +++ 8 files changed, 354 insertions(+), 13 deletions(-) create mode 100644 public/pages/workflow_detail/prototype/ingestor.tsx create mode 100644 public/pages/workflow_detail/prototype/utils.ts diff --git a/common/constants.ts b/common/constants.ts index 7e835875..7f8a507e 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -36,6 +36,7 @@ export const BASE_NODE_API_PATH = '/api/flow_framework'; export const BASE_OPENSEARCH_NODE_API_PATH = `${BASE_NODE_API_PATH}/opensearch`; export const CAT_INDICES_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/catIndices`; export const SEARCH_INDEX_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/search`; +export const INGEST_NODE_API_PATH = `${BASE_OPENSEARCH_NODE_API_PATH}/ingest`; // Flow Framework node APIs export const BASE_WORKFLOW_NODE_API_PATH = `${BASE_NODE_API_PATH}/workflow`; diff --git a/public/pages/workflow_detail/prototype/ingestor.tsx b/public/pages/workflow_detail/prototype/ingestor.tsx new file mode 100644 index 00000000..dee9be85 --- /dev/null +++ b/public/pages/workflow_detail/prototype/ingestor.tsx @@ -0,0 +1,218 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useState, useEffect } from 'react'; +import { + EuiButton, + EuiCodeEditor, + EuiFieldText, + EuiFlexGroup, + EuiFlexItem, + EuiText, +} from '@elastic/eui'; +import { + USE_CASE, + Workflow, + getIndexName, + getSemanticSearchValues, +} from '../../../../common'; +import { ingest, useAppDispatch } from '../../../store'; +import { getCore } from '../../../services'; +import { getFormattedJSONString } from './utils'; + +interface IngestorProps { + workflow: Workflow; +} + +type WorkflowValues = { + modelId: string; +}; + +type SemanticSearchValues = WorkflowValues & { + inputField: string; + vectorField: string; +}; + +type DocGeneratorFn = ( + queryText: string, + workflowValues: SemanticSearchValues +) => {}; + +/** + * A basic and flexible UI for ingesting some documents against an index. Sets up guardrails to limit + * what is customized in the document, and setting readonly values based on the workflow's use case + * and details. + * + * For example, given a semantic search workflow configured on index A, with model B, input field C, and vector field D, + * the UI will enforce the ingested document to include C, and ingest it against A. + */ +export function Ingestor(props: IngestorProps) { + const dispatch = useAppDispatch(); + // query state + const [workflowValues, setWorkflowValues] = useState(); + const [docGeneratorFn, setDocGeneratorFn] = useState(); + const [indexName, setIndexName] = useState(''); + const [docObj, setDocObj] = useState<{}>({}); + const [formattedDoc, setFormattedDoc] = useState(''); + const [userInput, setUserInput] = useState(''); + + // results state + const [response, setResponse] = useState<{}>({}); + const [formattedResponse, setFormattedResponse] = useState(''); + + // hook to set all of the workflow-related fields based on the use case + useEffect(() => { + setWorkflowValues(getWorkflowValues(props.workflow)); + setDocGeneratorFn(getDocGeneratorFn(props.workflow)); + setIndexName(getIndexName(props.workflow)); + }, [props.workflow]); + + // hook to generate the query once all dependent input vars are available + useEffect(() => { + if (docGeneratorFn && workflowValues) { + setDocObj(docGeneratorFn(userInput, workflowValues)); + } + }, [userInput, docGeneratorFn, workflowValues]); + + // hooks to persist the formatted data. this is so we don't + // re-execute the JSON formatting unless necessary + useEffect(() => { + setFormattedResponse(getFormattedJSONString(response)); + }, [response]); + useEffect(() => { + setFormattedDoc(getFormattedJSONString(docObj)); + }, [docObj]); + + // + function onExecuteIngest() { + dispatch(ingest({ index: indexName, doc: docObj })) + .unwrap() + .then(async (result) => { + setResponse(result); + }) + .catch((error: any) => { + getCore().notifications.toasts.addDanger(error); + setResponse({}); + }); + } + + return ( + + + + + Ingest some sample data to get started. + + + + + { + setUserInput(e.target.value); + }} + /> + + + + Ingest + + + + + + {}} + readOnly={true} + setOptions={{ + fontSize: '14px', + }} + aria-label="Code Editor" + tabSize={2} + /> + + + + + + + + Response + + + + + + + + + + {}} + readOnly={true} + setOptions={{ + fontSize: '14px', + }} + aria-label="Code Editor" + tabSize={2} + /> + + + + + ); +} + +// getting the appropriate doc generator function based on the use case +function getDocGeneratorFn(workflow: Workflow): DocGeneratorFn { + let fn; + switch (workflow.use_case) { + case USE_CASE.SEMANTIC_SEARCH: + default: { + fn = () => generateSemanticSearchDoc; + } + } + return fn; +} + +// getting the appropriate static values from the workflow based on the use case +function getWorkflowValues(workflow: Workflow): WorkflowValues { + let values; + switch (workflow.use_case) { + case USE_CASE.SEMANTIC_SEARCH: + default: { + values = getSemanticSearchValues(workflow); + } + } + return values; +} + +// utility fn to generate a document suited for semantic search +function generateSemanticSearchDoc( + docValue: string, + workflowValues: SemanticSearchValues +): {} { + return { + [workflowValues.inputField]: docValue, + }; +} diff --git a/public/pages/workflow_detail/prototype/prototype.tsx b/public/pages/workflow_detail/prototype/prototype.tsx index 40e73af8..dc1ac409 100644 --- a/public/pages/workflow_detail/prototype/prototype.tsx +++ b/public/pages/workflow_detail/prototype/prototype.tsx @@ -3,27 +3,49 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from 'react'; +import React, { useState } from 'react'; import { EuiEmptyPrompt, EuiFlexGroup, EuiFlexItem, EuiPageContent, EuiSpacer, + EuiTab, + EuiTabs, EuiText, EuiTitle, } from '@elastic/eui'; import { Workflow } from '../../../../common'; import { QueryExecutor } from './query_executor'; +import { Ingestor } from './ingestor'; interface PrototypeProps { workflow?: Workflow; } +enum TAB_ID { + INGEST = 'ingest', + QUERY = 'query', +} + +const inputTabs = [ + { + id: TAB_ID.INGEST, + name: '1. Ingest Data', + disabled: false, + }, + { + id: TAB_ID.QUERY, + name: '2. Query data', + disabled: false, + }, +]; + /** * A simple prototyping page to perform ingest and search. */ export function Prototype(props: PrototypeProps) { + const [selectedTabId, setSelectedTabId] = useState(TAB_ID.INGEST); return ( @@ -32,11 +54,35 @@ export function Prototype(props: PrototypeProps) { {props.workflow?.resourcesCreated && props.workflow?.resourcesCreated.length > 0 ? ( - - - - - + <> + + {inputTabs.map((tab, idx) => { + return ( + setSelectedTabId(tab.id)} + isSelected={tab.id === selectedTabId} + disabled={tab.disabled} + key={idx} + > + {tab.name} + + ); + })} + + + + {selectedTabId === TAB_ID.INGEST && ( + + + + )} + {selectedTabId === TAB_ID.QUERY && ( + + + + )} + + ) : ( - Query + Execute queries to test out the results! @@ -118,7 +119,7 @@ export function QueryExecutor(props: QueryExecutorProps) { - Search! + Search @@ -184,11 +185,6 @@ export function QueryExecutor(props: QueryExecutorProps) { ); } -// utility fn to get the displayable JSON string -function getFormattedJSONString(obj: {}): string { - return Object.values(obj).length > 0 ? JSON.stringify(obj, null, '\t') : ''; -} - // getting the appropriate query generator function based on the use case function getQueryGeneratorFn(workflow: Workflow): QueryGeneratorFn { let fn; diff --git a/public/pages/workflow_detail/prototype/utils.ts b/public/pages/workflow_detail/prototype/utils.ts new file mode 100644 index 00000000..c4e3a1a9 --- /dev/null +++ b/public/pages/workflow_detail/prototype/utils.ts @@ -0,0 +1,12 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Shared utility fns used in the prototyping page. + */ + +export function getFormattedJSONString(obj: {}): string { + return Object.values(obj).length > 0 ? JSON.stringify(obj, null, '\t') : ''; +} diff --git a/public/route_service.ts b/public/route_service.ts index 8eda5d01..ce57599a 100644 --- a/public/route_service.ts +++ b/public/route_service.ts @@ -18,6 +18,7 @@ import { UPDATE_WORKFLOW_NODE_API_PATH, WorkflowTemplate, SEARCH_INDEX_NODE_API_PATH, + INGEST_NODE_API_PATH, } from '../common'; /** @@ -42,6 +43,7 @@ export interface RouteService { getWorkflowPresets: () => Promise; catIndices: (pattern: string) => Promise; searchIndex: (index: string, body: {}) => Promise; + ingest: (index: string, doc: {}) => Promise; searchModels: (body: {}) => Promise; } @@ -172,6 +174,19 @@ export function configureRoutes(core: CoreStart): RouteService { return e as HttpFetchError; } }, + ingest: async (index: string, doc: {}) => { + try { + const response = await core.http.put<{ respString: string }>( + `${INGEST_NODE_API_PATH}/${index}`, + { + body: JSON.stringify(doc), + } + ); + return response; + } catch (e: any) { + return e as HttpFetchError; + } + }, searchModels: async (body: {}) => { try { const response = await core.http.post<{ respString: string }>( diff --git a/public/store/reducers/opensearch_reducer.ts b/public/store/reducers/opensearch_reducer.ts index 8e4f1ec9..10735776 100644 --- a/public/store/reducers/opensearch_reducer.ts +++ b/public/store/reducers/opensearch_reducer.ts @@ -17,6 +17,7 @@ const initialState = { const OPENSEARCH_PREFIX = 'opensearch'; const CAT_INDICES_ACTION = `${OPENSEARCH_PREFIX}/catIndices`; const SEARCH_INDEX_ACTION = `${OPENSEARCH_PREFIX}/search`; +const INGEST_ACTION = `${OPENSEARCH_PREFIX}/ingest`; export const catIndices = createAsyncThunk( CAT_INDICES_ACTION, @@ -52,6 +53,24 @@ export const searchIndex = createAsyncThunk( } ); +export const ingest = createAsyncThunk( + INGEST_ACTION, + async (ingestInfo: { index: string; doc: {} }, { rejectWithValue }) => { + const { index, doc } = ingestInfo; + const response: any | HttpFetchError = await getRouteService().ingest( + index, + doc + ); + if (response instanceof HttpFetchError) { + return rejectWithValue( + 'Error ingesting document: ' + response.body.message + ); + } else { + return response; + } + } +); + const opensearchSlice = createSlice({ name: OPENSEARCH_PREFIX, initialState, diff --git a/server/routes/opensearch_routes_service.ts b/server/routes/opensearch_routes_service.ts index f87a8bfe..aae7577a 100644 --- a/server/routes/opensearch_routes_service.ts +++ b/server/routes/opensearch_routes_service.ts @@ -13,6 +13,7 @@ import { } from '../../../../src/core/server'; import { CAT_INDICES_NODE_API_PATH, + INGEST_NODE_API_PATH, Index, SEARCH_INDEX_NODE_API_PATH, } from '../../common'; @@ -49,6 +50,18 @@ export function registerOpenSearchRoutes( }, opensearchRoutesService.searchIndex ); + router.put( + { + path: `${INGEST_NODE_API_PATH}/{index}`, + validate: { + params: schema.object({ + index: schema.string(), + }), + body: schema.any(), + }, + }, + opensearchRoutesService.ingest + ); } export class OpenSearchRoutesService { @@ -105,4 +118,25 @@ export class OpenSearchRoutesService { return generateCustomError(res, err); } }; + + ingest = async ( + context: RequestHandlerContext, + req: OpenSearchDashboardsRequest, + res: OpenSearchDashboardsResponseFactory + ): Promise> => { + const { index } = req.params as { index: string }; + const doc = req.body; + try { + const response = await this.client + .asScoped(req) + .callAsCurrentUser('index', { + index, + body: doc, + }); + + return res.ok({ body: response }); + } catch (err: any) { + return generateCustomError(res, err); + } + }; } From ecb2d5e04bdd5068613aede879ea90ed2d9997b7 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 18 Apr 2024 20:05:24 -0700 Subject: [PATCH 6/7] minor format change Signed-off-by: Tyler Ohlsen --- public/pages/workflow_detail/prototype/query_executor.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/pages/workflow_detail/prototype/query_executor.tsx b/public/pages/workflow_detail/prototype/query_executor.tsx index ec668453..ed7e6969 100644 --- a/public/pages/workflow_detail/prototype/query_executor.tsx +++ b/public/pages/workflow_detail/prototype/query_executor.tsx @@ -148,7 +148,7 @@ export function QueryExecutor(props: QueryExecutorProps) { - Results + Results From 85710002aec4b3c0abfe1d318c22312a860be2e2 Mon Sep 17 00:00:00 2001 From: Tyler Ohlsen Date: Thu, 18 Apr 2024 20:06:46 -0700 Subject: [PATCH 7/7] minor formatting Signed-off-by: Tyler Ohlsen --- public/pages/workflow_detail/prototype/query_executor.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/public/pages/workflow_detail/prototype/query_executor.tsx b/public/pages/workflow_detail/prototype/query_executor.tsx index ed7e6969..d8a794a1 100644 --- a/public/pages/workflow_detail/prototype/query_executor.tsx +++ b/public/pages/workflow_detail/prototype/query_executor.tsx @@ -135,8 +135,6 @@ export function QueryExecutor(props: QueryExecutorProps) { readOnly={true} setOptions={{ fontSize: '14px', - // enableBasicAutocompletion: true, - // enableLiveAutocompletion: true, }} aria-label="Code Editor" tabSize={2}