diff --git a/common/constants.ts b/common/constants.ts index f506f620..2753800b 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -170,6 +170,13 @@ export enum WORKFLOW_TYPE { CUSTOM = 'Custom Search', UNKNOWN = 'Unknown', } +export enum WORKFLOW_TYPE_LEGACY { + SEMANTIC_SEARCH = 'Semantic Search', + MULTIMODAL_SEARCH = 'Multimodal Search', + HYBRID_SEARCH = 'Hybrid Search', + CUSTOM = 'Custom Search', + UNKNOWN = 'Unknown', +} // If no datasource version is found, we default to 2.17.0 export const MIN_SUPPORTED_VERSION = '2.17.0'; // Min version to support ML processors diff --git a/public/pages/workflow_detail/tools/errors/errors.tsx b/public/pages/workflow_detail/tools/errors/errors.tsx index 71d5a561..96eaabd1 100644 --- a/public/pages/workflow_detail/tools/errors/errors.tsx +++ b/public/pages/workflow_detail/tools/errors/errors.tsx @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React from 'react'; +import React, { ReactNode } from 'react'; import { EuiCodeBlock, EuiEmptyPrompt, @@ -12,7 +12,7 @@ import { } from '@elastic/eui'; interface ErrorsProps { - errorMessages: string[]; + errorMessages: (string | ReactNode)[]; } /** diff --git a/public/pages/workflow_detail/tools/tools.tsx b/public/pages/workflow_detail/tools/tools.tsx index ea1e4c51..86f8955c 100644 --- a/public/pages/workflow_detail/tools/tools.tsx +++ b/public/pages/workflow_detail/tools/tools.tsx @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useEffect, useState } from 'react'; +import React, { ReactNode, useEffect, useState } from 'react'; import { useSelector } from 'react-redux'; import { AppState } from '../../../store'; import { isEmpty } from 'lodash'; @@ -26,6 +26,7 @@ import { Query } from './query'; import { Ingest } from './ingest'; import { Errors } from './errors'; import { + formatProcessorError, hasProvisionedIngestResources, hasProvisionedSearchResources, } from '../../../utils'; @@ -52,7 +53,9 @@ export function Tools(props: ToolsProps) { ingestPipeline: ingestPipelineErrors, searchPipeline: searchPipelineErrors, } = useSelector((state: AppState) => state.errors); - const [curErrorMessages, setCurErrorMessages] = useState([]); + const [curErrorMessages, setCurErrorMessages] = useState< + (string | ReactNode)[] + >([]); // Propagate any errors coming from opensearch API calls, including ingest/search pipeline verbose calls. useEffect(() => { @@ -66,12 +69,16 @@ export function Tools(props: ToolsProps) { } else if (!isEmpty(ingestPipelineErrors)) { setCurErrorMessages([ 'Data not ingested. Errors found with the following ingest processor(s):', - ...Object.values(ingestPipelineErrors).map((value) => value.errorMsg), + ...Object.values(ingestPipelineErrors).map((ingestPipelineError) => + formatProcessorError(ingestPipelineError) + ), ]); } else if (!isEmpty(searchPipelineErrors)) { setCurErrorMessages([ 'Errors found with the following search processor(s)', - ...Object.values(searchPipelineErrors).map((value) => value.errorMsg), + ...Object.values(searchPipelineErrors).map((searchPipelineError) => + formatProcessorError(searchPipelineError) + ), ]); } } else { diff --git a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx index 25c7ede8..0f2576be 100644 --- a/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx +++ b/public/pages/workflow_detail/workflow_inputs/ingest_inputs/source_data.tsx @@ -112,8 +112,8 @@ export function SourceData(props: SourceDataProps) {

Import sample data

- - {docsPopulated ? ( + {docsPopulated && ( + setIsEditModalOpen(true)} data-testid="editSourceDataButton" @@ -122,19 +122,8 @@ export function SourceData(props: SourceDataProps) { > Edit - ) : ( - setIsEditModalOpen(true)} - data-testid="selectDataToImportButton" - iconType="plus" - iconSide="left" - > - {`Import`} - - )} - + + )} {props.lastIngested !== undefined && ( @@ -208,6 +197,17 @@ export function SourceData(props: SourceDataProps) { Import a data sample to start configuring your ingest flow. + + setIsEditModalOpen(true)} + data-testid="selectDataToImportButton" + iconType="plus" + iconSide="left" + > + {`Import data`} + } /> diff --git a/public/pages/workflow_detail/workflow_inputs/input_fields/model_field.tsx b/public/pages/workflow_detail/workflow_inputs/input_fields/model_field.tsx index 1cf101e4..36ba5d4c 100644 --- a/public/pages/workflow_detail/workflow_inputs/input_fields/model_field.tsx +++ b/public/pages/workflow_detail/workflow_inputs/input_fields/model_field.tsx @@ -15,16 +15,19 @@ import { EuiCompressedSuperSelect, EuiSuperSelectOption, EuiText, - EuiSmallButton, + EuiSmallButtonIcon, + EuiFlexGroup, + EuiFlexItem, } from '@elastic/eui'; import { MODEL_STATE, WorkflowFormValues, ModelFormValue, ML_CHOOSE_MODEL_LINK, - ML_REMOTE_MODEL_LINK, + FETCH_ALL_QUERY_LARGE, } from '../../../../../common'; -import { AppState } from '../../../../store'; +import { AppState, searchModels, useAppDispatch } from '../../../../store'; +import { getDataSourceId } from '../../../../utils'; interface ModelFieldProps { fieldPath: string; // the full path in string-form to the field (e.g., 'ingest.enrich.processors.text_embedding_processor.inputField') @@ -47,6 +50,8 @@ type ModelItem = ModelFormValue & { * A specific field for selecting existing deployed models */ export function ModelField(props: ModelFieldProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); // Initial store is fetched when loading base page. We don't // re-fetch here as it could overload client-side if user clicks back and forth / // keeps re-rendering this component (and subsequently re-fetching data) as they're building flows @@ -94,31 +99,6 @@ export function ModelField(props: ModelFieldProps) { )} - {isEmpty(deployedModels) && ( - <> - - - To create and deploy models and make them accessible in - OpenSearch, see documentation. - - - - Documentation - - - - - )} {({ field, form }: FieldProps) => { const isInvalid = @@ -140,45 +120,64 @@ export function ModelField(props: ModelFieldProps) { isInvalid={isInvalid} error={props.showError && getIn(errors, `${field.name}.id`)} > - - ({ - value: option.id, - inputDisplay: ( - <> - {option.name} - - ), - dropdownDisplay: ( - <> - {option.name} - - Deployed - - - {option.algorithm} - - - ), - disabled: false, - } as EuiSuperSelectOption) - )} - valueOfSelected={field.value?.id || ''} - onChange={(option: string) => { - form.setFieldTouched(props.fieldPath, true); - form.setFieldValue(props.fieldPath, { - id: option, - } as ModelFormValue); - if (props.onModelChange) { - props.onModelChange(option); - } - }} - isInvalid={isInvalid} - /> + + + + ({ + value: option.id, + inputDisplay: ( + <> + {option.name} + + ), + dropdownDisplay: ( + <> + {option.name} + + Deployed + + + {option.algorithm} + + + ), + disabled: false, + } as EuiSuperSelectOption) + )} + valueOfSelected={field.value?.id || ''} + onChange={(option: string) => { + form.setFieldTouched(props.fieldPath, true); + form.setFieldValue(props.fieldPath, { + id: option, + } as ModelFormValue); + if (props.onModelChange) { + props.onModelChange(option); + } + }} + isInvalid={isInvalid} + /> + + + { + dispatch( + searchModels({ + apiBody: FETCH_ALL_QUERY_LARGE, + dataSourceId, + }) + ); + }} + /> + + ); }} diff --git a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx index c81abe7e..cbd67d9b 100644 --- a/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx +++ b/public/pages/workflow_detail/workflow_inputs/processor_inputs/ml_processor_inputs/ml_processor_inputs.tsx @@ -31,6 +31,7 @@ import { OutputMapArrayFormValue, EMPTY_OUTPUT_MAP_ENTRY, ML_REMOTE_MODEL_LINK, + FETCH_ALL_QUERY_LARGE, } from '../../../../../../common'; import { ModelField } from '../../input_fields'; import { @@ -39,9 +40,10 @@ import { } from '../../../../../../common'; import { OverrideQueryModal } from './modals'; import { ModelInputs } from './model_inputs'; -import { AppState } from '../../../../../store'; +import { AppState, searchModels, useAppDispatch } from '../../../../../store'; import { formikToPartialPipeline, + getDataSourceId, parseModelInputs, parseModelOutputs, } from '../../../../../utils'; @@ -62,6 +64,8 @@ interface MLProcessorInputsProps { * output map configuration forms, respectively. */ export function MLProcessorInputs(props: MLProcessorInputsProps) { + const dispatch = useAppDispatch(); + const dataSourceId = getDataSourceId(); const { models } = useSelector((state: AppState) => state.ml); const { values, setFieldValue, setFieldTouched } = useFormikContext< WorkflowFormValues @@ -191,11 +195,30 @@ export function MLProcessorInputs(props: MLProcessorInputsProps) { color="primary" size="s" title={ - - You have no models registered in your cluster.{' '} - Learn more about - integrating ML models. - + <> + + You have no models registered in your cluster.{' '} + + Learn more + {' '} + about integrating ML models. + + + { + dispatch( + searchModels({ + apiBody: FETCH_ALL_QUERY_LARGE, + dataSourceId, + }) + ); + }} + > + Refresh + + } /> ) : ( diff --git a/public/pages/workflow_detail/workflow_inputs/search_inputs/edit_query_modal.tsx b/public/pages/workflow_detail/workflow_inputs/search_inputs/edit_query_modal.tsx index e8e0d38b..0214c8c5 100644 --- a/public/pages/workflow_detail/workflow_inputs/search_inputs/edit_query_modal.tsx +++ b/public/pages/workflow_detail/workflow_inputs/search_inputs/edit_query_modal.tsx @@ -23,6 +23,7 @@ import { EuiText, EuiEmptyPrompt, EuiCallOut, + EuiSpacer, } from '@elastic/eui'; import { JsonField } from '../input_fields'; import { @@ -85,7 +86,7 @@ export function EditQueryModal(props: EditQueryModalProps) { const [popoverOpen, setPopoverOpen] = useState(false); // optional search panel state. allows searching within the modal - const [searchPanelOpen, setSearchPanelOpen] = useState(false); + const [searchPanelOpen, setSearchPanelOpen] = useState(true); // results state const [queryResponse, setQueryResponse] = useState< @@ -225,7 +226,7 @@ export function EditQueryModal(props: EditQueryModalProps) { data-testid="showOrHideSearchPanelButton" fill={false} iconType={ - searchPanelOpen ? 'menuLeft' : 'menuRight' + searchPanelOpen ? 'menuRight' : 'menuLeft' } iconSide="right" onClick={() => { @@ -260,42 +261,6 @@ export function EditQueryModal(props: EditQueryModalProps) { Test query - - { - dispatch( - searchIndex({ - apiBody: { - index: values?.search?.index?.name, - body: injectParameters( - queryParams, - tempRequest - ), - // Run the query independent of the pipeline inside this modal - searchPipeline: '_none', - }, - dataSourceId, - }) - ) - .unwrap() - .then(async (resp: SearchResponse) => { - setQueryResponse(resp); - setTempResultsError(''); - }) - .catch((error: any) => { - setQueryResponse(undefined); - const errorMsg = `Error running query: ${error}`; - setTempResultsError(errorMsg); - console.error(errorMsg); - }); - }} - > - Search - - {/** @@ -319,6 +284,41 @@ export function EditQueryModal(props: EditQueryModalProps) { Run a search to view results. + + { + dispatch( + searchIndex({ + apiBody: { + index: values?.search?.index?.name, + body: injectParameters( + queryParams, + tempRequest + ), + // Run the query independent of the pipeline inside this modal + searchPipeline: '_none', + }, + dataSourceId, + }) + ) + .unwrap() + .then(async (resp: SearchResponse) => { + setQueryResponse(resp); + setTempResultsError(''); + }) + .catch((error: any) => { + setQueryResponse(undefined); + const errorMsg = `Error running query: ${error}`; + setTempResultsError(errorMsg); + console.error(errorMsg); + }); + }} + > + Search + } /> diff --git a/public/pages/workflows/new_workflow/new_workflow.tsx b/public/pages/workflows/new_workflow/new_workflow.tsx index 1974e4e5..609a5b97 100644 --- a/public/pages/workflows/new_workflow/new_workflow.tsx +++ b/public/pages/workflows/new_workflow/new_workflow.tsx @@ -60,6 +60,7 @@ const filterPresetsByVersion = async ( WORKFLOW_TYPE.SEMANTIC_SEARCH, WORKFLOW_TYPE.MULTIMODAL_SEARCH, WORKFLOW_TYPE.HYBRID_SEARCH, + WORKFLOW_TYPE.CUSTOM, ]; const version = await getEffectiveVersion(dataSourceId); diff --git a/public/pages/workflows/workflow_list/workflow_list.test.tsx b/public/pages/workflows/workflow_list/workflow_list.test.tsx index 635ed1b1..1b1a0f6e 100644 --- a/public/pages/workflows/workflow_list/workflow_list.test.tsx +++ b/public/pages/workflows/workflow_list/workflow_list.test.tsx @@ -61,7 +61,6 @@ describe('WorkflowList', () => { }); test('renders the page', () => { const { getAllByText } = renderWithRouter(); - expect(getAllByText('Manage existing workflows').length).toBeGreaterThan(0); expect(getAllByText('Name').length).toBeGreaterThan(0); expect(getAllByText('Type').length).toBeGreaterThan(0); expect(getAllByText('Last saved').length).toBeGreaterThan(0); diff --git a/public/pages/workflows/workflow_list/workflow_list.tsx b/public/pages/workflows/workflow_list/workflow_list.tsx index 7abf6a92..c0fae1ba 100644 --- a/public/pages/workflows/workflow_list/workflow_list.tsx +++ b/public/pages/workflows/workflow_list/workflow_list.tsx @@ -6,6 +6,7 @@ import React, { useState, useEffect } from 'react'; import { useSelector } from 'react-redux'; import { debounce } from 'lodash'; +import semver from 'semver'; import { EuiInMemoryTable, Direction, @@ -19,13 +20,16 @@ import { EuiText, EuiFlyoutBody, EuiEmptyPrompt, + EuiSpacer, } from '@elastic/eui'; import { AppState } from '../../../store'; import { EMPTY_FIELD_STRING, MAX_WORKFLOW_NAME_TO_DISPLAY, + MINIMUM_FULL_SUPPORTED_VERSION, UIState, WORKFLOW_TYPE, + WORKFLOW_TYPE_LEGACY, Workflow, getCharacterLimitedString, } from '../../../../common'; @@ -38,6 +42,7 @@ import { isValidUiWorkflow } from '../../../utils'; interface WorkflowListProps { setSelectedTabId: (tabId: WORKFLOWS_TAB) => void; + dataSourceVersion?: string; } const sorting = { @@ -47,14 +52,6 @@ const sorting = { }, }; -const filterOptions = Object.values(WORKFLOW_TYPE).map((type) => { - // @ts-ignore - return { - name: type, - checked: 'on', - } as EuiFilterSelectItem; -}); - /** * The searchable list of created workflows. */ @@ -63,6 +60,19 @@ export function WorkflowList(props: WorkflowListProps) { (state: AppState) => state.workflows ); + // table filters. the list of filters depends on the datasource version, if applicable. + const isPreV219 = + props.dataSourceVersion !== undefined && + semver.lt(props.dataSourceVersion, MINIMUM_FULL_SUPPORTED_VERSION); + const filterType = isPreV219 ? WORKFLOW_TYPE_LEGACY : WORKFLOW_TYPE; + const filterOptions = Object.values(filterType).map((type) => { + // @ts-ignore + return { + name: type, + checked: 'on', + } as EuiFilterSelectItem; + }); + // actions state const [selectedWorkflow, setSelectedWorkflow] = useState< Workflow | undefined @@ -171,14 +181,7 @@ export function WorkflowList(props: WorkflowListProps) { )} - - - {`Manage existing workflows`} - - + diff --git a/public/pages/workflows/workflows.test.tsx b/public/pages/workflows/workflows.test.tsx index 169a8c4b..ae131cef 100644 --- a/public/pages/workflows/workflows.test.tsx +++ b/public/pages/workflows/workflows.test.tsx @@ -69,7 +69,6 @@ describe('Workflows', () => { queryByText('Select or drag and drop a file') ).not.toBeInTheDocument(); }); - expect(getAllByText('Manage existing workflows').length).toBeGreaterThan(0); // When the "Create Workflow" button is clicked, the "New workflow" tab opens // Create Workflow Testing diff --git a/public/pages/workflows/workflows.tsx b/public/pages/workflows/workflows.tsx index 1766eaca..ab3aff4c 100644 --- a/public/pages/workflows/workflows.tsx +++ b/public/pages/workflows/workflows.tsx @@ -38,6 +38,7 @@ import { DataSourceSelectableConfig } from '../../../../../src/plugins/data_sour import { dataSourceFilterFn, getDataSourceFromURL, + getEffectiveVersion, isDataSourceReady, } from '../../utils/utils'; import { @@ -90,6 +91,17 @@ export function Workflows(props: WorkflowsProps) { const [dataSourceId, setDataSourceId] = useState( queryParams.dataSourceId ); + const [dataSourceVersion, setDataSourceVersion] = useState< + string | undefined + >(undefined); + useEffect(() => { + async function getVersion() { + if (dataSourceId !== undefined) { + setDataSourceVersion(await getEffectiveVersion(dataSourceId)); + } + } + getVersion(); + }, [dataSourceId]); const { workflows, loading } = useSelector( (state: AppState) => state.workflows ); @@ -226,14 +238,18 @@ export function Workflows(props: WorkflowsProps) { ingest and search flows, test different configurations, and deploy them to your environment.`; const pageTitleAndDescription = USE_NEW_HOME_PAGE ? ( - + <> + + + + ) : ( @@ -246,6 +262,7 @@ export function Workflows(props: WorkflowsProps) { {DESCRIPTION} + ); @@ -265,7 +282,7 @@ export function Workflows(props: WorkflowsProps) { pageTitle={pageTitleAndDescription} bottomBorder={false} /> - {dataSourceEnabled && (dataSourceId === undefined) ? ( + {dataSourceEnabled && dataSourceId === undefined ? ( Incompatible data source} @@ -377,7 +394,10 @@ export function Workflows(props: WorkflowsProps) { bottomBorder={false} /> {selectedTabId === WORKFLOWS_TAB.MANAGE ? ( - + ) : ( <> diff --git a/public/utils/utils.ts b/public/utils/utils.tsx similarity index 97% rename from public/utils/utils.ts rename to public/utils/utils.tsx index 6101034b..4dda5c23 100644 --- a/public/utils/utils.ts +++ b/public/utils/utils.tsx @@ -3,9 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ +import React, { ReactNode } from 'react'; import yaml from 'js-yaml'; import jsonpath from 'jsonpath'; -import { escape, findKey, get, isEmpty, set, unset } from 'lodash'; +import { capitalize, escape, findKey, get, isEmpty, set, unset } from 'lodash'; +import { EuiText } from '@elastic/eui'; import semver from 'semver'; import queryString from 'query-string'; import { useLocation } from 'react-router-dom'; @@ -221,7 +223,7 @@ export function getIngestPipelineErrors( if (processorResult.error?.reason !== undefined) { ingestPipelineErrors[idx] = { processorType: processorResult.processor_type, - errorMsg: `Type: ${processorResult.processor_type}. Error: ${processorResult.error.reason}`, + errorMsg: processorResult.error.reason, }; } }); @@ -229,6 +231,7 @@ export function getIngestPipelineErrors( return ingestPipelineErrors; } +// Extract any processor-level errors from a verbose search API call export function getSearchPipelineErrors( searchResponseVerbose: SearchResponseVerbose ): SearchPipelineErrors { @@ -237,13 +240,30 @@ export function getSearchPipelineErrors( if (processorResult?.error !== undefined) { searchPipelineErrors[idx] = { processorType: processorResult.processor_name, - errorMsg: `Type: ${processorResult.processor_name}. Error: ${processorResult.error}`, + errorMsg: processorResult.error, }; } }); return searchPipelineErrors; } +// Generate a more UI-friendly layout of a processor error +export function formatProcessorError(processorError: { + processorType: string; + errorMsg: string; +}): ReactNode { + return ( + <> + + {`Processor type:`} {capitalize(processorError.processorType)} + + + {`Error:`} {processorError.errorMsg} + + + ); +} + // ML inference processors will use standard dot notation or JSONPath depending on the input. // We follow the same logic here to generate consistent results. export function generateTransform( @@ -770,9 +790,9 @@ export function getEmbeddingField( } } } else if (embedding !== undefined) { - embeddingField = embedding + embeddingField = embedding; } else if (fieldMap !== undefined) { - embeddingField = get(fieldMap, '0.value', embeddingField) + embeddingField = get(fieldMap, '0.value', embeddingField); } return embeddingField; }