diff --git a/common/constants.ts b/common/constants.ts index 223580a1..d34324ed 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -301,27 +301,108 @@ export const VECTOR_TEMPLATE_PLACEHOLDER = `\$\{${VECTOR}\}`; export const DEFAULT_K = 10; export const DEFAULT_FETCH_SIZE = 10; -export const FETCH_ALL_QUERY = { +// term-level queries +export const TERM_QUERY_TEXT = { query: { - match_all: {}, + term: { + [TEXT_FIELD_PATTERN]: { + value: QUERY_TEXT_PATTERN, + }, + }, }, - size: DEFAULT_FETCH_SIZE, }; -export const FETCH_ALL_QUERY_LARGE = { +export const EXISTS_QUERY_TEXT = { query: { - match_all: {}, + exists: { + field: TEXT_FIELD_PATTERN, + }, }, - size: 1000, }; -export const TERM_QUERY_TEXT = { +export const FUZZY_QUERY_TEXT = { query: { - term: { + fuzzy: { [TEXT_FIELD_PATTERN]: { value: QUERY_TEXT_PATTERN, }, }, }, }; +export const WILDCARD_QUERY_TEXT = { + query: { + wildcard: { + [TEXT_FIELD_PATTERN]: { + wildcard: QUERY_TEXT_PATTERN, + case_insensitive: false, + }, + }, + }, +}; +export const PREFIX_QUERY_TEXT = { + query: { + prefix: { + [TEXT_FIELD_PATTERN]: { + value: QUERY_TEXT_PATTERN, + }, + }, + }, +}; +// full-text queries +export const MATCH_QUERY_TEXT = { + query: { + match: { + [TEXT_FIELD_PATTERN]: { + query: QUERY_TEXT_PATTERN, + }, + }, + }, +}; +export const MATCH_BOOLEAN_QUERY_TEXT = { + query: { + match_bool_prefix: { + [TEXT_FIELD_PATTERN]: { + query: QUERY_TEXT_PATTERN, + }, + }, + }, +}; +export const MATCH_PHRASE_QUERY_TEXT = { + query: { + match_phrase: { + [TEXT_FIELD_PATTERN]: { + query: QUERY_TEXT_PATTERN, + }, + }, + }, +}; +export const MATCH_PHRASE_PREFIX_QUERY_TEXT = { + query: { + match_phrase_prefix: { + [TEXT_FIELD_PATTERN]: { + query: QUERY_TEXT_PATTERN, + }, + }, + }, +}; +export const QUERY_STRING_QUERY_TEXT = { + query: { + query_string: { + query: QUERY_TEXT_PATTERN, + }, + }, +}; +// misc / other queries +export const FETCH_ALL_QUERY = { + query: { + match_all: {}, + }, + size: DEFAULT_FETCH_SIZE, +}; +export const FETCH_ALL_QUERY_LARGE = { + query: { + match_all: {}, + }, + size: 1000, +}; export const KNN_QUERY = { _source: { excludes: [VECTOR_FIELD_PATTERN], @@ -471,6 +552,42 @@ export const QUERY_PRESETS = [ name: 'Term', query: customStringify(TERM_QUERY_TEXT), }, + { + name: 'Match', + query: customStringify(MATCH_QUERY_TEXT), + }, + { + name: 'Exists', + query: customStringify(EXISTS_QUERY_TEXT), + }, + { + name: 'Fuzzy', + query: customStringify(FUZZY_QUERY_TEXT), + }, + { + name: 'Wildcard', + query: customStringify(WILDCARD_QUERY_TEXT), + }, + { + name: 'Prefix', + query: customStringify(PREFIX_QUERY_TEXT), + }, + { + name: 'Match boolean', + query: customStringify(MATCH_BOOLEAN_QUERY_TEXT), + }, + { + name: 'Match phrase', + query: customStringify(MATCH_PHRASE_QUERY_TEXT), + }, + { + name: 'Match phrase prefix', + query: customStringify(MATCH_PHRASE_PREFIX_QUERY_TEXT), + }, + { + name: 'Query string', + query: customStringify(QUERY_STRING_QUERY_TEXT), + }, { name: 'Basic k-NN', query: customStringify(KNN_QUERY), diff --git a/public/utils/config_to_template_utils.test.tsx b/public/utils/config_to_template_utils.test.tsx new file mode 100644 index 00000000..e594bd24 --- /dev/null +++ b/public/utils/config_to_template_utils.test.tsx @@ -0,0 +1,119 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import '@testing-library/jest-dom'; +import { updatePathForExpandedQuery } from './config_to_template_utils'; + +describe('config_to_template_utils', () => { + beforeEach(() => {}); + describe('updatePathForExpandedQuery', () => { + test('term query', () => { + expect( + updatePathForExpandedQuery('query.term.a') === 'query.term.a.value' + ); + expect( + updatePathForExpandedQuery('query.term.a.value') === + 'query.term.a.value' + ); + expect( + updatePathForExpandedQuery('$.query.term.a') === '$.query.term.a.value' + ); + expect( + updatePathForExpandedQuery('a.b.c.d.query.term.a') === + 'a.b.c.d.query.term.a.value' + ); + expect( + updatePathForExpandedQuery('query.bool.must[0].term.a') === + 'query.bool.must[0].term.a.value' + ); + }); + test('prefix query', () => { + expect( + updatePathForExpandedQuery('query.prefix.a') === 'query.prefix.a.value' + ); + expect( + updatePathForExpandedQuery('query.prefix.a.value') === + 'query.prefix.a.value' + ); + }); + test('fuzzy query', () => { + expect( + updatePathForExpandedQuery('query.fuzzy.a') === 'query.fuzzy.a.value' + ); + expect( + updatePathForExpandedQuery('query.fuzzy.a.value') === + 'query.fuzzy.a.value' + ); + }); + test('wildcard query', () => { + expect( + updatePathForExpandedQuery('query.wildcard.a') === + 'query.wildcard.a.wildcard' + ); + expect( + updatePathForExpandedQuery('query.wildcard.a.wildcard') === + 'query.wildcard.a.wildcard' + ); + }); + test('regexp query', () => { + expect( + updatePathForExpandedQuery('query.regexp.a') === 'query.regexp.a.value' + ); + expect( + updatePathForExpandedQuery('query.regexp.a.value') === + 'query.regexp.a.value' + ); + }); + test('match query', () => { + expect( + updatePathForExpandedQuery('query.match.a') === 'query.match.a.query' + ); + expect( + updatePathForExpandedQuery('query.match.a.query') === + 'query.match.a.query' + ); + }); + test('match bool prefix query', () => { + expect( + updatePathForExpandedQuery('query.match_bool_prefix.a') === + 'query.match_bool_prefix.a.query' + ); + expect( + updatePathForExpandedQuery('query.match_bool_prefix.a.query') === + 'query.match_bool_prefix.a.query' + ); + }); + test('match phrase query', () => { + expect( + updatePathForExpandedQuery('query.match_phrase.a') === + 'query.match_phrase.a.query' + ); + expect( + updatePathForExpandedQuery('query.match_phrase.a.query') === + 'query.match_phrase.a.query' + ); + }); + test('match phrase prefix query', () => { + expect( + updatePathForExpandedQuery('query.match_phrase_prefix.a') === + 'query.match_phrase_prefix.a.query' + ); + expect( + updatePathForExpandedQuery('query.match_phrase_prefix.a.query') === + 'query.match_phrase_prefix.a.query' + ); + }); + test('aggs query', () => { + expect( + updatePathForExpandedQuery('aggs.avg_a.avg.field') === + 'aggregations.avg_a.avg.field' + ); + expect( + updatePathForExpandedQuery('aggs.b.c.d.aggs.avg_a.avg.field') === + 'aggregations.b.c.d.aggregations.avg_a.avg.field' + ); + }); + }); +}); diff --git a/public/utils/config_to_template_utils.ts b/public/utils/config_to_template_utils.ts index 65c7b093..fb305eb3 100644 --- a/public/utils/config_to_template_utils.ts +++ b/public/utils/config_to_template_utils.ts @@ -37,6 +37,7 @@ import { TRANSFORM_TYPE, OutputMapFormValue, NO_TRANSFORMATION, + PROCESSOR_CONTEXT, } from '../../common'; import { processorConfigToFormik } from './config_to_form_utils'; import { sanitizeJSONPath } from './utils'; @@ -105,7 +106,8 @@ function ingestConfigToTemplateNodes( const ingestPipelineName = ingestConfig.pipelineName.value; const ingestEnabled = ingestConfig.enabled.value; const ingestProcessors = processorConfigsToTemplateProcessors( - ingestConfig.enrich.processors + ingestConfig.enrich.processors, + PROCESSOR_CONTEXT.INGEST ); const hasProcessors = ingestProcessors.length > 0; @@ -132,7 +134,8 @@ function searchConfigToTemplateNodes( ): TemplateNode[] { const searchPipelineName = searchConfig.pipelineName.value; const searchRequestProcessors = processorConfigsToTemplateProcessors( - searchConfig.enrichRequest.processors + searchConfig.enrichRequest.processors, + PROCESSOR_CONTEXT.SEARCH_REQUEST ); // For the configured response processors, we don't maintain separate UI / config // between response processors and phase results processors. So, we parse @@ -143,12 +146,14 @@ function searchConfigToTemplateNodes( (processor) => processor.type === PROCESSOR_TYPE.NORMALIZATION ); const phaseResultsProcessors = processorConfigsToTemplateProcessors( - normalizationProcessor ? [normalizationProcessor] : [] + normalizationProcessor ? [normalizationProcessor] : [], + PROCESSOR_CONTEXT.SEARCH_RESPONSE ); const searchResponseProcessors = processorConfigsToTemplateProcessors( searchConfig.enrichResponse.processors.filter( (processor) => processor.type !== PROCESSOR_TYPE.NORMALIZATION - ) + ), + PROCESSOR_CONTEXT.SEARCH_RESPONSE ); const hasProcessors = searchRequestProcessors.length > 0 || @@ -177,7 +182,8 @@ function searchConfigToTemplateNodes( // General fn to process all processor configs and convert them // into a final list of template-formatted IngestProcessor/SearchProcessors. export function processorConfigsToTemplateProcessors( - processorConfigs: IProcessorConfig[] + processorConfigs: IProcessorConfig[], + context: PROCESSOR_CONTEXT ): (IngestProcessor | SearchProcessor)[] { const processorsList = [] as (IngestProcessor | SearchProcessor)[]; @@ -214,7 +220,7 @@ export function processorConfigsToTemplateProcessors( if (input_map?.length > 0) { processor.ml_inference.input_map = input_map.map( (inputMapFormValue: InputMapFormValue) => { - const res = processModelInputs(inputMapFormValue); + const res = processModelInputs(inputMapFormValue, context); if (!isEmpty(res.modelConfig)) { modelConfig = { ...modelConfig, @@ -552,7 +558,8 @@ function mergeMapIntoSingleObj( // Bucket the model inputs configured on the UI as input map entries containing dynamic data, // or model config entries containing static data. function processModelInputs( - mapFormValue: InputMapFormValue + mapFormValue: InputMapFormValue, + context: PROCESSOR_CONTEXT ): { inputMap: {}; modelConfig: {} } { let inputMap = {}; let modelConfig = {}; @@ -563,11 +570,14 @@ function processModelInputs( mapEntry.value.transformType === TRANSFORM_TYPE.EXPRESSION) && !isEmpty(mapEntry.value.value) ) { + const inputValue = + context === PROCESSOR_CONTEXT.SEARCH_REQUEST + ? updatePathForExpandedQuery(mapEntry.value.value as string) + : (mapEntry.value.value as string); + inputMap = { ...inputMap, - [sanitizeJSONPath(mapEntry.key)]: sanitizeJSONPath( - mapEntry.value.value as string - ), + [sanitizeJSONPath(mapEntry.key)]: sanitizeJSONPath(inputValue), }; // template with dynamic nested vars. Add the nested vars as input map entries, // and add the static template itself to the model config. @@ -659,3 +669,56 @@ function optionallyAddToFinalForm( } return finalFormValues; } + +// Try to catch and update paths when they are defined on a non-expanded version of a query. +// For more details & examples, see +// https://github.com/opensearch-project/dashboards-flow-framework/issues/574 +export function updatePathForExpandedQuery(path: string): string { + let updatedPath = path; + + // Several query types in expanded form nest the search value under a sub-field like "value" or "query". + // Update the path accordingly if it is defined on the non-expanded form of a query. + updatedPath = addSuffixToPath(updatedPath, 'term', 'value'); + updatedPath = addSuffixToPath(updatedPath, 'prefix', 'value'); + updatedPath = addSuffixToPath(updatedPath, 'fuzzy', 'value'); + updatedPath = addSuffixToPath(updatedPath, 'wildcard', 'wildcard'); + updatedPath = addSuffixToPath(updatedPath, 'regexp', 'value'); + updatedPath = addSuffixToPath(updatedPath, 'match', 'query'); + updatedPath = addSuffixToPath(updatedPath, 'match_bool_prefix', 'query'); + updatedPath = addSuffixToPath(updatedPath, 'match_phrase', 'query'); + updatedPath = addSuffixToPath(updatedPath, 'match_phrase_prefix', 'query'); + + // "aggs" expands to "aggregations" + updatedPath = updateAggsPath(updatedPath); + + // TODO handle range query + // TODO handle geo / xy queries + // TODO handle "fields" when returning subset of fields in the source response + // ^ all tracked in https://github.com/opensearch-project/dashboards-flow-framework/issues/574 + + return updatedPath; +} + +// Adds the appropriate suffix to the path, if not already found. +// For example, given some path "query.term.a", prefix "term", and suffix "value", +// then append the suffix to produce the final path "query.term.a.value". +// If the path already has the suffix present, do nothing. +function addSuffixToPath(path: string, prefix: string, suffix: string): string { + function generateRegex(prefix: string, suffix: string): RegExp { + // match the specified prefix, followed by some value in dot or bracket notation + const notationPattern = `\\b${prefix}\\b(\\.\\w+|\\[\\w+\\])`; + // ensure the suffix (in dot or bracket notation) is not present + const suffixPattern = `(?!(\\.${suffix}|\\[${suffix}\\]))`; + return new RegExp(notationPattern + suffixPattern, 'g'); + } + + // if the pattern matches, append the appropriate suffix via dot notation + const regexPattern = generateRegex(prefix, suffix); + return path.replace(regexPattern, (_, subStr) => { + return `${prefix}${subStr}.${suffix}`; + }); +} + +function updateAggsPath(path: string): string { + return path.replace(/\baggs\b/g, 'aggregations'); +}