Skip to content

Commit cd07da0

Browse files
authored
Onboard copy ingest processor (opensearch-project#561)
Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>
1 parent 83058e1 commit cd07da0

File tree

5 files changed

+78
-0
lines changed

5 files changed

+78
-0
lines changed

common/constants.ts

+1
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ export enum PROCESSOR_TYPE {
186186
RERANK = 'rerank',
187187
TEXT_EMBEDDING = 'text_embedding',
188188
TEXT_IMAGE_EMBEDDING = 'text_image_embedding',
189+
COPY = 'copy',
189190
}
190191

191192
export enum MODEL_TYPE {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
import { PROCESSOR_TYPE } from '../../../common';
7+
import { generateId } from '../../utils';
8+
import { Processor } from '../processor';
9+
10+
/**
11+
* The copy ingest processor
12+
*/
13+
export class CopyIngestProcessor extends Processor {
14+
constructor() {
15+
super();
16+
this.name = 'Copy Processor';
17+
this.type = PROCESSOR_TYPE.COPY;
18+
this.id = generateId('copy_processor_ingest');
19+
this.fields = [
20+
{
21+
id: 'source_field',
22+
type: 'string',
23+
},
24+
{
25+
id: 'target_field',
26+
type: 'string',
27+
},
28+
];
29+
this.optionalFields = [
30+
{
31+
id: 'ignore_missing',
32+
type: 'boolean',
33+
value: false,
34+
},
35+
{
36+
id: 'override_target',
37+
type: 'boolean',
38+
value: false,
39+
},
40+
{
41+
id: 'remove_source',
42+
type: 'boolean',
43+
value: false,
44+
},
45+
{
46+
id: 'ignore_failure',
47+
type: 'boolean',
48+
value: false,
49+
},
50+
{
51+
id: 'description',
52+
type: 'string',
53+
},
54+
{
55+
id: 'tag',
56+
type: 'string',
57+
},
58+
];
59+
}
60+
}

public/configs/ingest_processors/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ export * from './sort_ingest_processor';
99
export * from './text_chunking_ingest_processor';
1010
export * from './text_embedding_ingest_processor';
1111
export * from './text_image_embedding_ingest_processor';
12+
export * from './copy_ingest_processor';

public/pages/workflow_detail/workflow_inputs/processors_list.tsx

+8
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import { formikToUiConfig, getDataSourceFromURL } from '../../../utils';
2929

3030
import {
3131
CollapseProcessor,
32+
CopyIngestProcessor,
3233
MLIngestProcessor,
3334
MLSearchRequestProcessor,
3435
MLSearchResponseProcessor,
@@ -164,6 +165,13 @@ export function ProcessorsList(props: ProcessorsListProps) {
164165
addProcessor(new TextChunkingIngestProcessor().toObj());
165166
},
166167
},
168+
{
169+
name: 'Copy Processor',
170+
onClick: () => {
171+
closePopover();
172+
addProcessor(new CopyIngestProcessor().toObj());
173+
},
174+
},
167175
];
168176

169177
const searchRequestProcessors = [

public/utils/config_to_workspace_utils.ts

+8
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,14 @@ function processorsConfigToWorkspaceFlow(
355355
);
356356
break;
357357
}
358+
case PROCESSOR_TYPE.COPY: {
359+
transformer = new BaseTransformer(
360+
processorConfig.name,
361+
'Copy an entire object in an existing field to another field',
362+
context
363+
);
364+
break;
365+
}
358366
case PROCESSOR_TYPE.NORMALIZATION: {
359367
componentData = new BaseTransformer(
360368
processorConfig.name,

0 commit comments

Comments
 (0)