Skip to content

Commit f2378fa

Browse files
ohltylergithub-actions[bot]
ohltyler
authored and
github-actions[bot]
committed
Set up basic logic for parsing template -> UI flow and vice versa (#131)
Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com> (cherry picked from commit 34ef573)
1 parent fa7ff85 commit f2378fa

File tree

18 files changed

+568
-342
lines changed

18 files changed

+568
-342
lines changed

‎common/constants.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55

6-
import { WORKFLOW_STATE } from './interfaces';
6+
import { TemplateNode, WORKFLOW_STATE } from './interfaces';
77

88
export const PLUGIN_ID = 'flow-framework';
99

@@ -47,6 +47,12 @@ export const GET_PRESET_WORKFLOWS_NODE_API_PATH = `${BASE_WORKFLOW_NODE_API_PATH
4747
export const BASE_MODEL_NODE_API_PATH = `${BASE_NODE_API_PATH}/model`;
4848
export const SEARCH_MODELS_NODE_API_PATH = `${BASE_MODEL_NODE_API_PATH}/search`;
4949

50+
/**
51+
* BACKEND INTERFACES
52+
*/
53+
export const CREATE_INGEST_PIPELINE_STEP_TYPE = 'create_ingest_pipeline';
54+
export const CREATE_INDEX_STEP_TYPE = 'create_index';
55+
5056
/**
5157
* MISCELLANEOUS
5258
*/

‎common/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ export * from './interfaces';
88
export * from './utils';
99
export * from '../public/component_types';
1010
export * from '../public/utils';
11+
export * from '../public/pages/workflow_detail/utils';

‎common/interfaces.ts

+42-7
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ type ReactFlowViewport = {
2929
};
3030

3131
export type UIState = {
32-
workspaceFlow: WorkspaceFlowState;
32+
workspace_flow: WorkspaceFlowState;
3333
};
3434

3535
export type WorkspaceFlowState = {
@@ -42,21 +42,56 @@ export type WorkspaceFlowState = {
4242
********** USE CASE TEMPLATE TYPES/INTERFACES **********
4343
*/
4444

45+
export type IngestProcessor = {
46+
description?: string;
47+
};
48+
49+
export type TextEmbeddingProcessor = IngestProcessor & {
50+
text_embedding: {
51+
model_id: string;
52+
field_map: {};
53+
};
54+
};
55+
4556
export type TemplateNode = {
4657
id: string;
4758
type: string;
48-
previous_node_inputs?: Map<string, any>;
49-
user_inputs?: Map<string, any>;
59+
previous_node_inputs?: {};
60+
user_inputs?: {};
61+
};
62+
63+
export type CreateIngestPipelineNode = TemplateNode & {
64+
user_inputs: {
65+
pipeline_id: string;
66+
model_id?: string;
67+
input_field?: string;
68+
output_field?: string;
69+
configurations: {
70+
description?: string;
71+
processors: IngestProcessor[];
72+
};
73+
};
74+
};
75+
76+
export type CreateIndexNode = TemplateNode & {
77+
previous_node_inputs?: {
78+
[ingest_pipeline_step_id: string]: string;
79+
};
80+
user_inputs: {
81+
index_name: string;
82+
configurations: {
83+
settings: {};
84+
mappings: {};
85+
};
86+
};
5087
};
5188

5289
export type TemplateEdge = {
5390
source: string;
54-
target: string;
91+
dest: string;
5592
};
5693

5794
export type TemplateFlow = {
58-
user_inputs?: Map<string, any>;
59-
previous_node_inputs?: Map<string, any>;
6095
nodes: TemplateNode[];
6196
edges?: TemplateEdge[];
6297
};
@@ -91,7 +126,7 @@ export type Workflow = WorkflowTemplate & {
91126
};
92127

93128
export enum USE_CASE {
94-
PROVISION = 'PROVISION',
129+
SEMANTIC_SEARCH = 'SEMANTIC_SEARCH',
95130
}
96131

97132
/**

‎common/utils.ts

+1-211
Original file line numberDiff line numberDiff line change
@@ -4,217 +4,7 @@
44
*/
55

66
import moment from 'moment';
7-
import { MarkerType } from 'reactflow';
8-
import {
9-
WorkspaceFlowState,
10-
ReactFlowComponent,
11-
initComponentData,
12-
TextEmbeddingTransformer,
13-
KnnIndexer,
14-
generateId,
15-
ReactFlowEdge,
16-
TemplateFlows,
17-
WorkflowTemplate,
18-
DATE_FORMAT_PATTERN,
19-
COMPONENT_CATEGORY,
20-
NODE_CATEGORY,
21-
WorkspaceFormValues,
22-
} from './';
23-
24-
// TODO: implement this and remove hardcoded return values
25-
/**
26-
* Given a ReactFlow workspace flow and the set of current form values within such flow,
27-
* generate a backend-compatible set of sub-workflows.
28-
*
29-
*/
30-
export function toTemplateFlows(
31-
workspaceFlow: WorkspaceFlowState,
32-
formValues: WorkspaceFormValues
33-
): TemplateFlows {
34-
const textEmbeddingTransformerNodeId = Object.keys(formValues).find((key) =>
35-
key.includes('text_embedding')
36-
) as string;
37-
const knnIndexerNodeId = Object.keys(formValues).find((key) =>
38-
key.includes('knn')
39-
) as string;
40-
const textEmbeddingFields = formValues[textEmbeddingTransformerNodeId];
41-
const knnIndexerFields = formValues[knnIndexerNodeId];
42-
43-
return {
44-
provision: {
45-
nodes: [
46-
{
47-
id: 'create_ingest_pipeline',
48-
type: 'create_ingest_pipeline',
49-
user_inputs: {
50-
pipeline_id: 'test-pipeline',
51-
model_id: textEmbeddingFields['modelId'],
52-
input_field: textEmbeddingFields['inputField'],
53-
output_field: textEmbeddingFields['vectorField'],
54-
configurations: {
55-
description: 'A text embedding ingest pipeline',
56-
processors: [
57-
{
58-
text_embedding: {
59-
model_id: textEmbeddingFields['modelId'],
60-
field_map: {
61-
[textEmbeddingFields['inputField']]:
62-
textEmbeddingFields['vectorField'],
63-
},
64-
},
65-
},
66-
],
67-
},
68-
},
69-
},
70-
{
71-
id: 'create_index',
72-
type: 'create_index',
73-
previous_node_inputs: {
74-
create_ingest_pipeline: 'pipeline_id',
75-
},
76-
user_inputs: {
77-
index_name: knnIndexerFields['indexName'],
78-
configurations: {
79-
settings: {
80-
default_pipeline: '${{create_ingest_pipeline.pipeline_id}}',
81-
},
82-
mappings: {
83-
properties: {
84-
[textEmbeddingFields['vectorField']]: {
85-
type: 'knn_vector',
86-
dimension: 768,
87-
method: {
88-
engine: 'lucene',
89-
space_type: 'l2',
90-
name: 'hnsw',
91-
parameters: {},
92-
},
93-
},
94-
[textEmbeddingFields['inputField']]: {
95-
type: 'text',
96-
},
97-
},
98-
},
99-
},
100-
},
101-
},
102-
],
103-
},
104-
};
105-
}
106-
107-
// TODO: implement this and remove hardcoded return values
108-
/**
109-
* Converts a backend set of provision/ingest/search sub-workflows into a UI-compatible set of
110-
* ReactFlow nodes and edges
111-
*/
112-
export function toWorkspaceFlow(
113-
templateFlows: TemplateFlows
114-
): WorkspaceFlowState {
115-
const ingestId1 = generateId('text_embedding_processor');
116-
const ingestId2 = generateId('knn_index');
117-
const ingestGroupId = generateId(COMPONENT_CATEGORY.INGEST);
118-
const searchGroupId = generateId(COMPONENT_CATEGORY.SEARCH);
119-
const edgeId = generateId('edge');
120-
121-
const ingestNodes = [
122-
{
123-
id: ingestGroupId,
124-
position: { x: 400, y: 400 },
125-
type: NODE_CATEGORY.INGEST_GROUP,
126-
data: { label: COMPONENT_CATEGORY.INGEST },
127-
style: {
128-
width: 900,
129-
height: 400,
130-
},
131-
className: 'reactflow__group-node__ingest',
132-
selectable: true,
133-
deletable: false,
134-
},
135-
{
136-
id: ingestId1,
137-
position: { x: 100, y: 70 },
138-
data: initComponentData(
139-
new TextEmbeddingTransformer().toObj(),
140-
ingestId1
141-
),
142-
type: NODE_CATEGORY.CUSTOM,
143-
parentNode: ingestGroupId,
144-
extent: 'parent',
145-
draggable: true,
146-
deletable: false,
147-
},
148-
{
149-
id: ingestId2,
150-
position: { x: 500, y: 70 },
151-
data: initComponentData(new KnnIndexer().toObj(), ingestId2),
152-
type: NODE_CATEGORY.CUSTOM,
153-
parentNode: ingestGroupId,
154-
extent: 'parent',
155-
draggable: true,
156-
deletable: false,
157-
},
158-
] as ReactFlowComponent[];
159-
160-
const searchNodes = [
161-
{
162-
id: searchGroupId,
163-
position: { x: 400, y: 1000 },
164-
type: NODE_CATEGORY.SEARCH_GROUP,
165-
data: { label: COMPONENT_CATEGORY.SEARCH },
166-
style: {
167-
width: 900,
168-
height: 400,
169-
},
170-
className: 'reactflow__group-node__search',
171-
selectable: true,
172-
deletable: false,
173-
},
174-
] as ReactFlowComponent[];
175-
176-
return {
177-
nodes: [...ingestNodes, ...searchNodes],
178-
edges: [
179-
{
180-
id: edgeId,
181-
key: edgeId,
182-
source: ingestId1,
183-
target: ingestId2,
184-
markerEnd: {
185-
type: MarkerType.ArrowClosed,
186-
width: 20,
187-
height: 20,
188-
},
189-
zIndex: 2,
190-
deletable: false,
191-
},
192-
] as ReactFlowEdge[],
193-
};
194-
}
195-
196-
// TODO: implement this
197-
/**
198-
* Validates the UI workflow state.
199-
* Note we don't have to validate connections since that is done via input/output handlers.
200-
* But we need to validate there are no open connections
201-
*/
202-
export function validateWorkspaceFlow(
203-
workspaceFlow: WorkspaceFlowState
204-
): boolean {
205-
return true;
206-
}
207-
208-
// TODO: implement this
209-
/**
210-
* Validates the backend template. May be used when parsing persisted templates on server-side,
211-
* or when importing/exporting on the UI.
212-
*/
213-
export function validateWorkflowTemplate(
214-
workflowTemplate: WorkflowTemplate
215-
): boolean {
216-
return true;
217-
}
7+
import { DATE_FORMAT_PATTERN } from './';
2188

2199
export function toFormattedDate(timestampMillis: number): String {
22010
return moment(new Date(timestampMillis)).format(DATE_FORMAT_PATTERN);

‎public/component_types/indexer/knn_indexer.ts

+3
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55

6+
import { COMPONENT_CLASS } from '../../../common';
67
import { Indexer } from './indexer';
78

89
/**
@@ -11,8 +12,10 @@ import { Indexer } from './indexer';
1112
export class KnnIndexer extends Indexer {
1213
constructor() {
1314
super();
15+
this.type = COMPONENT_CLASS.KNN_INDEXER;
1416
this.label = 'K-NN Indexer';
1517
this.description = 'A specialized indexer for K-NN indices';
18+
this.baseClasses = [...this.baseClasses, this.type];
1619
this.createFields = [
1720
// @ts-ignore
1821
...this.createFields,

‎public/component_types/transformer/text_embedding_transformer.ts

+3
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* SPDX-License-Identifier: Apache-2.0
44
*/
55

6+
import { COMPONENT_CLASS } from '../../../common';
67
import { MLTransformer } from '.';
78

89
/**
@@ -11,8 +12,10 @@ import { MLTransformer } from '.';
1112
export class TextEmbeddingTransformer extends MLTransformer {
1213
constructor() {
1314
super();
15+
this.type = COMPONENT_CLASS.TEXT_EMBEDDING_TRANSFORMER;
1416
this.label = 'Text Embedding Transformer';
1517
this.description = 'A specialized ML transformer for embedding text';
18+
this.baseClasses = [...this.baseClasses, this.type];
1619
this.inputs = [];
1720
this.createFields = [
1821
{

‎public/pages/workflow_detail/utils/index.ts

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
export * from './utils';
7+
export * from './workflow_to_template_utils';

‎public/pages/workflow_detail/utils/utils.ts

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
import { WorkspaceFlowState } from '../../../../common';
7+
8+
// TODO: implement this
9+
/**
10+
* Validates the UI workflow state.
11+
* Note we don't have to validate connections since that is done via input/output handlers.
12+
* But we need to validate there are no open connections
13+
*/
14+
export function validateWorkspaceFlow(
15+
workspaceFlow: WorkspaceFlowState
16+
): boolean {
17+
return true;
18+
}

0 commit comments

Comments
 (0)