Skip to content

Commit cd1dd23

Browse files
SuZhou-JoeSuZhou-Joe
authored
feat: Setup assistant agent & workflow file. (opensearch-project#1002)
* feat: enable setup Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: using a external mock service to mock a dummy LLM Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: use local dummy server Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: add some basic test cases Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: run with server Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: use ml-commons api to setup test agent manually Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: remove flow_framework.enabled config Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: remove visualization as skills repo is not bundled in snapshot Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: add flag config Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: optimize stop command Signed-off-by: SuZhou-Joe <suzhou@amazon.com> * feat: move setup steps to support/index.js Signed-off-by: SuZhou-Joe <suzhou@amazon.com> --------- Signed-off-by: SuZhou-Joe <suzhou@amazon.com>
1 parent 860444c commit cd1dd23

File tree

12 files changed

+487
-2
lines changed

12 files changed

+487
-2
lines changed

‎.github/workflows/assistant-release-e2e-workflow.yml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: Assistant Release tests workflow in Bundled OpenSearch Dashboards
2+
on:
3+
pull_request:
4+
branches: ['**']
5+
jobs:
6+
changes:
7+
runs-on: ubuntu-latest
8+
outputs:
9+
tests: ${{ steps.filter.outputs.tests }}
10+
steps:
11+
- uses: dorny/paths-filter@v2
12+
id: filter
13+
with:
14+
filters: |
15+
tests:
16+
- 'cypress/**/dashboards-assistant/**'
17+
18+
tests:
19+
needs: changes
20+
if: ${{ needs.changes.outputs.tests == 'true' }}
21+
uses: ./.github/workflows/release-e2e-workflow-template.yml
22+
with:
23+
test-name: dashboards assistant
24+
test-command: env CYPRESS_DASHBOARDS_ASSISTANT_ENABLED=true yarn cypress:run-with-security --browser chromium --spec 'cypress/integration/plugins/dashboards-assistant/*'
25+
osd-serve-args: --assistant.chat.enabled=true --assistant.chat.rootAgentName="Cypress test agent"

‎cypress.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"VISBUILDER_ENABLED": true,
2121
"DATASOURCE_MANAGEMENT_ENABLED": false,
2222
"ML_COMMONS_DASHBOARDS_ENABLED": true,
23-
"WAIT_FOR_LOADER_BUFFER_MS": 0
23+
"WAIT_FOR_LOADER_BUFFER_MS": 0,
24+
"DASHBOARDS_ASSISTANT_ENABLED": false
2425
}
2526
}

‎cypress/fixtures/plugins/dashboards-assistant/agent-framework-response.json

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"completion": " ```json\n{\n \"thought\": \"Now I know the final answer\",\n \"final_answer\": \"The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices. This included index names like .plugins-ml-model-group, security-auditlog-2024.01.16, opensearch_dashboards_sample_data_ecommerce and others along with health, status and other details.\"\n}\n```\n",
3+
"stop_reason": "stop_sequence",
4+
"stop": "\n\nHuman:"
5+
}

‎cypress/fixtures/plugins/dashboards-assistant/cluster_settings.json

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"persistent": {
3+
"plugins.ml_commons.only_run_on_ml_node": false,
4+
"plugins.ml_commons.memory_feature_enabled": true,
5+
"plugins.ml_commons.trusted_connector_endpoints_regex": [
6+
"^http://127.0.0.1:3000$"
7+
]
8+
}
9+
}

‎cypress/fixtures/plugins/dashboards-assistant/flow-template.json

+157
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
{
2+
"name": "Cypress-register-agent",
3+
"description": "Cypress Flow template",
4+
"use_case": "REGISTER_AGENT",
5+
"version": {
6+
"template": "1.0.0",
7+
"compatibility": ["2.12.0", "3.0.0"]
8+
},
9+
"workflows": {
10+
"provision": {
11+
"user_params": {},
12+
"nodes": [
13+
{
14+
"id": "create_connector_1",
15+
"type": "create_connector",
16+
"previous_node_inputs": {},
17+
"user_inputs": {
18+
"version": "1",
19+
"name": "Claude instant runtime Connector",
20+
"protocol": "aws_sigv4",
21+
"description": "The connector to BedRock service for claude model",
22+
"actions": [
23+
{
24+
"headers": {
25+
"x-amz-content-sha256": "required",
26+
"content-type": "application/json"
27+
},
28+
"method": "GET",
29+
"request_body": "{\"prompt\":\"${parameters.prompt}\", \"max_tokens_to_sample\":${parameters.max_tokens_to_sample}, \"temperature\":${parameters.temperature}, \"anthropic_version\":\"${parameters.anthropic_version}\" }",
30+
"action_type": "predict",
31+
"url": "http://127.0.0.1:3000"
32+
}
33+
],
34+
"credential": {
35+
"access_key": "<key>",
36+
"secret_key": "<value>"
37+
},
38+
"parameters": {
39+
"endpoint": "bedrock-runtime.us-west-2.amazonaws.com",
40+
"content_type": "application/json",
41+
"auth": "Sig_V4",
42+
"max_tokens_to_sample": "8000",
43+
"service_name": "bedrock",
44+
"temperature": "0.0001",
45+
"response_filter": "$.completion",
46+
"region": "us-west-2",
47+
"anthropic_version": "bedrock-2023-05-31"
48+
}
49+
}
50+
},
51+
{
52+
"id": "register_model_2",
53+
"type": "register_remote_model",
54+
"previous_node_inputs": {
55+
"create_connector_1": "connector_id"
56+
},
57+
"user_inputs": {
58+
"description": "test model",
59+
"deploy": true,
60+
"name": "claude-instant"
61+
}
62+
},
63+
{
64+
"id": "cat_index_tool",
65+
"type": "create_tool",
66+
"previous_node_inputs": {},
67+
"user_inputs": {
68+
"type": "CatIndexTool",
69+
"name": "CatIndexTool",
70+
"description": "Use this tool to get OpenSearch index information: (health, status, index, uuid, primary count, replica count, docs.count, docs.deleted, store.size, primary.store.size).",
71+
"parameters": {
72+
"index": ".kibana"
73+
}
74+
}
75+
},
76+
{
77+
"id": "sub_agent",
78+
"type": "register_agent",
79+
"previous_node_inputs": {
80+
"cat_index_tool": "tools",
81+
"register_model_2": "model_id"
82+
},
83+
"user_inputs": {
84+
"parameters": {},
85+
"app_type": "chatbot",
86+
"name": "Cypress test sub Agent",
87+
"description": "this is a test agent",
88+
"llm.parameters": {
89+
"max_iteration": "5",
90+
"stop_when_no_tool_found": "true",
91+
"response_filter": "$.completion"
92+
},
93+
"memory": {
94+
"type": "conversation_index"
95+
},
96+
"type": "conversational"
97+
}
98+
},
99+
{
100+
"id": "agent_tool",
101+
"type": "create_tool",
102+
"previous_node_inputs": {
103+
"sub_agent": "agent_id"
104+
},
105+
"user_inputs": {
106+
"description": "Agent Tool",
107+
"include_output_in_agent_response": true,
108+
"type": "AgentTool",
109+
"parameters": {
110+
"max_iteration": "5"
111+
},
112+
"name": "AgentTool"
113+
}
114+
},
115+
{
116+
"id": "ml_model_tool",
117+
"type": "create_tool",
118+
"previous_node_inputs": {
119+
"register_model_2": "model_id"
120+
},
121+
"user_inputs": {
122+
"parameters": {
123+
"prompt": "\n\nHuman:\" turn\" You are an AI that only speaks JSON. Do not write normal text. Output should follow example JSON format: \n\n {\"response\": [\"question1\", \"question2\"]}\n\n. \n\nHuman:\" turn\":You will be given a chat history between OpenSearch Assistant and a Human.\nUse the context provided to generate follow up questions the Human would ask to the Assistant.\nThe Assistant can answer general questions about logs, traces and metrics.\nAssistant can access a set of tools listed below to answer questions given by the Human:\nQuestion suggestions generator tool\nHere's the chat history between the human and the Assistant.\n${parameters.AgentTool.output}\nUse the following steps to generate follow up questions Human may ask after the response of the Assistant:\nStep 1. Use the chat history to understand what human is trying to search and explore.\nStep 2. Understand what capabilities the assistant has with the set of tools it has access to.\nStep 3. Use the above context and generate follow up questions.Step4:You are an AI that only speaks JSON. Do not write normal text. Output should follow example JSON format: \n\n {\"response\": [\"question1\", \"question2\"]} \n \n----------------\n\nAssistant:"
124+
},
125+
"description": "A general tool to answer any question.",
126+
"alias": "language_model_tool",
127+
"include_output_in_agent_response": true,
128+
"name": "QuestionSuggestor",
129+
"type": "MLModelTool"
130+
}
131+
},
132+
{
133+
"id": "root_agent",
134+
"type": "register_agent",
135+
"previous_node_inputs": {
136+
"agent_tool": "tools",
137+
"register_model_2": "model_id",
138+
"ml_model_tool": "tools"
139+
},
140+
"user_inputs": {
141+
"parameters": {
142+
"prompt": "Answer the question as best you can."
143+
},
144+
"app_type": "chatbot",
145+
"name": "Cypress test agent",
146+
"description": "this is the root agent",
147+
"tools_order": ["agent_tool", "ml_model_tool"],
148+
"memory": {
149+
"type": "conversation_index"
150+
},
151+
"type": "flow"
152+
}
153+
}
154+
]
155+
}
156+
}
157+
}

‎cypress/fixtures/plugins/dashboards-assistant/suggestion-response.json

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"completion": {
3+
"response": ["suggestion1", "suggestion2"]
4+
}
5+
}

‎cypress/integration/plugins/dashboards-assistant/chatbot_agent_framework_spec.js

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
import { BASE_PATH } from '../../../utils/constants';
6+
7+
if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) {
8+
describe('Assistant basic spec', () => {
9+
before(() => {
10+
// Set welcome screen tracking to false
11+
localStorage.setItem('home:welcome:show', 'false');
12+
// Set new theme modal to false
13+
localStorage.setItem('home:newThemeModal:show', 'false');
14+
});
15+
16+
beforeEach(() => {
17+
// Visit ISM OSD
18+
cy.visit(`${BASE_PATH}/app/home`);
19+
20+
// Common text to wait for to confirm page loaded, give up to 60 seconds for initial load
21+
cy.get(`input[placeholder="Ask question"]`, { timeout: 60000 }).should(
22+
'be.length',
23+
1
24+
);
25+
});
26+
27+
describe('Interact with Agent framework', () => {
28+
it('toggle Chatbot and enable to interact', () => {
29+
// enable to toggle and show Chatbot
30+
cy.get(`img[aria-label="toggle chat flyout icon"]`).click();
31+
32+
// click suggestions to generate response
33+
cy.contains('What are the indices in my cluster?').click();
34+
35+
// should have a LLM Response
36+
cy.contains(
37+
'The indices in your cluster are the names listed in the response obtained from using a tool to get information about the OpenSearch indices.'
38+
);
39+
40+
// should have a suggestion section
41+
cy.get(`[aria-label="chat suggestions"]`).should('be.length', 1);
42+
});
43+
});
44+
});
45+
}

‎cypress/support/assistant-dummy-llm.js

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
const http = require('http');
6+
const agentFrameworkJson = require('../fixtures/plugins/dashboards-assistant/agent-framework-response.json');
7+
const suggestionJson = require('../fixtures/plugins/dashboards-assistant/suggestion-response.json');
8+
9+
const MATCH_AGENT_FRAMEWORK_PROMPT =
10+
'Assistant is designed to be able to assist with a wide range of tasks';
11+
const MATCH_SUGGESTION_PROMPT = 'You are an AI that only speaks JSON';
12+
13+
const server = http.createServer((req, res) => {
14+
// Set the content type to JSON
15+
res.setHeader('Content-Type', 'application/json');
16+
17+
let requestBody = '';
18+
19+
// Listen for data events to capture the request body
20+
req.on('data', (chunk) => {
21+
requestBody += chunk;
22+
});
23+
24+
// Listen for the end of the request
25+
req.on('end', () => {
26+
try {
27+
// Why add a delay here? reference: https://github.com/opensearch-project/ml-commons/issues/1894
28+
setTimeout(() => {
29+
if (requestBody.includes(MATCH_AGENT_FRAMEWORK_PROMPT)) {
30+
return res.end(JSON.stringify(agentFrameworkJson));
31+
} else if (requestBody.includes(MATCH_SUGGESTION_PROMPT)) {
32+
return res.end(JSON.stringify(suggestionJson));
33+
}
34+
35+
res.end('');
36+
}, 100);
37+
} catch (error) {
38+
// Handle JSON parsing errors
39+
res.statusCode = 400;
40+
res.end(JSON.stringify({ error: 'Invalid JSON in the request body' }));
41+
}
42+
});
43+
});
44+
45+
// Listen on port 3000
46+
const PORT = 3000;
47+
server.listen(PORT, () => {
48+
console.log(`Server is listening on port ${PORT}`);
49+
});

‎cypress/support/index.js

+17
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import '../utils/plugins/alerting-dashboards-plugin/commands';
3030
import '../utils/plugins/ml-commons-dashboards/commands';
3131
import '../utils/plugins/security-analytics-dashboards-plugin/commands';
3232
import '../utils/plugins/notifications-dashboards/commands';
33+
import '../utils/plugins/dashboards-assistant/commands';
3334

3435
import 'cypress-real-events';
3536

@@ -56,3 +57,19 @@ if (Cypress.env('ENDPOINT_WITH_PROXY')) {
5657
Cypress.Cookies.preserveOnce('security_authentication');
5758
});
5859
}
60+
61+
/**
62+
* Make setup step in here so that all the test files in dashboards-assistant
63+
* won't need to call these commands.
64+
*/
65+
if (Cypress.env('DASHBOARDS_ASSISTANT_ENABLED')) {
66+
before(() => {
67+
cy.addAssistantRequiredSettings();
68+
cy.registerRootAgent();
69+
cy.startDummyServer();
70+
});
71+
after(() => {
72+
cy.cleanRootAgent();
73+
cy.stopDummyServer();
74+
});
75+
}

0 commit comments

Comments
 (0)