Skip to content

Commit 63dacdf

Browse files
stephen-crawfordstephen-crawford
committed
Merge branch 'deyTest14' into finishJoinEngine
2 parents 8426e14 + a17c9d1 commit 63dacdf

File tree

47 files changed

+5626
-9
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+5626
-9
lines changed

‎plugins/events-correlation-engine/src/internalClusterTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginTransportIT.java

+751
Large diffs are not rendered by default.

‎plugins/events-correlation-engine/src/javaRestTest/java/org/opensearch/plugin/correlation/EventsCorrelationPluginRestIT.java

+173
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import java.io.IOException;
2121
import java.util.List;
22+
import java.util.Locale;
2223
import java.util.Map;
2324

2425
/**
@@ -102,6 +103,67 @@ public void testCreatingACorrelationRuleWithNoTimestampField() throws IOExceptio
102103
);
103104
}
104105

106+
@SuppressWarnings("unchecked")
107+
public void testCorrelationWithSingleRule() throws IOException {
108+
String windowsIndex = "windows";
109+
Request request = new Request("PUT", "/" + windowsIndex);
110+
request.setJsonEntity(windowsMappings());
111+
client().performRequest(request);
112+
113+
String appLogsIndex = "app_logs";
114+
request = new Request("PUT", "/" + appLogsIndex);
115+
request.setJsonEntity(appLogMappings());
116+
client().performRequest(request);
117+
118+
String correlationRule = windowsToAppLogsCorrelationRule();
119+
request = new Request("POST", "/_correlation/rules");
120+
request.setJsonEntity(correlationRule);
121+
client().performRequest(request);
122+
123+
request = new Request("POST", String.format(Locale.ROOT, "/%s/_doc?refresh", windowsIndex));
124+
request.setJsonEntity(sampleWindowsEvent());
125+
client().performRequest(request);
126+
127+
request = new Request("POST", String.format(Locale.ROOT, "/%s/_doc?refresh", appLogsIndex));
128+
request.setJsonEntity(sampleAppLogsEvent());
129+
Response response = client().performRequest(request);
130+
String appLogsId = responseAsMap(response).get("_id").toString();
131+
132+
request = new Request("POST", "/_correlation/events");
133+
request.setJsonEntity(prepareCorrelateEventRequest(appLogsIndex, appLogsId));
134+
response = client().performRequest(request);
135+
Map<String, Object> responseAsMap = responseAsMap(response);
136+
Assert.assertEquals(1, ((Map<String, Object>) responseAsMap.get("neighbor_events")).size());
137+
}
138+
139+
private String prepareCorrelateEventRequest(String index, String event) {
140+
return "{\n" + " \"index\": \"" + index + "\",\n" + " \"event\": \"" + event + "\",\n" + " \"store\": false\n" + "}";
141+
}
142+
143+
private String windowsToAppLogsCorrelationRule() {
144+
return "{\n"
145+
+ " \"name\": \"windows to app logs\",\n"
146+
+ " \"correlate\": [\n"
147+
+ " {\n"
148+
+ " \"index\": \"windows\",\n"
149+
+ " \"query\": \"host.hostname:EC2AMAZ*\",\n"
150+
+ " \"timestampField\": \"winlog.timestamp\",\n"
151+
+ " \"tags\": [\n"
152+
+ " \"windows\"\n"
153+
+ " ]\n"
154+
+ " },\n"
155+
+ " {\n"
156+
+ " \"index\": \"app_logs\",\n"
157+
+ " \"query\": \"endpoint:\\\\/customer_records.txt\",\n"
158+
+ " \"timestampField\": \"timestamp\",\n"
159+
+ " \"tags\": [\n"
160+
+ " \"others_application\"\n"
161+
+ " ]\n"
162+
+ " }\n"
163+
+ " ]\n"
164+
+ "}";
165+
}
166+
105167
private String sampleCorrelationRule() {
106168
return "{\n"
107169
+ " \"name\": \"s3 to app logs\",\n"
@@ -151,4 +213,115 @@ private String sampleCorrelationRuleWithNoTimestamp() {
151213
private String matchIdQuery(String id) {
152214
return "{\n" + " \"query\" : {\n" + " \"match\":{\n" + " \"_id\": \"" + id + "\"\n" + " }\n" + " }\n" + "}";
153215
}
216+
217+
private String windowsMappings() {
218+
return "{"
219+
+ " \"settings\": {"
220+
+ " \"number_of_shards\": 1"
221+
+ " },"
222+
+ " \"mappings\": {"
223+
+ " \"properties\": {\n"
224+
+ " \"server.user.hash\": {\n"
225+
+ " \"type\": \"text\"\n"
226+
+ " },\n"
227+
+ " \"winlog.event_id\": {\n"
228+
+ " \"type\": \"integer\"\n"
229+
+ " },\n"
230+
+ " \"host.hostname\": {\n"
231+
+ " \"type\": \"text\"\n"
232+
+ " },\n"
233+
+ " \"windows.message\": {\n"
234+
+ " \"type\": \"text\"\n"
235+
+ " },\n"
236+
+ " \"winlog.provider_name\": {\n"
237+
+ " \"type\": \"text\"\n"
238+
+ " },\n"
239+
+ " \"winlog.event_data.ServiceName\": {\n"
240+
+ " \"type\": \"text\"\n"
241+
+ " },\n"
242+
+ " \"winlog.timestamp\": {\n"
243+
+ " \"type\": \"long\"\n"
244+
+ " }\n"
245+
+ " }\n"
246+
+ " }\n"
247+
+ "}";
248+
}
249+
250+
private String appLogMappings() {
251+
return "{"
252+
+ " \"settings\": {"
253+
+ " \"number_of_shards\": 1"
254+
+ " },"
255+
+ " \"mappings\": {"
256+
+ " \"properties\": {\n"
257+
+ " \"http_method\": {\n"
258+
+ " \"type\": \"text\"\n"
259+
+ " },\n"
260+
+ " \"endpoint\": {\n"
261+
+ " \"type\": \"text\",\n"
262+
+ " \"analyzer\": \"whitespace\""
263+
+ " },\n"
264+
+ " \"keywords\": {\n"
265+
+ " \"type\": \"text\"\n"
266+
+ " },\n"
267+
+ " \"timestamp\": {\n"
268+
+ " \"type\": \"long\"\n"
269+
+ " }\n"
270+
+ " }\n"
271+
+ " }\n"
272+
+ "}";
273+
}
274+
275+
private String sampleWindowsEvent() {
276+
return "{\n"
277+
+ " \"EventTime\": \"2020-02-04T14:59:39.343541+00:00\",\n"
278+
+ " \"host.hostname\": \"EC2AMAZEPO7HKA\",\n"
279+
+ " \"Keywords\": \"9223372036854775808\",\n"
280+
+ " \"SeverityValue\": 2,\n"
281+
+ " \"Severity\": \"INFO\",\n"
282+
+ " \"winlog.event_id\": 22,\n"
283+
+ " \"SourceName\": \"Microsoft-Windows-Sysmon\",\n"
284+
+ " \"ProviderGuid\": \"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n"
285+
+ " \"Version\": 5,\n"
286+
+ " \"TaskValue\": 22,\n"
287+
+ " \"OpcodeValue\": 0,\n"
288+
+ " \"RecordNumber\": 9532,\n"
289+
+ " \"ExecutionProcessID\": 1996,\n"
290+
+ " \"ExecutionThreadID\": 2616,\n"
291+
+ " \"Channel\": \"Microsoft-Windows-Sysmon/Operational\",\n"
292+
+ " \"winlog.event_data.SubjectDomainName\": \"NTAUTHORITY\",\n"
293+
+ " \"AccountName\": \"SYSTEM\",\n"
294+
+ " \"UserID\": \"S-1-5-18\",\n"
295+
+ " \"AccountType\": \"User\",\n"
296+
+ " \"windows.message\": \"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n"
297+
+ " \"Category\": \"Dns query (rule: DnsQuery)\",\n"
298+
+ " \"Opcode\": \"Info\",\n"
299+
+ " \"UtcTime\": \"2020-02-04 14:59:38.349\",\n"
300+
+ " \"ProcessGuid\": \"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n"
301+
+ " \"ProcessId\": \"1904\",\n"
302+
+ " \"QueryName\": \"EC2AMAZ-EPO7HKA\",\n"
303+
+ " \"QueryStatus\": \"0\",\n"
304+
+ " \"QueryResults\": \"172.31.46.38;\",\n"
305+
+ " \"Image\": \"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n"
306+
+ " \"EventReceivedTime\": \"2020-02-04T14:59:40.780905+00:00\",\n"
307+
+ " \"SourceModuleName\": \"in\",\n"
308+
+ " \"SourceModuleType\": \"im_msvistalog\",\n"
309+
+ " \"CommandLine\": \"eachtest\",\n"
310+
+ " \"Initiated\": \"true\",\n"
311+
+ " \"winlog.timestamp\": "
312+
+ System.currentTimeMillis()
313+
+ "\n"
314+
+ "}";
315+
}
316+
317+
private String sampleAppLogsEvent() {
318+
return "{\n"
319+
+ " \"endpoint\": \"/customer_records.txt\",\n"
320+
+ " \"http_method\": \"POST\",\n"
321+
+ " \"keywords\": \"PermissionDenied\",\n"
322+
+ " \"timestamp\": "
323+
+ System.currentTimeMillis()
324+
+ "\n"
325+
+ "}";
326+
}
154327
}

‎plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/EventsCorrelationPlugin.java

+26-4
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,19 @@
3030
import org.opensearch.plugin.correlation.core.index.mapper.CorrelationVectorFieldMapper;
3131
import org.opensearch.plugin.correlation.core.index.mapper.VectorFieldMapper;
3232
import org.opensearch.plugin.correlation.core.index.query.CorrelationQueryBuilder;
33+
import org.opensearch.plugin.correlation.events.action.IndexCorrelationAction;
34+
import org.opensearch.plugin.correlation.events.action.SearchCorrelatedEventsAction;
35+
import org.opensearch.plugin.correlation.events.action.StoreCorrelationAction;
36+
import org.opensearch.plugin.correlation.events.resthandler.RestIndexCorrelationAction;
37+
import org.opensearch.plugin.correlation.events.resthandler.RestSearchCorrelatedEventsAction;
38+
import org.opensearch.plugin.correlation.events.transport.TransportIndexCorrelationAction;
39+
import org.opensearch.plugin.correlation.events.transport.TransportSearchCorrelatedEventsAction;
40+
import org.opensearch.plugin.correlation.events.transport.TransportStoreCorrelationAction;
3341
import org.opensearch.plugin.correlation.rules.action.IndexCorrelationRuleAction;
3442
import org.opensearch.plugin.correlation.rules.resthandler.RestIndexCorrelationRuleAction;
3543
import org.opensearch.plugin.correlation.rules.transport.TransportIndexCorrelationRuleAction;
3644
import org.opensearch.plugin.correlation.settings.EventsCorrelationSettings;
45+
import org.opensearch.plugin.correlation.utils.CorrelationIndices;
3746
import org.opensearch.plugin.correlation.utils.CorrelationRuleIndices;
3847
import org.opensearch.plugins.ActionPlugin;
3948
import org.opensearch.plugins.EnginePlugin;
@@ -67,9 +76,12 @@ public class EventsCorrelationPlugin extends Plugin implements ActionPlugin, Map
6776
* events-correlation-engine rules uri
6877
*/
6978
public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/rules";
79+
public static final String CORRELATION_EVENTS_BASE_URI = PLUGINS_BASE_URI + "/events";
7080

7181
private CorrelationRuleIndices correlationRuleIndices;
7282

83+
private CorrelationIndices correlationIndices;
84+
7385
/**
7486
* Default constructor
7587
*/
@@ -90,7 +102,8 @@ public Collection<Object> createComponents(
90102
Supplier<RepositoriesService> repositoriesServiceSupplier
91103
) {
92104
correlationRuleIndices = new CorrelationRuleIndices(client, clusterService);
93-
return List.of(correlationRuleIndices);
105+
correlationIndices = new CorrelationIndices(client, clusterService, clusterService.getSettings());
106+
return List.of(correlationRuleIndices, correlationIndices);
94107
}
95108

96109
@Override
@@ -103,7 +116,7 @@ public List<RestHandler> getRestHandlers(
103116
IndexNameExpressionResolver indexNameExpressionResolver,
104117
Supplier<DiscoveryNodes> nodesInCluster
105118
) {
106-
return List.of(new RestIndexCorrelationRuleAction());
119+
return List.of(new RestIndexCorrelationRuleAction(), new RestSearchCorrelatedEventsAction(), new RestIndexCorrelationAction());
107120
}
108121

109122
@Override
@@ -132,11 +145,20 @@ public List<QuerySpec<?>> getQueries() {
132145

133146
@Override
134147
public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
135-
return List.of(new ActionPlugin.ActionHandler<>(IndexCorrelationRuleAction.INSTANCE, TransportIndexCorrelationRuleAction.class));
148+
return List.of(
149+
new ActionPlugin.ActionHandler<>(IndexCorrelationRuleAction.INSTANCE, TransportIndexCorrelationRuleAction.class),
150+
new ActionPlugin.ActionHandler<>(IndexCorrelationAction.INSTANCE, TransportIndexCorrelationAction.class),
151+
new ActionPlugin.ActionHandler<>(StoreCorrelationAction.INSTANCE, TransportStoreCorrelationAction.class),
152+
new ActionPlugin.ActionHandler<>(SearchCorrelatedEventsAction.INSTANCE, TransportSearchCorrelatedEventsAction.class)
153+
);
136154
}
137155

138156
@Override
139157
public List<Setting<?>> getSettings() {
140-
return List.of(EventsCorrelationSettings.IS_CORRELATION_INDEX_SETTING, EventsCorrelationSettings.CORRELATION_TIME_WINDOW);
158+
return List.of(
159+
EventsCorrelationSettings.IS_CORRELATION_INDEX_SETTING,
160+
EventsCorrelationSettings.CORRELATION_HISTORY_INDEX_SHARDS,
161+
EventsCorrelationSettings.CORRELATION_TIME_WINDOW
162+
);
141163
}
142164
}

‎plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/events/action/IndexCorrelationAction.java

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.plugin.correlation.events.action;
10+
11+
import org.opensearch.action.ActionType;
12+
13+
/**
14+
* Transport Action for indexing correlations
15+
*
16+
* @opensearch.internal
17+
*/
18+
public class IndexCorrelationAction extends ActionType<IndexCorrelationResponse> {
19+
20+
/**
21+
* Instance of IndexCorrelationAction
22+
*/
23+
public static final IndexCorrelationAction INSTANCE = new IndexCorrelationAction();
24+
/**
25+
* Name of IndexCorrelationAction
26+
*/
27+
public static final String NAME = "cluster:admin/index/correlation/events";
28+
29+
private IndexCorrelationAction() {
30+
super(NAME, IndexCorrelationResponse::new);
31+
}
32+
}

0 commit comments

Comments
 (0)