Skip to content

Commit 6a41b10

Browse files
peterniedPeter Alfonsi
peternied
authored and
Peter Alfonsi
committed
Views, simplify data access and manipulation by providing a virtual layer over one or more indices Signed-off-by: Peter Nied <petern@amazon.com> Signed-off-by: Peter Nied <peternied@hotmail.com>
1 parent beced17 commit 6a41b10

34 files changed

+2913
-6
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
1616
- Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
1717
- [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887))
1818
- [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028))
19+
- Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
1920
- Add Remote Store Migration Experimental flag and allow mixed mode clusters under same ([#11986](https://github.com/opensearch-project/OpenSearch/pull/11986))
2021

2122
### Dependencies

‎server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java

+154
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.action.admin.indices.view;
10+
11+
import org.opensearch.cluster.metadata.View;
12+
import org.opensearch.index.IndexNotFoundException;
13+
import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
14+
import org.opensearch.test.OpenSearchIntegTestCase.Scope;
15+
import org.hamcrest.MatcherAssert;
16+
17+
import java.util.List;
18+
19+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
20+
import static org.hamcrest.Matchers.contains;
21+
import static org.hamcrest.Matchers.containsInAnyOrder;
22+
import static org.hamcrest.Matchers.hasSize;
23+
import static org.hamcrest.Matchers.is;
24+
import static org.hamcrest.Matchers.not;
25+
26+
@ClusterScope(scope = Scope.TEST, numDataNodes = 2)
27+
public class ViewIT extends ViewTestBase {
28+
29+
public void testCreateView() throws Exception {
30+
final String viewName = randomAlphaOfLength(8);
31+
final String indexPattern = randomAlphaOfLength(8);
32+
33+
logger.info("Testing createView with valid parameters");
34+
final View view = createView(viewName, indexPattern).getView();
35+
MatcherAssert.assertThat(view.getName(), is(viewName));
36+
MatcherAssert.assertThat(view.getTargets().size(), is(1));
37+
MatcherAssert.assertThat(view.getTargets().first().getIndexPattern(), is(indexPattern));
38+
39+
logger.info("Testing createView with existing view name");
40+
final Exception ex = assertThrows(ViewAlreadyExistsException.class, () -> createView(viewName, randomAlphaOfLength(8)));
41+
MatcherAssert.assertThat(ex.getMessage(), is("View [" + viewName + "] already exists"));
42+
}
43+
44+
public void testCreateViewTargetsSet() throws Exception {
45+
final String viewName = randomAlphaOfLength(8);
46+
final String indexPattern = "a" + randomAlphaOfLength(8);
47+
final String indexPattern2 = "b" + randomAlphaOfLength(8);
48+
final List<String> targetPatterns = List.of(indexPattern2, indexPattern, indexPattern);
49+
50+
logger.info("Testing createView with targets that will be reordered and deduplicated");
51+
final View view = createView(viewName, targetPatterns).getView();
52+
MatcherAssert.assertThat(view.getName(), is(viewName));
53+
MatcherAssert.assertThat(view.getTargets().size(), is(2));
54+
MatcherAssert.assertThat(view.getTargets().first().getIndexPattern(), is(indexPattern));
55+
MatcherAssert.assertThat(view.getTargets().last().getIndexPattern(), is(indexPattern2));
56+
}
57+
58+
public void testGetView() throws Exception {
59+
final String viewName = randomAlphaOfLength(8);
60+
createView(viewName, randomAlphaOfLength(8));
61+
62+
final View view = getView(viewName).getView();
63+
MatcherAssert.assertThat(view.getName(), is(viewName));
64+
65+
logger.info("Testing getView with non-existent view");
66+
final String nonExistentView = "non-existent-" + randomAlphaOfLength(8);
67+
final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> getView(nonExistentView));
68+
MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist"));
69+
}
70+
71+
public void testDeleteView() throws Exception {
72+
final String viewName = randomAlphaOfLength(8);
73+
createView(viewName, randomAlphaOfLength(8));
74+
75+
logger.info("Testing deleteView with existing view");
76+
deleteView(viewName);
77+
final Exception whenDeletedEx = assertThrows(ViewNotFoundException.class, () -> getView(viewName));
78+
MatcherAssert.assertThat(whenDeletedEx.getMessage(), is("View [" + viewName + "] does not exist"));
79+
80+
logger.info("Testing deleteView with non-existent view");
81+
final String nonExistentView = "non-existent-" + randomAlphaOfLength(8);
82+
final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> deleteView(nonExistentView));
83+
MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist"));
84+
}
85+
86+
public void testUpdateView() throws Exception {
87+
final String viewName = randomAlphaOfLength(8);
88+
final String originalIndexPattern = randomAlphaOfLength(8);
89+
final View originalView = createView(viewName, originalIndexPattern).getView();
90+
91+
logger.info("Testing updateView with existing view");
92+
final String newDescription = randomAlphaOfLength(20);
93+
final String newIndexPattern = "newPattern-" + originalIndexPattern;
94+
final View updatedView = updateView(viewName, newDescription, newIndexPattern).getView();
95+
96+
MatcherAssert.assertThat(updatedView, not(is(originalView)));
97+
MatcherAssert.assertThat(updatedView.getDescription(), is(newDescription));
98+
MatcherAssert.assertThat(updatedView.getTargets(), hasSize(1));
99+
MatcherAssert.assertThat(updatedView.getTargets().first().getIndexPattern(), is(newIndexPattern));
100+
101+
logger.info("Testing updateView with non-existent view");
102+
final String nonExistentView = "non-existent-" + randomAlphaOfLength(8);
103+
final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> updateView(nonExistentView, null, "index-*"));
104+
MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist"));
105+
}
106+
107+
public void testListViewNames() throws Exception {
108+
logger.info("Testing listViewNames when no views have been created");
109+
MatcherAssert.assertThat(listViewNames(), is(List.of()));
110+
111+
final String view1 = "view1";
112+
final String view2 = "view2";
113+
createView(view1, "index-1-*");
114+
createView(view2, "index-2-*");
115+
116+
logger.info("Testing listViewNames");
117+
final List<String> views = listViewNames();
118+
MatcherAssert.assertThat(views, containsInAnyOrder(view1, view2));
119+
120+
logger.info("Testing listViewNames after deleting a view");
121+
deleteView(view1);
122+
final List<String> viewsAfterDeletion = listViewNames();
123+
MatcherAssert.assertThat(viewsAfterDeletion, not(contains(view1)));
124+
MatcherAssert.assertThat(viewsAfterDeletion, contains(view2));
125+
}
126+
127+
public void testSearchOperations() throws Exception {
128+
final String indexInView1 = "index-1";
129+
final String indexInView2 = "index-2";
130+
final String indexNotInView = "another-index-1";
131+
132+
final int indexInView1DocCount = createIndexWithDocs(indexInView1);
133+
final int indexInView2DocCount = createIndexWithDocs(indexInView2);
134+
createIndexWithDocs(indexNotInView);
135+
136+
logger.info("Testing view with no matches");
137+
createView("no-matches", "this-pattern-will-match-nothing");
138+
final Exception ex = assertThrows(IndexNotFoundException.class, () -> searchView("no-matches"));
139+
MatcherAssert.assertThat(ex.getMessage(), is("no such index [this-pattern-will-match-nothing]"));
140+
141+
logger.info("Testing view with exact index match");
142+
createView("only-index-1", "index-1");
143+
assertHitCount(searchView("only-index-1"), indexInView1DocCount);
144+
145+
logger.info("Testing view with wildcard matches");
146+
createView("both-indices", "index-*");
147+
assertHitCount(searchView("both-indices"), indexInView1DocCount + indexInView2DocCount);
148+
149+
logger.info("Testing searchView with non-existent view");
150+
final String nonExistentView = "non-existent-" + randomAlphaOfLength(8);
151+
final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> searchView(nonExistentView));
152+
MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist"));
153+
}
154+
}

‎server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.action.admin.indices.view;
10+
11+
import org.opensearch.action.search.SearchRequest;
12+
import org.opensearch.action.search.SearchResponse;
13+
import org.opensearch.test.BackgroundIndexer;
14+
import org.opensearch.test.OpenSearchIntegTestCase;
15+
16+
import java.util.List;
17+
import java.util.stream.Collectors;
18+
19+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
20+
21+
public abstract class ViewTestBase extends OpenSearchIntegTestCase {
22+
23+
protected int createIndexWithDocs(final String indexName) throws Exception {
24+
createIndex(indexName);
25+
ensureGreen(indexName);
26+
27+
final int numOfDocs = scaledRandomIntBetween(0, 200);
28+
try (final BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) {
29+
waitForDocs(numOfDocs, indexer);
30+
}
31+
32+
refresh(indexName);
33+
assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs);
34+
return numOfDocs;
35+
}
36+
37+
protected GetViewAction.Response createView(final String name, final String indexPattern) throws Exception {
38+
return createView(name, List.of(indexPattern));
39+
}
40+
41+
protected GetViewAction.Response createView(final String name, final List<String> targets) throws Exception {
42+
final CreateViewAction.Request request = new CreateViewAction.Request(
43+
name,
44+
null,
45+
targets.stream().map(CreateViewAction.Request.Target::new).collect(Collectors.toList())
46+
);
47+
return client().admin().indices().createView(request).actionGet();
48+
}
49+
50+
protected GetViewAction.Response getView(final String name) {
51+
return client().admin().indices().getView(new GetViewAction.Request(name)).actionGet();
52+
53+
}
54+
55+
protected void deleteView(final String name) {
56+
client().admin().indices().deleteView(new DeleteViewAction.Request(name)).actionGet();
57+
performRemoteStoreTestAction();
58+
}
59+
60+
protected List<String> listViewNames() {
61+
return client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames();
62+
}
63+
64+
protected SearchResponse searchView(final String viewName) throws Exception {
65+
final SearchViewAction.Request request = new SearchViewAction.Request(viewName, new SearchRequest());
66+
final SearchResponse response = client().searchView(request).actionGet();
67+
return response;
68+
}
69+
70+
protected GetViewAction.Response updateView(final String name, final String description, final String indexPattern) {
71+
final CreateViewAction.Request request = new CreateViewAction.Request(
72+
name,
73+
description,
74+
List.of(new CreateViewAction.Request.Target(indexPattern))
75+
);
76+
final GetViewAction.Response response = client().admin().indices().updateView(request).actionGet();
77+
return response;
78+
}
79+
}

‎server/src/main/java/org/opensearch/OpenSearchServerException.java

+31-5
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,11 @@
88

99
package org.opensearch;
1010

11-
import org.opensearch.core.index.snapshots.IndexShardSnapshotException;
12-
import org.opensearch.crypto.CryptoRegistryException;
13-
1411
import static org.opensearch.OpenSearchException.OpenSearchExceptionHandle;
1512
import static org.opensearch.OpenSearchException.OpenSearchExceptionHandleRegistry.registerExceptionHandle;
1613
import static org.opensearch.OpenSearchException.UNKNOWN_VERSION_ADDED;
1714
import static org.opensearch.Version.V_2_10_0;
15+
import static org.opensearch.Version.V_2_13_0;
1816
import static org.opensearch.Version.V_2_1_0;
1917
import static org.opensearch.Version.V_2_4_0;
2018
import static org.opensearch.Version.V_2_5_0;
@@ -678,7 +676,12 @@ public static void registerExceptions() {
678676
)
679677
);
680678
registerExceptionHandle(
681-
new OpenSearchExceptionHandle(IndexShardSnapshotException.class, IndexShardSnapshotException::new, 98, UNKNOWN_VERSION_ADDED)
679+
new OpenSearchExceptionHandle(
680+
org.opensearch.core.index.snapshots.IndexShardSnapshotException.class,
681+
org.opensearch.core.index.snapshots.IndexShardSnapshotException::new,
682+
98,
683+
UNKNOWN_VERSION_ADDED
684+
)
682685
);
683686
registerExceptionHandle(
684687
new OpenSearchExceptionHandle(
@@ -1174,7 +1177,30 @@ public static void registerExceptions() {
11741177
V_2_7_0
11751178
)
11761179
);
1177-
registerExceptionHandle(new OpenSearchExceptionHandle(CryptoRegistryException.class, CryptoRegistryException::new, 171, V_2_10_0));
1180+
registerExceptionHandle(
1181+
new OpenSearchExceptionHandle(
1182+
org.opensearch.crypto.CryptoRegistryException.class,
1183+
org.opensearch.crypto.CryptoRegistryException::new,
1184+
171,
1185+
V_2_10_0
1186+
)
1187+
);
1188+
registerExceptionHandle(
1189+
new OpenSearchExceptionHandle(
1190+
org.opensearch.action.admin.indices.view.ViewNotFoundException.class,
1191+
org.opensearch.action.admin.indices.view.ViewNotFoundException::new,
1192+
172,
1193+
V_2_13_0
1194+
)
1195+
);
1196+
registerExceptionHandle(
1197+
new OpenSearchExceptionHandle(
1198+
org.opensearch.action.admin.indices.view.ViewAlreadyExistsException.class,
1199+
org.opensearch.action.admin.indices.view.ViewAlreadyExistsException::new,
1200+
173,
1201+
V_2_13_0
1202+
)
1203+
);
11781204
registerExceptionHandle(
11791205
new OpenSearchExceptionHandle(
11801206
org.opensearch.cluster.block.IndexCreateBlockException.class,

‎server/src/main/java/org/opensearch/action/ActionModule.java

+23
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,12 @@
224224
import org.opensearch.action.admin.indices.upgrade.post.UpgradeSettingsAction;
225225
import org.opensearch.action.admin.indices.validate.query.TransportValidateQueryAction;
226226
import org.opensearch.action.admin.indices.validate.query.ValidateQueryAction;
227+
import org.opensearch.action.admin.indices.view.CreateViewAction;
228+
import org.opensearch.action.admin.indices.view.DeleteViewAction;
229+
import org.opensearch.action.admin.indices.view.GetViewAction;
230+
import org.opensearch.action.admin.indices.view.ListViewNamesAction;
231+
import org.opensearch.action.admin.indices.view.SearchViewAction;
232+
import org.opensearch.action.admin.indices.view.UpdateViewAction;
227233
import org.opensearch.action.bulk.BulkAction;
228234
import org.opensearch.action.bulk.TransportBulkAction;
229235
import org.opensearch.action.bulk.TransportShardBulkAction;
@@ -409,6 +415,7 @@
409415
import org.opensearch.rest.action.admin.indices.RestUpgradeAction;
410416
import org.opensearch.rest.action.admin.indices.RestUpgradeStatusAction;
411417
import org.opensearch.rest.action.admin.indices.RestValidateQueryAction;
418+
import org.opensearch.rest.action.admin.indices.RestViewAction;
412419
import org.opensearch.rest.action.cat.AbstractCatAction;
413420
import org.opensearch.rest.action.cat.RestAliasAction;
414421
import org.opensearch.rest.action.cat.RestAllocationAction;
@@ -721,6 +728,14 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
721728
actions.register(ResolveIndexAction.INSTANCE, ResolveIndexAction.TransportAction.class);
722729
actions.register(DataStreamsStatsAction.INSTANCE, DataStreamsStatsAction.TransportAction.class);
723730

731+
// Views:
732+
actions.register(CreateViewAction.INSTANCE, CreateViewAction.TransportAction.class);
733+
actions.register(DeleteViewAction.INSTANCE, DeleteViewAction.TransportAction.class);
734+
actions.register(GetViewAction.INSTANCE, GetViewAction.TransportAction.class);
735+
actions.register(UpdateViewAction.INSTANCE, UpdateViewAction.TransportAction.class);
736+
actions.register(ListViewNamesAction.INSTANCE, ListViewNamesAction.TransportAction.class);
737+
actions.register(SearchViewAction.INSTANCE, SearchViewAction.TransportAction.class);
738+
724739
// Persistent tasks:
725740
actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class);
726741
actions.register(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class);
@@ -915,6 +930,14 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
915930
registerHandler.accept(new RestResolveIndexAction());
916931
registerHandler.accept(new RestDataStreamsStatsAction());
917932

933+
// View API
934+
registerHandler.accept(new RestViewAction.CreateViewHandler());
935+
registerHandler.accept(new RestViewAction.DeleteViewHandler());
936+
registerHandler.accept(new RestViewAction.GetViewHandler());
937+
registerHandler.accept(new RestViewAction.UpdateViewHandler());
938+
registerHandler.accept(new RestViewAction.SearchViewHandler());
939+
registerHandler.accept(new RestViewAction.ListViewNamesHandler());
940+
918941
// CAT API
919942
registerHandler.accept(new RestAllocationAction());
920943
registerHandler.accept(new RestCatSegmentReplicationAction());

0 commit comments

Comments
 (0)