Skip to content

Commit 1fa4eb2

Browse files
shiv0408shiv0408
committed
Add rolling upgrade bwc with remote publication enabled
Signed-off-by: Shivansh Arora <hishiv@amazon.com>
1 parent 6f1b59e commit 1fa4eb2

File tree

2 files changed

+358
-12
lines changed

2 files changed

+358
-12
lines changed

‎qa/rolling-upgrade/build.gradle

+6
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,12 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
6262
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
6363
setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
6464
setting 'http.content_type.required', 'true'
65+
setting 'node.attr.remote_store.routing_table.repository', 'my-fs-repository'
66+
setting 'node.attr.remote_store.state.repository', 'my-fs-repository'
67+
setting 'node.attr.remote_store.repository.my-fs-repository.type', 'fs'
68+
setting 'node.attr.remote_store.repository.my-fs-repository.settings.location', "${buildDir}/cluster/shared/repo/${baseName}/remote_state"
69+
setting 'cluster.remote_store.state.enabled', 'true'
70+
setting 'cluster.remote_store.publication.enabled', 'true'
6571
}
6672
}
6773

‎qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStateIT.java

+352-12
Original file line numberDiff line numberDiff line change
@@ -8,30 +8,370 @@
88

99
package org.opensearch.upgrades;
1010

11+
import org.apache.logging.log4j.LogManager;
12+
import org.apache.logging.log4j.Logger;
1113
import org.opensearch.client.Request;
1214
import org.opensearch.client.Response;
15+
import org.opensearch.cluster.metadata.IndexTemplateMetadata;
16+
import org.opensearch.common.settings.Settings;
1317

18+
import java.util.Arrays;
19+
import java.util.List;
1420
import java.util.Map;
1521

22+
@SuppressWarnings("unchecked")
1623
public class ClusterStateIT extends AbstractRollingTestCase{
24+
private static final Logger logger = LogManager.getLogger(ClusterStateIT.class);
25+
private static final String templateName = "my_template";
26+
private static final String indexName = "test_cluster_state";
27+
private static final String componentTemplateName = "test_component_template";
28+
29+
// Add different types of metadata to Cluster State to ensure the new versions understand these metadata types in future versions
30+
public void testUpgradeWithRemotePublicationEnabled() throws Exception {
31+
if (CLUSTER_TYPE == ClusterType.OLD) {
32+
// verify that Remote Publication is Enabled on the cluster
33+
verifyRemotePublicationEnabled();
34+
// Create Index
35+
Settings indexSettings = Settings.builder()
36+
.put("index.number_of_shards", 1)
37+
.put("index.number_of_replicas", 1)
38+
.put("index.refresh_interval", "5s").build();
39+
String indexMappings = "\"properties\": {\"name\": {\"type\":\"text\"}}";
40+
String aliases = "\"main\":{\"is_write_index\":true}";
41+
createIndex(indexName, indexSettings, indexMappings, aliases);
42+
// Create Index Template
43+
createIndexTemplate();
44+
// Create Component Template
45+
createComponentTemplate();
46+
// Create Composable Template
47+
createComposableTemplate();
48+
// Add Settings
49+
addPersistentSettings();
50+
addTransientSettings();
51+
52+
Request request = new Request("GET", "_cluster/state");
53+
Response response = client().performRequest(request);
54+
assertOK(response);
55+
56+
verifyIndexInClusterState(response);
57+
verifyTemplateMetadataInClusterState(response);
58+
verifyComponentTemplateInClusterState(response);
59+
verifyComposableTemplateInClusterState(response);
60+
verifySettingsInClusterState();
61+
} else {
62+
verifyRemotePublicationEnabled();
63+
64+
Request request = new Request("GET", "_cluster/state");
65+
Response response = client().performRequest(request);
66+
assertOK(response);
67+
68+
verifyIndexInClusterState(response);
69+
verifyTemplateMetadataInClusterState(response);
70+
verifyComponentTemplateInClusterState(response);
71+
verifyComposableTemplateInClusterState(response);
72+
verifySettingsInClusterState();
73+
}
74+
}
75+
1776
public void testTemplateMetadataUpgrades() throws Exception {
1877
if (CLUSTER_TYPE == ClusterType.OLD) {
19-
String templateName = "my_template";
20-
Request putIndexTemplate = new Request("PUT", "_template/" + templateName);
21-
putIndexTemplate.setJsonEntity("{\"index_patterns\": [\"pattern-1\", \"log-*\"]}");
22-
client().performRequest(putIndexTemplate);
23-
verifyTemplateMetadataInClusterState();
78+
createIndexTemplate();
79+
Request request = new Request("GET", "_cluster/state/metadata");
80+
Response response = client().performRequest(request);
81+
assertOK(response);
82+
83+
verifyTemplateMetadataInClusterState(response);
2484
} else {
25-
verifyTemplateMetadataInClusterState();
85+
Request request = new Request("GET", "_cluster/state/metadata");
86+
Response response = client().performRequest(request);
87+
assertOK(response);
88+
89+
verifyTemplateMetadataInClusterState(response);
90+
}
91+
}
92+
93+
private static void createIndexTemplate() throws Exception {
94+
Request putIndexTemplate = new Request("PUT", "_template/" + templateName);
95+
String indexTemplateJson = "{\n" +
96+
" \"index_patterns\": [\"pattern-*\", \"log-*\"],\n" +
97+
" \"settings\": {\n" +
98+
" \"number_of_shards\": 3,\n" +
99+
" \"number_of_replicas\": 1,\n" +
100+
" \"index.refresh_interval\": \"5s\"\n" +
101+
" },\n" +
102+
" \"mappings\": {\n" +
103+
" \"properties\": {\n" +
104+
" \"timestamp\": {\n" +
105+
" \"type\": \"date\"\n" +
106+
" },\n" +
107+
" \"message\": {\n" +
108+
" \"type\": \"text\",\n" +
109+
" \"fields\": {\n" +
110+
" \"keyword\": {\n" +
111+
" \"type\": \"keyword\",\n" +
112+
" \"ignore_above\": 256\n" +
113+
" }\n" +
114+
" }\n" +
115+
" },\n" +
116+
" \"level\": {\n" +
117+
" \"type\": \"keyword\"\n" +
118+
" }\n" +
119+
" }\n" +
120+
" },\n" +
121+
" \"aliases\": {\n" +
122+
" \"mydata\": {}\n" +
123+
" },\n" +
124+
" \"version\": 1\n" +
125+
"}";
126+
127+
putIndexTemplate.setJsonEntity(indexTemplateJson);
128+
client().performRequest(putIndexTemplate);
129+
}
130+
131+
private static void createComponentTemplate() throws Exception {
132+
Request putComponentTemplate = new Request("PUT", "_component_template/" + componentTemplateName);
133+
String componentTemplateJson = "{\n" +
134+
" \"template\": {\n" +
135+
" \"mappings\": {\n" +
136+
" \"properties\": {\n" +
137+
" \"name\": {\n" +
138+
" \"type\": \"text\"\n" +
139+
" }\n" +
140+
" }\n" +
141+
" }\n" +
142+
" },\n" +
143+
" \"version\": 1\n" +
144+
"}";
145+
146+
putComponentTemplate.setJsonEntity(componentTemplateJson);
147+
client().performRequest(putComponentTemplate);
148+
}
149+
150+
private static void createComposableTemplate() throws Exception {
151+
Request putComposableTemplate = new Request("PUT", "_index_template/composable_template");
152+
String composableTemplateJson = "{\n" +
153+
" \"index_patterns\": [\"te-*\", \"bar-*\"],\n" +
154+
" \"template\": {\n" +
155+
" \"mappings\": {\n" +
156+
" \"properties\": {\n" +
157+
" \"email\": {\n" +
158+
" \"type\": \"keyword\"\n" +
159+
" }\n" +
160+
" }\n" +
161+
" }\n" +
162+
" },\n" +
163+
" \"version\": 1,\n" +
164+
" \"composed_of\": [\n" +
165+
" \"" + componentTemplateName + "\"\n" +
166+
" ]\n" +
167+
"}";
168+
169+
putComposableTemplate.setJsonEntity(composableTemplateJson);
170+
client().performRequest(putComposableTemplate);
171+
}
172+
173+
private static void addPersistentSettings() throws Exception {
174+
Request putSettings = new Request("PUT", "_cluster/settings");
175+
String settingsJson = "{\n" +
176+
" \"persistent\": {\n" +
177+
" \"cluster\": {\n" +
178+
" \"remote\": {\n" +
179+
" \"cluster\": {\n" +
180+
" \"seeds\": [\n" +
181+
" \"127.0.0.1:9300\"\n" +
182+
" ]\n" +
183+
" }\n" +
184+
" }\n" +
185+
" }\n" +
186+
" }\n" +
187+
"}";
188+
189+
putSettings.setJsonEntity(settingsJson);
190+
client().performRequest(putSettings);
191+
}
192+
193+
private static void addTransientSettings() throws Exception {
194+
Request putSettings = new Request("PUT", "_cluster/settings");
195+
String settingsJson = "{\n" +
196+
" \"transient\": {\n" +
197+
" \"cluster\": {\n" +
198+
" \"remote\": {\n" +
199+
" \"cluster\": {\n" +
200+
" \"seeds\": [\n" +
201+
" \"127.0.0.1:9300\"\n" +
202+
" ]\n" +
203+
" }\n" +
204+
" }\n" +
205+
" }\n" +
206+
" }\n" +
207+
"}";
208+
209+
putSettings.setJsonEntity(settingsJson);
210+
client().performRequest(putSettings);
211+
}
212+
213+
private static void verifyIndexInClusterState(Response clusterStateResponse) throws Exception {
214+
Map<String, Object> responseMap = entityAsMap(clusterStateResponse);
215+
Map<String, Object> metadata = (Map<String, Object>) responseMap.get("metadata");
216+
assertNotNull("Metadata should exist in response", metadata);
217+
218+
Map<String, Object> indices = (Map<String, Object>) metadata.get("indices");
219+
assertNotNull("Indices should exist in metadata", indices);
220+
221+
Map<String, Object> index = (Map<String, Object>) indices.get(indexName);
222+
assertNotNull("Index " + indexName + " should exist in cluster state", index);
223+
224+
logger.info(index);
225+
// Verify index settings
226+
Map<String, Object> settings = (Map<String, Object>) index.get("settings");
227+
assertNotNull("Settings should exist in index", settings);
228+
229+
// Verify index mappings
230+
Map<String, Object> mappings = (Map<String, Object>) index.get("mappings");
231+
assertNotNull("Mappings should exist in index", mappings);
232+
233+
}
234+
235+
private static void verifyTemplateMetadataInClusterState(Response clusterStateResponse) throws Exception {
236+
Map<String, Object> responseMap = entityAsMap(clusterStateResponse);
237+
Map<String, Object> metadata = (Map<String, Object>) responseMap.get("metadata");
238+
assertNotNull("Metadata should exist in response", metadata);
239+
240+
Map<String, Object> templates = (Map<String, Object>) metadata.get("templates");
241+
assertNotNull("Templates should exist in metadata", templates);
242+
243+
Map<String, Object> templateMetadata = (Map<String, Object>) templates.get(templateName);
244+
assertNotNull("Template " + templateName + " should exist in cluster state", templateMetadata);
245+
246+
// Verify index patterns
247+
List<String> indexPatterns = (List<String>) templateMetadata.get("index_patterns");
248+
assertEquals("Index patterns should match", Arrays.asList("pattern-*", "log-*"), indexPatterns);
249+
250+
// Verify settings
251+
Map<String, Object> settings = (Map<String, Object>) getNestedValue(templateMetadata, "settings");
252+
253+
assertEquals("Refresh interval should be 5s", "5s", getNestedValue(settings, "index", "refresh_interval"));
254+
255+
// Verify mappings
256+
Map<String, Object> mappings = (Map<String, Object>) getNestedValue(templateMetadata, "mappings");
257+
assertNotNull("Mappings should exist", mappings);
258+
259+
Map<String, Object> properties = (Map<String, Object>) getNestedValue(mappings, "_doc", "properties");
260+
assertNotNull("Properties should exist in mappings", properties);
261+
262+
assertEquals("timestamp should be of type date", "date", getNestedValue(properties, "timestamp", "type"));
263+
264+
Map<String, Object> messageField = (Map<String, Object>) properties.get("message");
265+
assertEquals("message should be of type text", "text", messageField.get("type"));
266+
267+
Map<String, Object> messageFields = (Map<String, Object>) messageField.get("fields");
268+
assertNotNull("message should have subfields", messageFields);
269+
270+
Map<String, Object> keywordField = (Map<String, Object>) messageFields.get("keyword");
271+
assertEquals("message.keyword should be of type keyword", "keyword", keywordField.get("type"));
272+
assertEquals("message.keyword should ignore above 256", 256, ((Integer) keywordField.get("ignore_above")).intValue());
273+
274+
assertEquals("level should be of type keyword", "keyword", ((Map<String, Object>) properties.get("level")).get("type"));
275+
276+
// Verify aliases
277+
Map<String, Object> aliases = (Map<String, Object>) templateMetadata.get("aliases");
278+
assertTrue("Alias 'mydata' should exist", aliases.containsKey("mydata"));
279+
280+
// Verify version
281+
assertEquals("Template version should be 1", 1, templateMetadata.get("version"));
282+
}
283+
284+
private static void verifyComponentTemplateInClusterState(Response clusterStateResponse) throws Exception {
285+
Map<String, Object> responseMap = entityAsMap(clusterStateResponse);
286+
Map<String, Object> metadata = (Map<String, Object>) responseMap.get("metadata");
287+
assertNotNull("Metadata should exist in response", metadata);
288+
289+
Map<String, Object> componentTemplate = (Map<String, Object>) getNestedValue(metadata, "component_template", "component_template", componentTemplateName);
290+
assertNotNull("Component template should exist in metadata", componentTemplate);
291+
292+
Map<String, Object> templateMetadata = (Map<String, Object>) componentTemplate.get("template");
293+
assertNotNull("Template should exist in component template", templateMetadata);
294+
295+
// Verify component template mappings
296+
Map<String, Object> mappings = (Map<String, Object>) templateMetadata.get("mappings");
297+
assertNotNull("Mappings should exist in component template", mappings);
298+
299+
// Verify mappings present
300+
assertNotNull("Mappings should have email field", getNestedValue(mappings, "properties", "name"));
301+
}
302+
303+
private static void verifyComposableTemplateInClusterState(Response clusterStateResponse) throws Exception {
304+
Map<String, Object> responseMap = entityAsMap(clusterStateResponse);
305+
Map<String, Object> metadata = (Map<String, Object>) responseMap.get("metadata");
306+
assertNotNull("Metadata should exist in response", metadata);
307+
308+
Map<String, Object> indexTemplates = (Map<String, Object>) getNestedValue(metadata, "index_template", "index_template");
309+
assertNotNull("Index templates should exist in metadata", indexTemplates);
310+
311+
Map<String, Object> composableTemplate = (Map<String, Object>) indexTemplates.get("composable_template");
312+
assertNotNull("Composable template should exist in metadata", composableTemplate);
313+
314+
// Verify index patterns
315+
List<String> indexPatterns = (List<String>) composableTemplate.get("index_patterns");
316+
assertEquals("Index patterns should match", Arrays.asList("te-*", "bar-*"), indexPatterns);
317+
318+
// Verify composed_of
319+
List<String> composedOf = (List<String>) composableTemplate.get("composed_of");
320+
assertEquals("Composed templates should match", Arrays.asList(componentTemplateName), composedOf);
321+
}
322+
323+
private static void verifySettingsInClusterState() throws Exception {
324+
Request getSettingRequest = new Request("GET", "/_cluster/settings");
325+
Response response = client().performRequest(getSettingRequest);
326+
assertOK(response);
327+
Map<String, Object> responseMap = entityAsMap(response);
328+
329+
Map<String, Object> persistent = (Map<String, Object>) responseMap.get("persistent");
330+
assertNotNull("Persistent settings should exist in metadata", persistent);
331+
332+
Map<String, Object> remote = (Map<String, Object>) getNestedValue(persistent, "cluster", "remote");
333+
assertNotNull("Remote settings should exist in persistent settings", remote);
334+
335+
Map<String, Object> cluster = (Map<String, Object>) remote.get("cluster");
336+
assertNotNull("Cluster settings should exist in remote settings", cluster);
337+
338+
List<String> seeds = (List<String>) cluster.get("seeds");
339+
assertEquals("Seeds should match", Arrays.asList("127.0.0.1:9300"), seeds);
340+
341+
Map<String, Object> transientSettings = (Map<String, Object>) responseMap.get("transient");
342+
assertNotNull("Transient settings should exist in metadata", transientSettings);
343+
344+
remote = (Map<String, Object>) getNestedValue(transientSettings, "cluster", "remote");
345+
assertNotNull("Remote settings should exist in transient settings", remote);
346+
347+
cluster = (Map<String, Object>) remote.get("cluster");
348+
assertNotNull("Cluster settings should exist in remote settings", cluster);
349+
350+
seeds = (List<String>) cluster.get("seeds");
351+
assertEquals("Seeds should match", Arrays.asList("127.0.0.1:9300"), seeds);
352+
}
353+
354+
private static Object getNestedValue(Map<String, Object> map, String... keys) {
355+
Object current = map;
356+
for (String key : keys) {
357+
if (!(current instanceof Map)) {
358+
return null;
359+
}
360+
current = ((Map<String, Object>) current).get(key);
26361
}
362+
return current;
27363
}
28364

29-
@SuppressWarnings("unchecked")
30-
private static void verifyTemplateMetadataInClusterState() throws Exception {
31-
Request request = new Request("GET", "_cluster/state/metadata");
32-
Response response = client().performRequest(request);
365+
private void verifyRemotePublicationEnabled() throws Exception {
366+
Request getSettingRequest = new Request("GET", "/_cluster/settings");
367+
getSettingRequest.addParameter("include_defaults", "true");
368+
Response response = client().performRequest(getSettingRequest);
33369
assertOK(response);
34-
Map<String, Object> metadata = (Map<String, Object>) entityAsMap(response).get("metadata");
35-
assertNotNull(metadata.get("templates"));
370+
371+
Map<String, Object> responseMap = entityAsMap(response);
372+
Map<String, Object> defaultsSettings = (Map<String, Object>) responseMap.get("defaults");
373+
374+
Object enabled = getNestedValue(defaultsSettings, "cluster", "remote_store", "publication", "enabled");
375+
assertEquals("true", enabled);
36376
}
37377
}

0 commit comments

Comments
 (0)