Skip to content

Commit 296a5cb

Browse files
ashking94ashking94
committed
Save work
Signed-off-by: Ashish Singh <ssashish@amazon.com>
1 parent 9bef705 commit 296a5cb

File tree

15 files changed

+566
-29
lines changed

15 files changed

+566
-29
lines changed

‎plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RemoteStoreIT.java

+309
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,309 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.repositories.s3;
10+
11+
import org.opensearch.common.SuppressForbidden;
12+
import org.opensearch.common.settings.MockSecureSettings;
13+
import org.opensearch.common.settings.SecureSettings;
14+
import org.opensearch.common.settings.Settings;
15+
import org.opensearch.core.common.Strings;
16+
import org.opensearch.index.remote.RemoteStoreEnums;
17+
import org.opensearch.indices.RemoteStoreSettings;
18+
import org.opensearch.plugins.Plugin;
19+
import org.opensearch.remotestore.RemoteStoreBaseIT;
20+
import org.opensearch.repositories.blobstore.BlobStoreRepository;
21+
import org.junit.Before;
22+
23+
import java.io.IOException;
24+
import java.util.Collection;
25+
import java.util.Collections;
26+
import java.util.Locale;
27+
import java.util.concurrent.ExecutionException;
28+
29+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
30+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
31+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
32+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
33+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
34+
35+
public class S3RemoteStoreIT extends RemoteStoreBaseIT {
36+
37+
@Override
38+
@Before
39+
@SuppressForbidden(reason = "Need to set system property here for AWS SDK v2")
40+
public void setUp() throws Exception {
41+
super.setUp();
42+
}
43+
44+
@Override
45+
@SuppressForbidden(reason = "Need to reset system property here for AWS SDK v2")
46+
public void tearDown() throws Exception {
47+
SocketAccess.doPrivileged(() -> System.clearProperty("opensearch.path.conf"));
48+
super.tearDown();
49+
}
50+
51+
@Override
52+
protected Collection<Class<? extends Plugin>> nodePlugins() {
53+
return Collections.singletonList(S3RepositoryPlugin.class);
54+
}
55+
56+
@Override
57+
protected Settings nodeSettings(int nodeOrdinal) {
58+
return Settings.builder().put(super.nodeSettings(nodeOrdinal)).setSecureSettings(credentials()).build();
59+
}
60+
61+
private SecureSettings credentials() {
62+
assertFalse(Strings.isNullOrEmpty(System.getProperty("test.s3.account")));
63+
assertFalse(Strings.isNullOrEmpty(System.getProperty("test.s3.key")));
64+
assertFalse(Strings.isNullOrEmpty(System.getProperty("test.s3.bucket")));
65+
66+
MockSecureSettings secureSettings = new MockSecureSettings();
67+
secureSettings.setString("s3.client.default.access_key", System.getProperty("test.s3.account"));
68+
secureSettings.setString("s3.client.default.secret_key", System.getProperty("test.s3.key"));
69+
return secureSettings;
70+
}
71+
72+
@Override
73+
protected Settings remoteStoreRepoSettings() {
74+
75+
String segmentRepoName = REPOSITORY_NAME;
76+
String translogRepoName = REPOSITORY_2_NAME;
77+
String stateRepoName = REPOSITORY_3_NAME;
78+
String segmentRepoTypeAttributeKey = String.format(
79+
Locale.getDefault(),
80+
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
81+
segmentRepoName
82+
);
83+
String segmentRepoSettingsAttributeKeyPrefix = String.format(
84+
Locale.getDefault(),
85+
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
86+
segmentRepoName
87+
);
88+
String translogRepoTypeAttributeKey = String.format(
89+
Locale.getDefault(),
90+
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
91+
translogRepoName
92+
);
93+
String translogRepoSettingsAttributeKeyPrefix = String.format(
94+
Locale.getDefault(),
95+
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
96+
translogRepoName
97+
);
98+
String stateRepoTypeAttributeKey = String.format(
99+
Locale.getDefault(),
100+
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
101+
stateRepoName
102+
);
103+
String stateRepoSettingsAttributeKeyPrefix = String.format(
104+
Locale.getDefault(),
105+
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
106+
stateRepoName
107+
);
108+
109+
String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey();
110+
111+
String bucket = System.getProperty("test.s3.bucket");
112+
String region = System.getProperty("test.s3.region", "us-west-2");
113+
String basePath = System.getProperty("test.s3.base", "testpath");
114+
String segmentBasePath = basePath + "-segments";
115+
String translogBasePath = basePath + "-translog";
116+
String stateBasePath = basePath + "-state";
117+
118+
Settings.Builder settings = Settings.builder()
119+
.put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)
120+
.put(segmentRepoTypeAttributeKey, S3Repository.TYPE)
121+
.put(segmentRepoSettingsAttributeKeyPrefix + "bucket", bucket)
122+
.put(segmentRepoSettingsAttributeKeyPrefix + "region", region)
123+
.put(segmentRepoSettingsAttributeKeyPrefix + "base_path", segmentBasePath)
124+
.put(segmentRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable)
125+
.put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName)
126+
.put(translogRepoTypeAttributeKey, S3Repository.TYPE)
127+
.put(translogRepoSettingsAttributeKeyPrefix + "bucket", bucket)
128+
.put(translogRepoSettingsAttributeKeyPrefix + "region", region)
129+
.put(translogRepoSettingsAttributeKeyPrefix + "base_path", translogBasePath)
130+
.put(translogRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable)
131+
.put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, stateRepoName)
132+
.put(stateRepoTypeAttributeKey, S3Repository.TYPE)
133+
.put(stateRepoSettingsAttributeKeyPrefix + "bucket", bucket)
134+
.put(stateRepoSettingsAttributeKeyPrefix + "region", region)
135+
.put(stateRepoSettingsAttributeKeyPrefix + "base_path", stateBasePath)
136+
.put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, prefixModeVerificationEnable);
137+
138+
final String endpoint = System.getProperty("test.s3.endpoint");
139+
if (endpoint != null) {
140+
settings.put(segmentRepoSettingsAttributeKeyPrefix + "endpoint", endpoint);
141+
settings.put(translogRepoSettingsAttributeKeyPrefix + "endpoint", endpoint);
142+
settings.put(stateRepoSettingsAttributeKeyPrefix + "endpoint", endpoint);
143+
}
144+
145+
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(RemoteStoreEnums.PathType.values()));
146+
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean());
147+
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean());
148+
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "a" : "");
149+
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "b" : "");
150+
settings.put(BlobStoreRepository.SNAPSHOT_SHARD_PATH_PREFIX_SETTING.getKey(), snapshotShardPathFixedPrefix ? "c" : "");
151+
152+
return settings.build();
153+
}
154+
155+
@Override
156+
public void testRemoteStoreIndexCreationAndDeletionWithReferencedStore() throws InterruptedException, ExecutionException {
157+
super.testRemoteStoreIndexCreationAndDeletionWithReferencedStore();
158+
}
159+
160+
@Override
161+
public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogNoDataFlush() throws Exception {
162+
super.testPeerRecoveryWithRemoteStoreAndRemoteTranslogNoDataFlush();
163+
}
164+
165+
@Override
166+
public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogFlush() throws Exception {
167+
super.testPeerRecoveryWithRemoteStoreAndRemoteTranslogFlush();
168+
}
169+
170+
@Override
171+
public void testPeerRecoveryWithLowActivityTimeout() throws Exception {
172+
super.testPeerRecoveryWithLowActivityTimeout();
173+
}
174+
175+
@Override
176+
public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogNoDataRefresh() throws Exception {
177+
super.testPeerRecoveryWithRemoteStoreAndRemoteTranslogNoDataRefresh();
178+
}
179+
180+
@Override
181+
public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogRefresh() throws Exception {
182+
super.testPeerRecoveryWithRemoteStoreAndRemoteTranslogRefresh();
183+
}
184+
185+
@Override
186+
public void testRemoteTranslogCleanup() throws Exception {
187+
super.testRemoteTranslogCleanup();
188+
}
189+
190+
@Override
191+
public void testStaleCommitDeletionWithInvokeFlush() throws Exception {
192+
super.testStaleCommitDeletionWithInvokeFlush();
193+
}
194+
195+
@Override
196+
public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception {
197+
super.testStaleCommitDeletionWithoutInvokeFlush();
198+
}
199+
200+
@Override
201+
public void testStaleCommitDeletionWithMinSegmentFiles_3() throws Exception {
202+
super.testStaleCommitDeletionWithMinSegmentFiles_3();
203+
}
204+
205+
@Override
206+
public void testStaleCommitDeletionWithMinSegmentFiles_Disabled() throws Exception {
207+
super.testStaleCommitDeletionWithMinSegmentFiles_Disabled();
208+
}
209+
210+
@Override
211+
public void testDefaultBufferInterval() throws ExecutionException, InterruptedException {
212+
super.testDefaultBufferInterval();
213+
}
214+
215+
@Override
216+
public void testOverriddenBufferInterval() throws ExecutionException, InterruptedException {
217+
super.testOverriddenBufferInterval();
218+
}
219+
220+
@Override
221+
public void testOverriddenBufferIntervalValidation() {
222+
super.testOverriddenBufferIntervalValidation();
223+
}
224+
225+
@Override
226+
public void testClusterBufferIntervalValidation() {
227+
super.testClusterBufferIntervalValidation();
228+
}
229+
230+
@Override
231+
public void testRequestDurabilityWhenRestrictSettingExplicitFalse() throws ExecutionException, InterruptedException {
232+
super.testRequestDurabilityWhenRestrictSettingExplicitFalse();
233+
}
234+
235+
@Override
236+
public void testAsyncDurabilityWhenRestrictSettingExplicitFalse() throws ExecutionException, InterruptedException {
237+
super.testAsyncDurabilityWhenRestrictSettingExplicitFalse();
238+
}
239+
240+
@Override
241+
public void testRequestDurabilityWhenRestrictSettingImplicitFalse() throws ExecutionException, InterruptedException {
242+
super.testRequestDurabilityWhenRestrictSettingImplicitFalse();
243+
}
244+
245+
@Override
246+
public void testAsyncDurabilityWhenRestrictSettingImplicitFalse() throws ExecutionException, InterruptedException {
247+
super.testAsyncDurabilityWhenRestrictSettingImplicitFalse();
248+
}
249+
250+
@Override
251+
public void testAsyncDurabilityThrowsExceptionWhenRestrictSettingTrue() throws ExecutionException, InterruptedException {
252+
super.testAsyncDurabilityThrowsExceptionWhenRestrictSettingTrue();
253+
}
254+
255+
@Override
256+
public void testRestoreSnapshotToIndexWithSameNameDifferentUUID() throws Exception {
257+
super.testRestoreSnapshotToIndexWithSameNameDifferentUUID();
258+
}
259+
260+
@Override
261+
public void testNoSearchIdleForAnyReplicaCount() throws ExecutionException, InterruptedException {
262+
super.testNoSearchIdleForAnyReplicaCount();
263+
}
264+
265+
@Override
266+
public void testFallbackToNodeToNodeSegmentCopy() throws Exception {
267+
super.testFallbackToNodeToNodeSegmentCopy();
268+
}
269+
270+
@Override
271+
public void testNoMultipleWriterDuringPrimaryRelocation() throws ExecutionException, InterruptedException {
272+
super.testNoMultipleWriterDuringPrimaryRelocation();
273+
}
274+
275+
@Override
276+
public void testResumeUploadAfterFailedPrimaryRelocation() throws ExecutionException, InterruptedException, IOException {
277+
super.testResumeUploadAfterFailedPrimaryRelocation();
278+
}
279+
280+
@Override
281+
public void testLocalOnlyTranslogCleanupOnNodeRestart() throws Exception {
282+
super.testLocalOnlyTranslogCleanupOnNodeRestart();
283+
}
284+
285+
@Override
286+
public void testFlushOnTooManyRemoteTranslogFiles() throws Exception {
287+
super.testFlushOnTooManyRemoteTranslogFiles();
288+
}
289+
290+
@Override
291+
public void testAsyncTranslogDurabilityRestrictionsThroughIdxTemplates() throws Exception {
292+
super.testAsyncTranslogDurabilityRestrictionsThroughIdxTemplates();
293+
}
294+
295+
@Override
296+
public void testCloseIndexWithNoOpSyncAndFlushForSyncTranslog() throws InterruptedException {
297+
super.testCloseIndexWithNoOpSyncAndFlushForSyncTranslog();
298+
}
299+
300+
@Override
301+
public void testCloseIndexWithNoOpSyncAndFlushForAsyncTranslog() throws InterruptedException {
302+
super.testCloseIndexWithNoOpSyncAndFlushForAsyncTranslog();
303+
}
304+
305+
@Override
306+
public void testSuccessfulShallowV1SnapshotPostIndexClose() throws Exception {
307+
super.testSuccessfulShallowV1SnapshotPostIndexClose();
308+
}
309+
}

‎plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@
6363
/**
6464
* A container for settings used to create an S3 client.
6565
*/
66-
final class S3ClientSettings {
66+
public final class S3ClientSettings {
6767

6868
private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(S3ClientSettings.class);
6969

‎plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy

+4
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ grant {
6060
permission java.util.PropertyPermission "aws.sharedCredentialsFile", "read,write";
6161
permission java.util.PropertyPermission "aws.configFile", "read,write";
6262
permission java.util.PropertyPermission "opensearch.path.conf", "read,write";
63+
permission java.util.PropertyPermission "test.s3.account", "read,write";
64+
permission java.util.PropertyPermission "test.s3.key", "read,write";
65+
permission java.util.PropertyPermission "test.s3.bucket", "read,write";
66+
permission java.util.PropertyPermission "test.s3.endpoint", "read,write";
6367
permission java.io.FilePermission "config", "read";
6468

6569
permission java.lang.RuntimePermission "accessDeclaredMembers";

‎server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java ‎server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartBaseIT.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
import org.opensearch.common.unit.TimeValue;
1515
import org.opensearch.core.common.unit.ByteSizeUnit;
1616
import org.opensearch.plugins.Plugin;
17-
import org.opensearch.remotestore.RemoteStoreIT;
17+
import org.opensearch.remotestore.RemoteStoreBaseIT;
1818
import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin;
1919
import org.opensearch.repositories.RepositoriesService;
2020
import org.opensearch.repositories.blobstore.BlobStoreRepository;
@@ -34,7 +34,7 @@
3434
import static org.hamcrest.Matchers.equalTo;
3535
import static org.hamcrest.Matchers.greaterThan;
3636

37-
public class RemoteStoreMultipartIT extends RemoteStoreIT {
37+
public class RemoteStoreMultipartBaseIT extends RemoteStoreBaseIT {
3838

3939
Path repositoryLocation;
4040
boolean compress;

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java ‎test/framework/src/main/java/org/opensearch/remotestore/RemoteStoreBaseIT.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@
7878
import static org.hamcrest.Matchers.oneOf;
7979

8080
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
81-
public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase {
81+
public class RemoteStoreBaseIT extends RemoteStoreBaseIntegTestCase {
8282

8383
protected final String INDEX_NAME = "remote-store-test-idx-1";
8484

0 commit comments

Comments
 (0)