Skip to content

Commit 324e57f

Browse files
Support for Google Application Default Credentials (#8394)
* fixed conflicts Signed-off-by: fahadshamiinsta <fshami@netapp.com> * applying spotless Java check Signed-off-by: fahadshamiinsta <fshami@netapp.com> * added a comment to test helper method Signed-off-by: fahadshamiinsta <fshami@netapp.com> * added a comment to GoogleApplicationDefaultCredentials class with document reference Signed-off-by: fahadshamiinsta <fshami@netapp.com> * to rerun gradle checks Signed-off-by: fahadshamiinsta <fshami@netapp.com> * increasing coverage by adding another test Signed-off-by: fahadshamiinsta <fshami@netapp.com> * test name change Signed-off-by: fahadshamiinsta <fshami@netapp.com> * rerun ci Signed-off-by: fahadshamiinsta <fshami@netapp.com> * rerun ci Signed-off-by: fahadshamiinsta <fshami@netapp.com> * force push to rerun ci Signed-off-by: fahadshamiinsta <fshami@netapp.com> * pushing to trigger ci checks Signed-off-by: fahadshamiinsta <fshami@netapp.com> --------- Signed-off-by: fahadshamiinsta <fshami@netapp.com> (cherry picked from commit 4c283a7)
1 parent 90748fa commit 324e57f

File tree

3 files changed

+156
-29
lines changed

3 files changed

+156
-29
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.repositories.gcs;
10+
11+
import com.google.auth.oauth2.GoogleCredentials;
12+
import org.apache.logging.log4j.LogManager;
13+
import org.apache.logging.log4j.Logger;
14+
15+
import java.io.IOException;
16+
17+
/**
18+
* This class facilitates to fetch Application Default Credentials
19+
* see <a href="https://cloud.google.com/docs/authentication/application-default-credentials">How Application Default Credentials works</a>
20+
*/
21+
public class GoogleApplicationDefaultCredentials {
22+
private static final Logger logger = LogManager.getLogger(GoogleApplicationDefaultCredentials.class);
23+
24+
public GoogleCredentials get() {
25+
GoogleCredentials credentials = null;
26+
try {
27+
credentials = SocketAccess.doPrivilegedIOException(GoogleCredentials::getApplicationDefault);
28+
} catch (IOException e) {
29+
logger.error("Failed to retrieve \"Application Default Credentials\"", e);
30+
}
31+
return credentials;
32+
}
33+
}

plugins/repository-gcs/src/main/java/org/opensearch/repositories/gcs/GoogleCloudStorageService.java

+16-4
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.google.api.client.http.HttpRequestInitializer;
3737
import com.google.api.client.http.HttpTransport;
3838
import com.google.api.client.http.javanet.NetHttpTransport;
39+
import com.google.auth.oauth2.GoogleCredentials;
3940
import com.google.auth.oauth2.ServiceAccountCredentials;
4041
import com.google.cloud.ServiceOptions;
4142
import com.google.cloud.http.HttpTransportOptions;
@@ -68,6 +69,16 @@ public class GoogleCloudStorageService {
6869
*/
6970
private volatile Map<String, Storage> clientCache = emptyMap();
7071

72+
final private GoogleApplicationDefaultCredentials googleApplicationDefaultCredentials;
73+
74+
public GoogleCloudStorageService() {
75+
this.googleApplicationDefaultCredentials = new GoogleApplicationDefaultCredentials();
76+
}
77+
78+
public GoogleCloudStorageService(GoogleApplicationDefaultCredentials googleApplicationDefaultCredentials) {
79+
this.googleApplicationDefaultCredentials = googleApplicationDefaultCredentials;
80+
}
81+
7182
/**
7283
* Refreshes the client settings and clears the client cache. Subsequent calls to
7384
* {@code GoogleCloudStorageService#client} will return new clients constructed
@@ -195,10 +206,11 @@ StorageOptions createStorageOptions(
195206
storageOptionsBuilder.setProjectId(clientSettings.getProjectId());
196207
}
197208
if (clientSettings.getCredential() == null) {
198-
logger.warn(
199-
"\"Application Default Credentials\" are not supported out of the box."
200-
+ " Additional file system permissions have to be granted to the plugin."
201-
);
209+
logger.info("\"Application Default Credentials\" will be in use");
210+
final GoogleCredentials credentials = googleApplicationDefaultCredentials.get();
211+
if (credentials != null) {
212+
storageOptionsBuilder.setCredentials(credentials);
213+
}
202214
} else {
203215
ServiceAccountCredentials serviceAccountCredentials = clientSettings.getCredential();
204216
// override token server URI

plugins/repository-gcs/src/test/java/org/opensearch/repositories/gcs/GoogleCloudStorageServiceTests.java

+107-25
Original file line numberDiff line numberDiff line change
@@ -33,40 +33,50 @@
3333
package org.opensearch.repositories.gcs;
3434

3535
import com.google.auth.Credentials;
36+
import com.google.auth.oauth2.GoogleCredentials;
3637
import com.google.cloud.http.HttpTransportOptions;
3738
import com.google.cloud.storage.Storage;
3839

3940
import org.opensearch.common.bytes.BytesReference;
41+
import com.google.cloud.storage.StorageOptions;
4042
import org.opensearch.common.settings.MockSecureSettings;
4143
import org.opensearch.common.settings.Setting;
4244
import org.opensearch.common.settings.Settings;
4345
import org.opensearch.common.unit.TimeValue;
4446
import org.opensearch.common.xcontent.XContentBuilder;
4547
import org.opensearch.test.OpenSearchTestCase;
48+
import org.hamcrest.MatcherAssert;
4649
import org.hamcrest.Matchers;
4750

51+
import java.io.IOException;
52+
import java.net.Proxy;
53+
import java.net.URI;
54+
import java.net.URISyntaxException;
4855
import java.security.KeyPair;
4956
import java.security.KeyPairGenerator;
5057
import java.util.Base64;
5158
import java.util.Locale;
5259
import java.util.UUID;
5360

61+
import org.mockito.Mockito;
62+
5463
import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
5564
import static org.hamcrest.Matchers.equalTo;
5665
import static org.hamcrest.Matchers.containsString;
5766

5867
public class GoogleCloudStorageServiceTests extends OpenSearchTestCase {
5968

69+
final TimeValue connectTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
70+
final TimeValue readTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
71+
final String applicationName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
72+
final String endpoint = randomFrom("http://", "https://")
73+
+ randomFrom("www.opensearch.org", "www.googleapis.com", "localhost/api", "google.com/oauth")
74+
+ ":"
75+
+ randomIntBetween(1, 65535);
76+
final String projectIdName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
77+
6078
public void testClientInitializer() throws Exception {
6179
final String clientName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
62-
final TimeValue connectTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
63-
final TimeValue readTimeValue = TimeValue.timeValueNanos(randomIntBetween(0, 2000000));
64-
final String applicationName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
65-
final String endpoint = randomFrom("http://", "https://")
66-
+ randomFrom("www.opensearch.org", "www.googleapis.com", "localhost/api", "google.com/oauth")
67-
+ ":"
68-
+ randomIntBetween(1, 65535);
69-
final String projectIdName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
7080
final Settings settings = Settings.builder()
7181
.put(
7282
GoogleCloudStorageClientSettings.CONNECT_TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(),
@@ -83,31 +93,35 @@ public void testClientInitializer() throws Exception {
8393
.put(GoogleCloudStorageClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint)
8494
.put(GoogleCloudStorageClientSettings.PROJECT_ID_SETTING.getConcreteSettingForNamespace(clientName).getKey(), projectIdName)
8595
.build();
86-
final GoogleCloudStorageService service = new GoogleCloudStorageService();
96+
GoogleCredentials mockGoogleCredentials = Mockito.mock(GoogleCredentials.class);
97+
GoogleApplicationDefaultCredentials mockDefaultCredentials = Mockito.mock(GoogleApplicationDefaultCredentials.class);
98+
Mockito.when(mockDefaultCredentials.get()).thenReturn(mockGoogleCredentials);
99+
100+
final GoogleCloudStorageService service = new GoogleCloudStorageService(mockDefaultCredentials);
87101
service.refreshAndClearCache(GoogleCloudStorageClientSettings.load(settings));
88102
GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket");
89103
final IllegalArgumentException e = expectThrows(
90104
IllegalArgumentException.class,
91105
() -> service.client("another_client", "repo", statsCollector)
92106
);
93-
assertThat(e.getMessage(), Matchers.startsWith("Unknown client name"));
107+
MatcherAssert.assertThat(e.getMessage(), Matchers.startsWith("Unknown client name"));
94108
assertSettingDeprecationsAndWarnings(
95109
new Setting<?>[] { GoogleCloudStorageClientSettings.APPLICATION_NAME_SETTING.getConcreteSettingForNamespace(clientName) }
96110
);
97111
final Storage storage = service.client(clientName, "repo", statsCollector);
98-
assertThat(storage.getOptions().getApplicationName(), Matchers.containsString(applicationName));
99-
assertThat(storage.getOptions().getHost(), Matchers.is(endpoint));
100-
assertThat(storage.getOptions().getProjectId(), Matchers.is(projectIdName));
101-
assertThat(storage.getOptions().getTransportOptions(), Matchers.instanceOf(HttpTransportOptions.class));
102-
assertThat(
112+
MatcherAssert.assertThat(storage.getOptions().getApplicationName(), Matchers.containsString(applicationName));
113+
MatcherAssert.assertThat(storage.getOptions().getHost(), Matchers.is(endpoint));
114+
MatcherAssert.assertThat(storage.getOptions().getProjectId(), Matchers.is(projectIdName));
115+
MatcherAssert.assertThat(storage.getOptions().getTransportOptions(), Matchers.instanceOf(HttpTransportOptions.class));
116+
MatcherAssert.assertThat(
103117
((HttpTransportOptions) storage.getOptions().getTransportOptions()).getConnectTimeout(),
104118
Matchers.is((int) connectTimeValue.millis())
105119
);
106-
assertThat(
120+
MatcherAssert.assertThat(
107121
((HttpTransportOptions) storage.getOptions().getTransportOptions()).getReadTimeout(),
108122
Matchers.is((int) readTimeValue.millis())
109123
);
110-
assertThat(storage.getOptions().getCredentials(), Matchers.nullValue(Credentials.class));
124+
MatcherAssert.assertThat(storage.getOptions().getCredentials(), Matchers.instanceOf(Credentials.class));
111125
}
112126

113127
public void testReinitClientSettings() throws Exception {
@@ -123,33 +137,33 @@ public void testReinitClientSettings() throws Exception {
123137
final GoogleCloudStorageService storageService = plugin.storageService;
124138
GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket");
125139
final Storage client11 = storageService.client("gcs1", "repo1", statsCollector);
126-
assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
140+
MatcherAssert.assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
127141
final Storage client12 = storageService.client("gcs2", "repo2", statsCollector);
128-
assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
142+
MatcherAssert.assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
129143
// client 3 is missing
130144
final IllegalArgumentException e1 = expectThrows(
131145
IllegalArgumentException.class,
132146
() -> storageService.client("gcs3", "repo3", statsCollector)
133147
);
134-
assertThat(e1.getMessage(), containsString("Unknown client name [gcs3]."));
148+
MatcherAssert.assertThat(e1.getMessage(), containsString("Unknown client name [gcs3]."));
135149
// update client settings
136150
plugin.reload(settings2);
137151
// old client 1 not changed
138-
assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
152+
MatcherAssert.assertThat(client11.getOptions().getProjectId(), equalTo("project_gcs11"));
139153
// new client 1 is changed
140154
final Storage client21 = storageService.client("gcs1", "repo1", statsCollector);
141-
assertThat(client21.getOptions().getProjectId(), equalTo("project_gcs21"));
155+
MatcherAssert.assertThat(client21.getOptions().getProjectId(), equalTo("project_gcs21"));
142156
// old client 2 not changed
143-
assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
157+
MatcherAssert.assertThat(client12.getOptions().getProjectId(), equalTo("project_gcs12"));
144158
// new client2 is gone
145159
final IllegalArgumentException e2 = expectThrows(
146160
IllegalArgumentException.class,
147161
() -> storageService.client("gcs2", "repo2", statsCollector)
148162
);
149-
assertThat(e2.getMessage(), containsString("Unknown client name [gcs2]."));
163+
MatcherAssert.assertThat(e2.getMessage(), containsString("Unknown client name [gcs2]."));
150164
// client 3 emerged
151165
final Storage client23 = storageService.client("gcs3", "repo3", statsCollector);
152-
assertThat(client23.getOptions().getProjectId(), equalTo("project_gcs23"));
166+
MatcherAssert.assertThat(client23.getOptions().getProjectId(), equalTo("project_gcs23"));
153167
}
154168
}
155169

@@ -194,4 +208,72 @@ public void testToTimeout() {
194208
assertEquals(-1, GoogleCloudStorageService.toTimeout(TimeValue.ZERO).intValue());
195209
assertEquals(0, GoogleCloudStorageService.toTimeout(TimeValue.MINUS_ONE).intValue());
196210
}
211+
212+
/**
213+
* The following method test the Google Application Default Credential instead of
214+
* using service account file.
215+
* Considered use of JUnit Mocking due to static method GoogleCredentials.getApplicationDefault
216+
* and avoiding environment variables to set which later use GCE.
217+
* @throws Exception
218+
*/
219+
public void testApplicationDefaultCredential() throws Exception {
220+
GoogleCloudStorageClientSettings settings = getGCSClientSettingsWithoutCredentials();
221+
GoogleCredentials mockGoogleCredentials = Mockito.mock(GoogleCredentials.class);
222+
HttpTransportOptions mockHttpTransportOptions = Mockito.mock(HttpTransportOptions.class);
223+
GoogleApplicationDefaultCredentials mockDefaultCredentials = Mockito.mock(GoogleApplicationDefaultCredentials.class);
224+
Mockito.when(mockDefaultCredentials.get()).thenReturn(mockGoogleCredentials);
225+
226+
GoogleCloudStorageService service = new GoogleCloudStorageService(mockDefaultCredentials);
227+
StorageOptions storageOptions = service.createStorageOptions(settings, mockHttpTransportOptions);
228+
assertNotNull(storageOptions);
229+
assertEquals(storageOptions.getCredentials().toString(), mockGoogleCredentials.toString());
230+
}
231+
232+
/**
233+
* The application default credential throws exception when there are
234+
* no Environment Variables provided or Google Compute Engine is not running
235+
* @throws Exception
236+
*/
237+
public void testApplicationDefaultCredentialsWhenNoSettingProvided() throws Exception {
238+
GoogleCloudStorageClientSettings settings = getGCSClientSettingsWithoutCredentials();
239+
HttpTransportOptions mockHttpTransportOptions = Mockito.mock(HttpTransportOptions.class);
240+
GoogleCloudStorageService service = new GoogleCloudStorageService();
241+
StorageOptions storageOptions = service.createStorageOptions(settings, mockHttpTransportOptions);
242+
243+
Exception exception = assertThrows(IOException.class, GoogleCredentials::getApplicationDefault);
244+
assertNotNull(storageOptions);
245+
assertNull(storageOptions.getCredentials());
246+
MatcherAssert.assertThat(exception.getMessage(), containsString("The Application Default Credentials are not available"));
247+
}
248+
249+
/**
250+
* The application default credential throws IOException when it is
251+
* used without GoogleCloudStorageService
252+
*/
253+
public void testDefaultCredentialsThrowsExceptionWithoutGCStorageService() {
254+
GoogleApplicationDefaultCredentials googleApplicationDefaultCredentials = new GoogleApplicationDefaultCredentials();
255+
GoogleCredentials credentials = googleApplicationDefaultCredentials.get();
256+
assertNull(credentials);
257+
Exception exception = assertThrows(IOException.class, GoogleCredentials::getApplicationDefault);
258+
MatcherAssert.assertThat(exception.getMessage(), containsString("The Application Default Credentials are not available"));
259+
}
260+
261+
/**
262+
* This is a helper method to provide GCS Client settings without credentials
263+
* @return GoogleCloudStorageClientSettings
264+
* @throws URISyntaxException
265+
*/
266+
private GoogleCloudStorageClientSettings getGCSClientSettingsWithoutCredentials() throws URISyntaxException {
267+
return new GoogleCloudStorageClientSettings(
268+
null,
269+
endpoint,
270+
projectIdName,
271+
connectTimeValue,
272+
readTimeValue,
273+
applicationName,
274+
new URI(""),
275+
new ProxySettings(Proxy.Type.DIRECT, null, 0, null, null)
276+
);
277+
}
278+
197279
}

0 commit comments

Comments
 (0)