From f7983fcc75d370e7da43f91fd5d66c8f3ac3a8a4 Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Thu, 14 Mar 2024 20:44:04 -0400 Subject: [PATCH 1/3] Add spotlessJavaCheck. Format existing code. - add spotlessJavaCheck. - format existing code. - upgrade deprecated gradle constructs. Signed-off-by: Mulugeta Mammo --- README.md | 3 +- build.gradle | 27 +- .../bwc/CustomCodecsBwcCompatibilityIT.java | 121 +++--- .../org/opensearch/customcodecs/bwc/Song.java | 30 +- .../customcodecs/bwc/helper/RestHelper.java | 25 +- gradle/formatterConfig.xml | 362 ++++++++++++++++++ gradle/formatting.gradle | 78 ++++ settings.gradle | 1 - .../codec/rest/CreateIndexWithCodecIT.java | 49 +-- .../Lucene99CustomStoredFieldsFormat.java | 14 +- .../backward_codecs/Lucene95CustomCodec.java | 2 +- .../Lucene95CustomStoredFieldsFormat.java | 30 +- .../backward_codecs/Zstd95Codec.java | 1 - ...Lucene95CustomStoredFieldsFormatTests.java | 20 +- ...Lucene99CustomStoredFieldsFormatTests.java | 8 +- 15 files changed, 606 insertions(+), 165 deletions(-) create mode 100644 gradle/formatterConfig.xml create mode 100644 gradle/formatting.gradle diff --git a/README.md b/README.md index f586e92..39b68fa 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # Custom Codecs Custom Codecs plugin makes it possible for users to provide custom Lucene codecs for loading through Apache Lucene's `NamedSPILoader`. -These codecs can be used to customize the on-disk representation of the opensearch indexes. For example, zstd compression can be used for +These codecs can be used to customize the on-disk representation of the opensearch indexes. For example, zstd compression can be used for `StoredField` types through the `ZstdCodec`. ## Security @@ -13,4 +13,3 @@ See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more inform ## License This project is licensed under the Apache-2.0 License. - diff --git a/build.gradle b/build.gradle index 00de61c..ec5a827 100644 --- a/build.gradle +++ b/build.gradle @@ -9,6 +9,25 @@ * GitHub history for details. */ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + buildscript { ext { opensearch_group = "org.opensearch" @@ -32,12 +51,14 @@ buildscript { plugins { id 'java' + id 'com.diffplug.spotless' version '6.25.0' } apply plugin: 'opensearch.opensearchplugin' apply plugin: 'opensearch.internal-cluster-test' apply plugin: 'opensearch.pluginzip' apply plugin: 'opensearch.rest-test' +apply from: 'gradle/formatting.gradle' repositories { mavenLocal() @@ -56,7 +77,7 @@ allprojects { version += "-SNAPSHOT" } - plugins.withId('java') { + 'java' { targetCompatibility = JavaVersion.VERSION_11 sourceCompatibility = JavaVersion.VERSION_11 } @@ -210,3 +231,7 @@ task updateVersion { ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) } } + +configurations.all { + resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-core:2.17.0' +} diff --git a/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/CustomCodecsBwcCompatibilityIT.java b/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/CustomCodecsBwcCompatibilityIT.java index e18f31d..b564655 100644 --- a/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/CustomCodecsBwcCompatibilityIT.java +++ b/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/CustomCodecsBwcCompatibilityIT.java @@ -7,16 +7,6 @@ */ package org.opensearch.customcodecs.bwc; -import java.io.IOException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import javax.net.ssl.SSLEngine; - import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; @@ -29,10 +19,6 @@ import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.hc.core5.reactor.ssl.TlsDetails; import org.apache.hc.core5.ssl.SSLContextBuilder; - -import org.junit.AfterClass; -import org.junit.Assume; -import org.junit.Before; import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; @@ -45,6 +31,20 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.customcodecs.bwc.helper.RestHelper; import org.opensearch.test.rest.OpenSearchRestTestCase; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.Before; + +import javax.net.ssl.SSLEngine; + +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; @@ -62,14 +62,10 @@ public void testSetup() throws IOException { logger.info("Running Test for Cluster Type: {}", CLUSTER_TYPE); CLUSTER_NAME = System.getProperty("tests.clustername"); if (testUserRestClient == null) { - testUserRestClient = buildClient( - super.restClientSettings(), - super.getClusterHosts().toArray(new HttpHost[0]) - ); + testUserRestClient = buildClient(super.restClientSettings(), super.getClusterHosts().toArray(new HttpHost[0])); } } - @Override protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); @@ -97,30 +93,26 @@ protected void configureHttpOrHttpsClient(RestClientBuilder builder, Settings se credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(username, password.toCharArray())); try { - final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder - .create() - .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) - // See https://issues.apache.org/jira/browse/HTTPCLIENT-2219 - .setTlsDetailsFactory(new Factory() { - @Override - public TlsDetails create(final SSLEngine sslEngine) { - return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); - } - }) - .build(); + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) + // See https://issues.apache.org/jira/browse/HTTPCLIENT-2219 + .setTlsDetailsFactory(new Factory() { + @Override + public TlsDetails create(final SSLEngine sslEngine) { + return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); + } + }) + .build(); builder.setHttpClientConfigCallback(httpClientBuilder -> { - final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder - .create() - .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE) - .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL) - .setTlsStrategy(tlsStrategy) - .build(); - - return httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider) - .setConnectionManager(connectionManager); + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE) + .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL) + .setTlsStrategy(tlsStrategy) + .build(); + + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager); }); } catch (final NoSuchAlgorithmException | KeyManagementException | KeyStoreException ex) { throw new IOException(ex); @@ -149,19 +141,17 @@ protected final boolean preserveReposUponCompletion() { return true; } - @Override protected final Settings restClientSettings() { return Settings.builder() - .put(super.restClientSettings()) - // increase the timeout here to 90 seconds to handle long waits for a green - // cluster health. the waits for green need to be longer than a minute to - // account for delayed shards - .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") - .build(); + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") + .build(); } - /** * Tests backward compatibility by created a test user and role with DLS, FLS and masked field settings. Ingests * data into a test index and runs a matchAll query against the same. @@ -176,7 +166,6 @@ public void testDataIngestionAndSearchBackwardsCompatibility() throws Exception searchMatchAll(index); } - /** * Ingests data into the test index * @@ -195,7 +184,7 @@ private void ingestData(String index) throws IOException { put("_index", index); } }); - + try (final XContentBuilder contentBuilder = MediaTypeRegistry.JSON.contentBuilder()) { contentBuilder.map(indexRequest); bulkRequestBody.append(contentBuilder.toString() + "\n"); @@ -204,10 +193,10 @@ private void ingestData(String index) throws IOException { bulkRequestBody.append(Song.randomSong().asJson() + "\n"); } List responses = RestHelper.requestAgainstAllNodes( - testUserRestClient, - "POST", - "_bulk?refresh=wait_for", - RestHelper.toHttpEntity(bulkRequestBody.toString()) + testUserRestClient, + "POST", + "_bulk?refresh=wait_for", + RestHelper.toHttpEntity(bulkRequestBody.toString()) ); responses.forEach(r -> assertEquals(200, r.getStatusLine().getStatusCode())); } @@ -223,10 +212,10 @@ private void searchMatchAll(String index) throws IOException { int numberOfRequests = Randomness.get().nextInt(10); while (numberOfRequests-- > 0) { List responses = RestHelper.requestAgainstAllNodes( - testUserRestClient, - "POST", - index + "/_search", - RestHelper.toHttpEntity(matchAllQuery) + testUserRestClient, + "POST", + index + "/_search", + RestHelper.toHttpEntity(matchAllQuery) ); responses.forEach(r -> assertEquals(200, r.getStatusLine().getStatusCode())); } @@ -247,13 +236,13 @@ private void createIndexIfNotExists(String index) throws IOException { // creating index createIndex( - index, - Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom("zstd", "zstd_no_dict")) - .put("index.codec.compression_level", randomIntBetween(1, 6)) - .build() + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.codec", randomFrom("zstd", "zstd_no_dict")) + .put("index.codec.compression_level", randomIntBetween(1, 6)) + .build() ); ensureGreen(index); } diff --git a/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/Song.java b/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/Song.java index 768f815..dee386b 100644 --- a/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/Song.java +++ b/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/Song.java @@ -9,15 +9,15 @@ */ package org.opensearch.customcodecs.bwc; +import org.opensearch.common.Randomness; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.XContentBuilder; + import java.io.IOException; import java.util.Map; import java.util.Objects; import java.util.UUID; -import org.opensearch.common.Randomness; -import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.core.xcontent.XContentBuilder; - public class Song { public static final String FIELD_TITLE = "title"; @@ -57,12 +57,12 @@ public class Song { public static final String QUERY_TITLE_MAGNUM_OPUS = FIELD_TITLE + ":" + TITLE_MAGNUM_OPUS; public static final Song[] SONGS = { - new Song(ARTIST_FIRST, TITLE_MAGNUM_OPUS, LYRICS_1, 1, GENRE_ROCK), - new Song(ARTIST_STRING, TITLE_SONG_1_PLUS_1, LYRICS_2, 2, GENRE_BLUES), - new Song(ARTIST_TWINS, TITLE_NEXT_SONG, LYRICS_3, 3, GENRE_JAZZ), - new Song(ARTIST_NO, TITLE_POISON, LYRICS_4, 4, GENRE_ROCK), - new Song(ARTIST_YES, TITLE_AFFIRMATIVE, LYRICS_5, 5, GENRE_BLUES), - new Song(ARTIST_UNKNOWN, TITLE_CONFIDENTIAL, LYRICS_6, 6, GENRE_JAZZ)}; + new Song(ARTIST_FIRST, TITLE_MAGNUM_OPUS, LYRICS_1, 1, GENRE_ROCK), + new Song(ARTIST_STRING, TITLE_SONG_1_PLUS_1, LYRICS_2, 2, GENRE_BLUES), + new Song(ARTIST_TWINS, TITLE_NEXT_SONG, LYRICS_3, 3, GENRE_JAZZ), + new Song(ARTIST_NO, TITLE_POISON, LYRICS_4, 4, GENRE_ROCK), + new Song(ARTIST_YES, TITLE_AFFIRMATIVE, LYRICS_5, 5, GENRE_BLUES), + new Song(ARTIST_UNKNOWN, TITLE_CONFIDENTIAL, LYRICS_6, 6, GENRE_JAZZ) }; private final String artist; private final String title; @@ -111,11 +111,11 @@ public String asJson() throws IOException { public static Song randomSong() { return new Song( - UUID.randomUUID().toString(), - UUID.randomUUID().toString(), - UUID.randomUUID().toString(), - Randomness.get().nextInt(5), - UUID.randomUUID().toString() + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + UUID.randomUUID().toString(), + Randomness.get().nextInt(5), + UUID.randomUUID().toString() ); } } diff --git a/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/helper/RestHelper.java b/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/helper/RestHelper.java index 0e9c384..195f32d 100644 --- a/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/helper/RestHelper.java +++ b/bwc-test/src/test/java/org/opensearch/customcodecs/bwc/helper/RestHelper.java @@ -8,23 +8,22 @@ package org.opensearch.customcodecs.bwc.helper; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.io.entity.StringEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.WarningsHandler; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + import static org.apache.hc.core5.http.ContentType.APPLICATION_JSON; public class RestHelper { @@ -44,7 +43,7 @@ public static Response makeRequest(RestClient client, String method, String endp } public static Response makeRequest(RestClient client, String method, String endpoint, HttpEntity entity, List
headers) - throws IOException { + throws IOException { log.info("Making request " + method + " " + endpoint + ", with headers " + headers); Request request = new Request(method, endpoint); @@ -66,16 +65,16 @@ public static Response makeRequest(RestClient client, String method, String endp } public static List requestAgainstAllNodes(RestClient client, String method, String endpoint, HttpEntity entity) - throws IOException { + throws IOException { return requestAgainstAllNodes(client, method, endpoint, entity, null); } public static List requestAgainstAllNodes( - RestClient client, - String method, - String endpoint, - HttpEntity entity, - List
headers + RestClient client, + String method, + String endpoint, + HttpEntity entity, + List
headers ) throws IOException { int nodeCount = client.getNodes().size(); List responses = new ArrayList<>(); @@ -88,4 +87,4 @@ public static List requestAgainstAllNodes( public static Header getAuthorizationHeader(String username, String password) { return new BasicHeader("Authorization", "Basic " + username + ":" + password); } -} \ No newline at end of file +} diff --git a/gradle/formatterConfig.xml b/gradle/formatterConfig.xml new file mode 100644 index 0000000..713e552 --- /dev/null +++ b/gradle/formatterConfig.xml @@ -0,0 +1,362 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle new file mode 100644 index 0000000..0cd9485 --- /dev/null +++ b/gradle/formatting.gradle @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +spotless { + java { + target '**/*.java' + + removeUnusedImports() + importOrder( + 'de.thetaphi', + 'com.carrotsearch', + 'com.fasterxml', + 'com.avast', + 'com.sun', + 'com.maxmind|com.github|com.networknt|groovy|nebula', + 'org.antlr', + 'software.amazon', + 'com.azure|com.microsoft|com.ibm|com.google|joptsimple|org.apache|org.bouncycastle|org.codehaus|org.opensearch|org.objectweb|org.joda|org.hamcrest|org.openjdk|org.gradle|org.junit', + 'javax', + 'java', + '', + '\\#java|\\#org.opensearch|\\#org.hamcrest|\\#' + ) + + eclipse().configFile rootProject.file('gradle/formatterConfig.xml') + trimTrailingWhitespace() + endWithNewline() + + custom 'Refuse wildcard imports', { + // Wildcard imports can't be resolved; fail the build + if (it =~ /\s+import .*\*;/) { + throw new AssertionError("Do not use wildcard imports. 'spotlessApply' cannot resolve this issue.") + } + } + + // See DEVELOPER_GUIDE.md for details of when to enable this. + if (System.getProperty('spotless.paddedcell') != null) { + paddedCell() + } + } + format 'misc', { + target '*.md', '*.gradle', '**/*.json', '**/*.yaml', '**/*.yml', '**/*.svg' + + targetExclude '**/simple-bulk11.json', '**/simple-msearch5.json' + + trimTrailingWhitespace() + endWithNewline() + } +} + +precommit.dependsOn 'spotlessJavaCheck' diff --git a/settings.gradle b/settings.gradle index bc41600..6b599a5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,4 +8,3 @@ rootProject.name = "opensearch-custom-codecs" startParameter.excludedTaskNames=["publishPluginZipPublicationToMavenLocal", "publishPluginZipPublicationToStagingRepository"] - diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java index ba9ca1f..c332094 100644 --- a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java +++ b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java @@ -16,33 +16,24 @@ import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.apache.hc.core5.function.Factory; -import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHost; -import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.hc.core5.reactor.ssl.TlsDetails; import org.apache.hc.core5.ssl.SSLContextBuilder; -import org.apache.hc.core5.util.Timeout; -import org.junit.After; -import org.junit.BeforeClass; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.Strings; import org.opensearch.test.rest.OpenSearchRestTestCase; +import javax.net.ssl.SSLEngine; + import java.io.IOException; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; -import java.util.Map; import java.util.Objects; -import java.util.Optional; - -import javax.net.ssl.SSLEngine; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; @@ -81,7 +72,7 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE protected void configureHttpOrHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { configureClient(builder, settings); - + if (getProtocol().equalsIgnoreCase("https")) { final String username = System.getProperty("user"); if (Strings.isNullOrEmpty(username)) { @@ -98,30 +89,26 @@ protected void configureHttpOrHttpsClient(RestClientBuilder builder, Settings se credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(username, password.toCharArray())); try { - final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder - .create() - .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) - // See https://issues.apache.org/jira/browse/HTTPCLIENT-2219 - .setTlsDetailsFactory(new Factory() { - @Override - public TlsDetails create(final SSLEngine sslEngine) { - return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); - } - }) - .build(); + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) + // See https://issues.apache.org/jira/browse/HTTPCLIENT-2219 + .setTlsDetailsFactory(new Factory() { + @Override + public TlsDetails create(final SSLEngine sslEngine) { + return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); + } + }) + .build(); builder.setHttpClientConfigCallback(httpClientBuilder -> { - final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder - .create() + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE) .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL) .setTlsStrategy(tlsStrategy) .build(); - - return httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider) - .setConnectionManager(connectionManager); + + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager); }); } catch (final NoSuchAlgorithmException | KeyManagementException | KeyStoreException ex) { throw new IOException(ex); @@ -134,7 +121,7 @@ public TlsDetails create(final SSLEngine sslEngine) { protected String getProtocol() { return Objects.equals(System.getProperty("https"), "true") ? "https" : "http"; } - + /** * wipeAllIndices won't work since it cannot delete security index */ diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java index ff32c11..235c801 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java @@ -17,7 +17,6 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.opensearch.index.codec.customcodecs.backward_codecs.Lucene95CustomCodec; import java.io.IOException; import java.util.Objects; @@ -74,7 +73,7 @@ public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode, int compr */ @Override public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { - if (si.getAttribute(MODE_KEY) !=null){ + if (si.getAttribute(MODE_KEY) != null) { String value = si.getAttribute(MODE_KEY); Lucene99CustomCodec.Mode mode = Lucene99CustomCodec.Mode.valueOf(value); return impl(mode).fieldsReader(directory, si, fn, context); @@ -111,14 +110,13 @@ StoredFieldsFormat impl(Lucene99CustomCodec.Mode mode) { } } - private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { return new Lucene90CompressingStoredFieldsFormat( - formatName, - compressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT + formatName, + compressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT ); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomCodec.java index 2924590..8b45ae2 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomCodec.java @@ -8,9 +8,9 @@ package org.opensearch.index.codec.customcodecs.backward_codecs; +import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec; import java.util.Collections; import java.util.Set; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormat.java index be92032..6f51d52 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormat.java @@ -62,8 +62,10 @@ public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode) { public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; - zstdCompressionMode = new ZstdCompressionMode(compressionLevel){}; - zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel){}; + zstdCompressionMode = new ZstdCompressionMode(compressionLevel) { + }; + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel) { + }; } /** @@ -94,7 +96,7 @@ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOCo String previous = si.putAttribute(MODE_KEY, mode.name()); if (previous != null && previous.equals(mode.name()) == false) { throw new IllegalStateException( - "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() ); } return impl(mode).fieldsWriter(directory, si, context); @@ -105,19 +107,19 @@ StoredFieldsFormat impl(Lucene95CustomCodec.Mode mode) { case ZSTD: case ZSTD_DEPRECATED: return new Lucene90CompressingStoredFieldsFormat( - "CustomStoredFieldsZstd", - zstdCompressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT + "CustomStoredFieldsZstd", + zstdCompressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT ); case ZSTD_NO_DICT: return new Lucene90CompressingStoredFieldsFormat( - "CustomStoredFieldsZstdNoDict", - zstdNoDictCompressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT + "CustomStoredFieldsZstdNoDict", + zstdNoDictCompressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT ); default: throw new AssertionError(); @@ -139,4 +141,4 @@ public CompressionMode getCompressionMode() { return mode == Lucene95CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; } -} \ No newline at end of file +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Zstd95Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Zstd95Codec.java index e41f690..c2d347c 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Zstd95Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Zstd95Codec.java @@ -27,7 +27,6 @@ public Zstd95Codec() { super(Mode.ZSTD); } - /** The name for this codec. */ @Override public String toString() { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java index f03f164..8e32b17 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java @@ -21,7 +21,7 @@ public void testDefaultLucene95CustomCodecMode() { public void testZstdNoDictLucene95CustomCodecMode() { Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD_NO_DICT + Lucene95CustomCodec.Mode.ZSTD_NO_DICT ); assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); } @@ -29,8 +29,8 @@ public void testZstdNoDictLucene95CustomCodecMode() { public void testZstdModeWithCompressionLevel() { int randomCompressionLevel = randomIntBetween(1, 6); Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD, - randomCompressionLevel + Lucene95CustomCodec.Mode.ZSTD, + randomCompressionLevel ); assertEquals(Lucene95CustomCodec.Mode.ZSTD, lucene95CustomStoredFieldsFormat.getMode()); assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); @@ -39,21 +39,23 @@ public void testZstdModeWithCompressionLevel() { public void testZstdNoDictLucene95CustomCodecModeWithCompressionLevel() { int randomCompressionLevel = randomIntBetween(1, 6); Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD_NO_DICT, - randomCompressionLevel + Lucene95CustomCodec.Mode.ZSTD_NO_DICT, + randomCompressionLevel ); assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); } - public void testCompressionModes(){ + public void testCompressionModes() { Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat(); assertTrue(lucene95CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); } - public void testZstdNoDictCompressionModes(){ - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode.ZSTD_NO_DICT); + public void testZstdNoDictCompressionModes() { + Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( + Lucene95CustomCodec.Mode.ZSTD_NO_DICT + ); assertTrue(lucene95CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); } -} \ No newline at end of file +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java index c966901..d0ab2ff 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java @@ -44,13 +44,15 @@ public void testZstdNoDictLucene99CustomCodecModeWithCompressionLevel() { assertEquals(randomCompressionLevel, lucene99CustomStoredFieldsFormat.getCompressionLevel()); } - public void testCompressionModes(){ + public void testCompressionModes() { Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat(); assertTrue(lucene99CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); } - public void testZstdNoDictCompressionModes(){ - Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode.ZSTD_NO_DICT); + public void testZstdNoDictCompressionModes() { + Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat( + Lucene99CustomCodec.Mode.ZSTD_NO_DICT + ); assertTrue(lucene99CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); } From 7e851b5cd03cc5b56bfc2b99b607d0d64168c836 Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Fri, 15 Mar 2024 11:04:53 -0400 Subject: [PATCH 2/3] Apply spotless to files in the .github directory. Signed-off-by: Mulugeta Mammo --- .github/workflows/add-untriaged.yml | 38 +++++++++---------- .github/workflows/delete-backport-branch.yml | 4 +- .github/workflows/publish-maven-snapshots.yml | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml index 9dcc702..15b9a55 100644 --- a/.github/workflows/add-untriaged.yml +++ b/.github/workflows/add-untriaged.yml @@ -1,19 +1,19 @@ -name: Apply 'untriaged' label during issue lifecycle - -on: - issues: - types: [opened, reopened, transferred] - -jobs: - apply-label: - runs-on: ubuntu-latest - steps: - - uses: actions/github-script@v6 - with: - script: | - github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: ['untriaged'] - }) +name: Apply 'untriaged' label during issue lifecycle + +on: + issues: + types: [opened, reopened, transferred] + +jobs: + apply-label: + runs-on: ubuntu-latest + steps: + - uses: actions/github-script@v6 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['untriaged'] + }) diff --git a/.github/workflows/delete-backport-branch.yml b/.github/workflows/delete-backport-branch.yml index 387a124..f24f022 100644 --- a/.github/workflows/delete-backport-branch.yml +++ b/.github/workflows/delete-backport-branch.yml @@ -1,9 +1,9 @@ name: Delete merged branch of the backport PRs -on: +on: pull_request: types: - closed - + jobs: delete-branch: runs-on: ubuntu-latest diff --git a/.github/workflows/publish-maven-snapshots.yml b/.github/workflows/publish-maven-snapshots.yml index 86e6668..b1374b0 100644 --- a/.github/workflows/publish-maven-snapshots.yml +++ b/.github/workflows/publish-maven-snapshots.yml @@ -32,4 +32,4 @@ jobs: export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) echo "::add-mask::$SONATYPE_USERNAME" echo "::add-mask::$SONATYPE_PASSWORD" - ./gradlew publishPluginZipPublicationToSnapshotsRepository \ No newline at end of file + ./gradlew publishPluginZipPublicationToSnapshotsRepository From 536ac886f2a35824c21b8b5766463558776a6f39 Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Fri, 15 Mar 2024 14:09:33 -0400 Subject: [PATCH 3/3] Fix three minor issues: 1) Remove the Elasticsearch attribution in spotless formatter. 2) Use java instead of 'java'. 3) Remove version resolution for jackson. Signed-off-by: Mulugeta Mammo --- build.gradle | 6 +----- gradle/formatting.gradle | 19 ------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/build.gradle b/build.gradle index ec5a827..56d4261 100644 --- a/build.gradle +++ b/build.gradle @@ -77,7 +77,7 @@ allprojects { version += "-SNAPSHOT" } - 'java' { + java { targetCompatibility = JavaVersion.VERSION_11 sourceCompatibility = JavaVersion.VERSION_11 } @@ -231,7 +231,3 @@ task updateVersion { ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true) } } - -configurations.all { - resolutionStrategy.force 'com.fasterxml.jackson.core:jackson-core:2.17.0' -} diff --git a/gradle/formatting.gradle b/gradle/formatting.gradle index 0cd9485..37b4b07 100644 --- a/gradle/formatting.gradle +++ b/gradle/formatting.gradle @@ -9,25 +9,6 @@ * GitHub history for details. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - spotless { java { target '**/*.java'