From ba77db0b6a4bbdbcdf4d8832be168d3b09f23443 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Tue, 25 Feb 2025 22:33:46 +0530 Subject: [PATCH 01/16] Bumping custom-codecs plugin version to 3.0.0.0-alpha1 Signed-off-by: Mohit Kumar --- build.gradle | 4 ++-- bwc-test/build.gradle | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 00a6f89..54025f9 100644 --- a/build.gradle +++ b/build.gradle @@ -31,9 +31,9 @@ buildscript { ext { opensearch_group = "org.opensearch" - opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.0.0-alpha1-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") - buildVersionQualifier = System.getProperty("build.version_qualifier", "") + buildVersionQualifier = System.getProperty("build.version_qualifier", "alpha1") } repositories { diff --git a/bwc-test/build.gradle b/bwc-test/build.gradle index 397cbf5..bed34fe 100644 --- a/bwc-test/build.gradle +++ b/bwc-test/build.gradle @@ -44,7 +44,7 @@ ext { buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.0.0-alpha1-SNAPSHOT") opensearch_group = "org.opensearch" } repositories { From eb09e5d4ab14ec0a5afe65155b4b87f9f65ba58f Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Tue, 25 Feb 2025 23:14:13 +0530 Subject: [PATCH 02/16] Adding v4 actions as v2 and v3 are deprecated Signed-off-by: Mohit Kumar --- .github/actions/create-bwc-build/action.yaml | 4 ++-- .github/actions/run-bwc-suite/action.yaml | 2 +- .github/workflows/check.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/publish-maven-snapshots.yml | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/create-bwc-build/action.yaml b/.github/actions/create-bwc-build/action.yaml index e09a6ec..d2bb8e9 100644 --- a/.github/actions/create-bwc-build/action.yaml +++ b/.github/actions/create-bwc-build/action.yaml @@ -25,7 +25,7 @@ runs: with: path: ${{ inputs.plugin-branch }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 if: ${{ inputs.plugin-branch != 'current_branch' }} with: repository: opensearch-project/custom-codecs @@ -33,7 +33,7 @@ runs: path: ${{ inputs.plugin-branch }} - name: Build - uses: gradle/gradle-build-action@v3 + uses: gradle/gradle-build-action@v4 with: cache-disabled: true arguments: assemble diff --git a/.github/actions/run-bwc-suite/action.yaml b/.github/actions/run-bwc-suite/action.yaml index d66b7bc..191a3f8 100644 --- a/.github/actions/run-bwc-suite/action.yaml +++ b/.github/actions/run-bwc-suite/action.yaml @@ -37,7 +37,7 @@ runs: plugin-branch: ${{ inputs.plugin-next-branch }} - name: Run BWC tests - uses: gradle/gradle-build-action@v3 + uses: gradle/gradle-build-action@v4 with: cache-disabled: true arguments: | diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 63f46ec..6cf41c4 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -44,7 +44,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: temurin diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5393586..a652fa6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v4 - name: Build BWC tests - uses: gradle/gradle-build-action@v3 + uses: gradle/gradle-build-action@v4 with: cache-disabled: true arguments: | diff --git a/.github/workflows/publish-maven-snapshots.yml b/.github/workflows/publish-maven-snapshots.yml index 6e5db19..1609860 100644 --- a/.github/workflows/publish-maven-snapshots.yml +++ b/.github/workflows/publish-maven-snapshots.yml @@ -17,11 +17,11 @@ jobs: contents: write steps: - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin java-version: 21 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} From bd9622f89db056ec5067a421209fafe08df61010 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Wed, 26 Feb 2025 22:20:09 +0530 Subject: [PATCH 03/16] Adding lucene 101 classes and test cases Signed-off-by: Mohit Kumar --- .github/actions/create-bwc-build/action.yaml | 2 +- .github/actions/run-bwc-suite/action.yaml | 2 +- .github/workflows/ci.yml | 2 +- .../codec/rest/CreateIndexWithCodecIT.java | 2 +- .../codec/customcodecs/CustomCodecPlugin.java | 1 + .../customcodecs/CustomCodecService.java | 4 + .../customcodecs/Lucene101CustomCodec.java | 114 +++++++++++ .../Lucene101CustomStoredFieldsFormat.java | 142 +++++++++++++ .../codec/customcodecs/Lucene101QatCodec.java | 145 ++++++++++++++ .../Lucene101QatStoredFieldsFormat.java | 189 ++++++++++++++++++ .../customcodecs/QatCompressionMode.java | 2 +- .../customcodecs/QatDeflate101Codec.java | 97 +++++++++ .../codec/customcodecs/QatLz4101Codec.java | 97 +++++++++ .../codec/customcodecs/Zstd101Codec.java | 68 +++++++ .../customcodecs/ZstdCompressionMode.java | 2 +- .../customcodecs/ZstdNoDict101Codec.java | 66 ++++++ .../ZstdNoDictCompressionMode.java | 2 +- .../lucene912}/Lucene912CustomCodec.java | 4 +- .../Lucene912CustomStoredFieldsFormat.java | 4 +- .../lucene912}/Lucene912QatCodec.java | 4 +- .../Lucene912QatStoredFieldsFormat.java | 3 +- .../lucene912}/QatDeflate912Codec.java | 3 +- .../lucene912}/QatLz4912Codec.java | 3 +- .../lucene912}/Zstd912Codec.java | 2 +- .../lucene912}/ZstdNoDict912Codec.java | 2 +- .../services/org.apache.lucene.codecs.Codec | 8 +- .../codec/customcodecs/CustomCodecTests.java | 6 +- ...ucene101CustomStoredFieldsFormatTests.java | 63 ++++++ .../Lucene101QatStoredFieldsFormatTests.java | 71 +++++++ .../codec/customcodecs/QatCodecTests.java | 2 + ...ucene912CustomStoredFieldsFormatTests.java | 6 +- .../Lucene912QatStoredFieldsFormatTests.java | 6 +- 32 files changed, 1100 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene912}/Lucene912CustomCodec.java (96%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene912}/Lucene912CustomStoredFieldsFormat.java (96%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene912}/Lucene912QatCodec.java (97%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene912}/Lucene912QatStoredFieldsFormat.java (98%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene912}/QatDeflate912Codec.java (95%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene912}/QatLz4912Codec.java (95%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene912}/Zstd912Codec.java (96%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene912}/ZstdNoDict912Codec.java (96%) create mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java create mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java rename src/test/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene912}/Lucene912CustomStoredFieldsFormatTests.java (86%) rename src/test/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene912}/Lucene912QatStoredFieldsFormatTests.java (89%) diff --git a/.github/actions/create-bwc-build/action.yaml b/.github/actions/create-bwc-build/action.yaml index d2bb8e9..624600e 100644 --- a/.github/actions/create-bwc-build/action.yaml +++ b/.github/actions/create-bwc-build/action.yaml @@ -33,7 +33,7 @@ runs: path: ${{ inputs.plugin-branch }} - name: Build - uses: gradle/gradle-build-action@v4 + uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: assemble diff --git a/.github/actions/run-bwc-suite/action.yaml b/.github/actions/run-bwc-suite/action.yaml index 191a3f8..d66b7bc 100644 --- a/.github/actions/run-bwc-suite/action.yaml +++ b/.github/actions/run-bwc-suite/action.yaml @@ -37,7 +37,7 @@ runs: plugin-branch: ${{ inputs.plugin-next-branch }} - name: Run BWC tests - uses: gradle/gradle-build-action@v4 + uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a652fa6..5393586 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v4 - name: Build BWC tests - uses: gradle/gradle-build-action@v4 + uses: gradle/gradle-build-action@v3 with: cache-disabled: true arguments: | diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java index df2ef08..806a471 100644 --- a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java +++ b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java @@ -26,7 +26,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.codec.customcodecs.Lucene912QatCodec; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.rest.OpenSearchRestTestCase; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 7015704..f2a7535 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -11,6 +11,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecServiceFactory; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.plugins.EnginePlugin; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java index f953505..1b3085e 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -13,6 +13,10 @@ import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecService; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatDeflate912Codec; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatLz4912Codec; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Zstd912Codec; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.ZstdNoDict912Codec; import org.opensearch.index.mapper.MapperService; import java.util.Arrays; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java new file mode 100644 index 0000000..6c7f4fe --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; +import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * Supports two modes zstd and zstd_no_dict. + * Uses Lucene101 as the delegate codec + * + * @opensearch.internal + */ +public abstract class Lucene101CustomCodec extends FilterCodec { + + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** + * ZStandard mode with dictionary + */ + ZSTD("ZSTD101", Set.of("zstd")), + /** + * ZStandard mode without dictionary + */ + ZSTD_NO_DICT("ZSTDNODICT101", Set.of("zstd_no_dict")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** + * Returns the Codec that is registered with Lucene + */ + public String getCodec() { + return codec; + } + + /** + * Returns the aliases of the Codec + */ + public Set getAliases() { + return aliases; + } + } + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + */ + public Lucene101CustomCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param compressionLevel The compression level. + */ + public Lucene101CustomCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene101Codec()); + this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + */ + public Lucene101CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java new file mode 100644 index 0000000..c7f7036 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java @@ -0,0 +1,142 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; + +import java.io.IOException; +import java.util.Objects; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** Stored field format used by pluggable codec */ +public class Lucene101CustomStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene101CustomStoredFieldsFormat.class.getSimpleName() + ".mode"; + + protected static final int ZSTD_BLOCK_LENGTH = 10 * 48 * 1024; + protected static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096; + protected static final int ZSTD_BLOCK_SHIFT = 10; + + private final CompressionMode zstdCompressionMode; + private final CompressionMode zstdNoDictCompressionMode; + + private final Lucene101CustomCodec.Mode mode; + private final int compressionLevel; + + /** default constructor */ + public Lucene101CustomStoredFieldsFormat() { + this(Lucene101CustomCodec.Mode.ZSTD, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents ZSTD or ZSTDNODICT + */ + public Lucene101CustomStoredFieldsFormat(Lucene101CustomCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents ZSTD or ZSTDNODICT + * @param compressionLevel The compression level for the mode. + */ + public Lucene101CustomStoredFieldsFormat(Lucene101CustomCodec.Mode mode, int compressionLevel) { + this.mode = Objects.requireNonNull(mode); + this.compressionLevel = compressionLevel; + zstdCompressionMode = new ZstdCompressionMode(compressionLevel); + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene101CustomCodec.Mode mode = Lucene101CustomCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + StoredFieldsFormat impl(Lucene101CustomCodec.Mode mode) { + switch (mode) { + case ZSTD: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode); + case ZSTD_NO_DICT: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } + + private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { + return new Lucene90CompressingStoredFieldsFormat( + formatName, + compressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT + ); + } + + public Lucene101CustomCodec.Mode getMode() { + return mode; + } + + /** + * Returns the compression level. + */ + public int getCompressionLevel() { + return compressionLevel; + } + + public CompressionMode getCompressionMode() { + return mode == Lucene101CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; + } + +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java new file mode 100644 index 0000000..88170eb --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java @@ -0,0 +1,145 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; +import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * + * @opensearch.internal + */ +public abstract class Lucene101QatCodec extends FilterCodec { + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** QAT lz4 mode. */ + QAT_LZ4("QATLZ4101", Set.of("qat_lz4")), + + /** QAT deflate mode. */ + QAT_DEFLATE("QATDEFLATE101", Set.of("qat_deflate")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** Returns the Codec that is registered with Lucene */ + public String getCodec() { + return codec; + } + + /** Returns the aliases of the Codec */ + public Set getAliases() { + return aliases; + } + } + + /** The default compression mode. */ + public static final Mode DEFAULT_COMPRESSION_MODE = Mode.QAT_LZ4; + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + */ + public Lucene101QatCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + */ + public Lucene101QatCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene101Codec()); + this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + * @param supplier supplier for QAT mode. + */ + public Lucene101QatCodec(Mode mode, int compressionLevel, Supplier supplier) { + super(mode.getCodec(), new Lucene101Codec()); + this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + */ + public Lucene101QatCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + * @param supplier supplier for QAT mode. + */ + public Lucene101QatCodec( + Mode mode, + int compressionLevel, + MapperService mapperService, + Logger logger, + Supplier supplier + ) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java new file mode 100644 index 0000000..c88183c --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java @@ -0,0 +1,189 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_QAT_MODE; + +/** Stored field format used by pluggable codec */ +public class Lucene101QatStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene101QatStoredFieldsFormat.class.getSimpleName() + ".mode"; + + private static final int QAT_DEFLATE_BLOCK_LENGTH = 10 * 48 * 1024; + private static final int QAT_DEFLATE_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_DEFLATE_BLOCK_SHIFT = 10; + + private static final int QAT_LZ4_BLOCK_LENGTH = 10 * 8 * 1024; + private static final int QAT_LZ4_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_LZ4_BLOCK_SHIFT = 10; + + private final QatCompressionMode qatCompressionMode; + private final Lucene101QatCodec.Mode mode; + + /** default constructor */ + public Lucene101QatStoredFieldsFormat() { + this(Lucene101QatCodec.DEFAULT_COMPRESSION_MODE, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + */ + public Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param compressionLevel The compression level for the mode. + */ + public Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode mode, Supplier supplier) { + this(mode, DEFAULT_COMPRESSION_LEVEL, supplier); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param compressionLevel The compression level for the mode. + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode mode, int compressionLevel, Supplier supplier) { + this.mode = Objects.requireNonNull(mode); + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene101QatCodec.Mode mode = Lucene101QatCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + private StoredFieldsFormat impl(Lucene101QatCodec.Mode mode) { + switch (mode) { + case QAT_LZ4: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsLz4", + qatCompressionMode, + QAT_LZ4_BLOCK_LENGTH, + QAT_LZ4_MAX_DOCS_PER_BLOCK, + QAT_LZ4_BLOCK_SHIFT + ); + case QAT_DEFLATE: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsDeflate", + qatCompressionMode, + QAT_DEFLATE_BLOCK_LENGTH, + QAT_DEFLATE_MAX_DOCS_PER_BLOCK, + QAT_DEFLATE_BLOCK_SHIFT + ); + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } + + private StoredFieldsFormat getQatCompressingStoredFieldsFormat( + String formatName, + CompressionMode compressionMode, + int blockSize, + int maxDocs, + int blockShift + ) { + return new Lucene90CompressingStoredFieldsFormat(formatName, compressionMode, blockSize, maxDocs, blockShift); + } + + /** + * Gets the mode of compression. + * + * @return either QAT_LZ4 or QAT_DEFLATE + */ + public Lucene101QatCodec.Mode getMode() { + return mode; + } + + /** + * + * @return the CompressionMode instance. + */ + public QatCompressionMode getCompressionMode() { + return qatCompressionMode; + } + + /** + * Returns {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene101QatCodec.Mode mode} + * @param mode codec's {@link Lucene101QatCodec.Mode mode} + * @return the {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene101QatCodec.Mode mode} + */ + private static QatZipper.Algorithm getAlgorithm(Lucene101QatCodec.Mode mode) { + return (mode == Lucene101QatCodec.Mode.QAT_LZ4) ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java index 86046c6..4b7162a 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java @@ -125,7 +125,7 @@ private void compress(byte[] bytes, int offset, int length, DataOutput out) thro @Override public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { - final int length = (int) buffersInput.size(); + final int length = (int) buffersInput.length(); byte[] bytes = new byte[length]; buffersInput.readBytes(bytes, 0, length); compress(bytes, 0, length, out); diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java new file mode 100644 index 0000000..9e6946e --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java @@ -0,0 +1,97 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * QatDeflate101Codec provides a DEFLATE compressor using the qat-java library. + */ +public class QatDeflate101Codec extends Lucene101QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatDeflate101Codec instance with the default compression level. */ + public QatDeflate101Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatDeflate101Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatDeflate101Codec(int compressionLevel) { + super(Mode.QAT_DEFLATE, compressionLevel); + } + + /** + * Creates a new QatDeflate101Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatDeflate101Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_DEFLATE, compressionLevel, supplier); + } + + /** + * Creates a new QatDeflate101Codec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + */ + public QatDeflate101Codec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger); + } + + /** + * Creates a new QatDeflate101Codec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatDeflate101Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { + super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger, supplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_DEFLATE.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java new file mode 100644 index 0000000..d0544f5 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java @@ -0,0 +1,97 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * QatLz4101Codec provides an LZ4 compressor using the qat-java library. + */ +public class QatLz4101Codec extends Lucene101QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatLz4101Codec instance with the default compression level. */ + public QatLz4101Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatLz4101Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatLz4101Codec(int compressionLevel) { + super(Mode.QAT_LZ4, compressionLevel); + } + + /** + * Creates a new QatLz4101Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatLz4101Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_LZ4, compressionLevel, supplier); + } + + /** + * Creates a new QatLz4101Codec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + */ + public QatLz4101Codec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.QAT_LZ4, compressionLevel, mapperService, logger); + } + + /** + * Creates a new QatLz4101Codec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatLz4101Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { + super(Mode.QAT_LZ4, compressionLevel, mapperService, logger, supplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_LZ4.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java new file mode 100644 index 0000000..c0ebc0e --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * ZstdCodec provides ZSTD compressor using the zstd-jni library. + */ +public class Zstd101Codec extends Lucene101CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new ZstdCodec instance with the default compression level. */ + public Zstd101Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdCodec instance. + * + * @param compressionLevel The compression level. + */ + public Zstd101Codec(int compressionLevel) { + super(Mode.ZSTD, compressionLevel); + } + + /** + * Creates a new ZstdCodec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + */ + public Zstd101Codec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.ZSTD, compressionLevel, mapperService, logger); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java index 4fb53b7..5bb47a5 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java @@ -117,7 +117,7 @@ private void compress(byte[] bytes, int offset, int length, DataOutput out) thro @Override public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { - final int length = (int) buffersInput.size(); + final int length = (int) buffersInput.length(); byte[] bytes = new byte[length]; buffersInput.readBytes(bytes, 0, length); compress(bytes, 0, length, out); diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java new file mode 100644 index 0000000..f5d694a --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ +public class ZstdNoDict101Codec extends Lucene101Codec implements CodecSettings, CodecAliases { + + /** Creates a new ZstdNoDictCodec instance with the default compression level. */ + public ZstdNoDict101Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdNoDictCodec instance. + * + * @param compressionLevel The compression level. + */ + public ZstdNoDict101Codec(int compressionLevel) { + super(Mode.ZSTD_NO_DICT, compressionLevel); + } + + /** + * Creates a new ZstdNoDictCodec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + */ + public ZstdNoDict101Codec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD_NO_DICT.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java index fdee56d..688c902 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java @@ -105,7 +105,7 @@ private void compress(byte[] bytes, int offset, int length, DataOutput out) thro @Override public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { - final int length = (int) buffersInput.size(); + final int length = (int) buffersInput.length(); byte[] bytes = new byte[length]; buffersInput.readBytes(bytes, 0, length); compress(bytes, 0, length, out); diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java similarity index 96% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java index 692360e..9893c79 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java @@ -6,12 +6,12 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java similarity index 96% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java index 86005d5..880c0b5 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; @@ -17,6 +17,8 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import java.io.IOException; import java.util.Objects; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java similarity index 97% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java index 4010741..dab9329 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java @@ -6,12 +6,12 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java similarity index 98% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java index 71d6b7e..c541401 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; @@ -23,6 +23,7 @@ import java.util.function.Supplier; import com.intel.qat.QatZipper; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_QAT_MODE; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatDeflate912Codec.java similarity index 95% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatDeflate912Codec.java index 62365ad..6c8892c 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatDeflate912Codec.java @@ -6,12 +6,13 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.mapper.MapperService; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatLz4912Codec.java similarity index 95% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatLz4912Codec.java index 5a85aa4..44e38a9 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatLz4912Codec.java @@ -6,12 +6,13 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.mapper.MapperService; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Zstd912Codec.java similarity index 96% rename from src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Zstd912Codec.java index 6a643e7..1f9b566 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Zstd912Codec.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Setting; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/ZstdNoDict912Codec.java similarity index 96% rename from src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/ZstdNoDict912Codec.java index b256e4d..191a48e 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/ZstdNoDict912Codec.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Setting; diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 4979cdf..33cdc5a 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -5,7 +5,7 @@ org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Zstd99Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.ZstdNoDict99Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.QatDeflate99Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.QatLz499Codec -org.opensearch.index.codec.customcodecs.Zstd912Codec -org.opensearch.index.codec.customcodecs.ZstdNoDict912Codec -org.opensearch.index.codec.customcodecs.QatDeflate912Codec -org.opensearch.index.codec.customcodecs.QatLz4912Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Zstd912Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.ZstdNoDict912Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatDeflate912Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatLz4912Codec diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java index 692cbca..552f464 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -52,6 +52,10 @@ import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912CustomCodec; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912CustomStoredFieldsFormat; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatDeflate912Codec; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatLz4912Codec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.similarity.SimilarityService; diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java new file mode 100644 index 0000000..981fdc1 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; +import org.opensearch.index.codec.customcodecs.Lucene101CustomCodec; +import org.opensearch.index.codec.customcodecs.Lucene101CustomStoredFieldsFormat; +import org.opensearch.test.OpenSearchTestCase; + +public class Lucene101CustomStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testDefaultLucene101CustomCodecMode() { + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat(); + assertEquals(Lucene101CustomCodec.Mode.ZSTD, lucene101CustomStoredFieldsFormat.getMode()); + } + + public void testZstdNoDictLucene101CustomCodecMode() { + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( + Lucene101CustomCodec.Mode.ZSTD_NO_DICT + ); + assertEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, lucene101CustomStoredFieldsFormat.getMode()); + } + + public void testZstdModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( + Lucene101CustomCodec.Mode.ZSTD, + randomCompressionLevel + ); + assertEquals(Lucene101CustomCodec.Mode.ZSTD, lucene101CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene101CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictLucene101CustomCodecModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( + Lucene101CustomCodec.Mode.ZSTD_NO_DICT, + randomCompressionLevel + ); + assertEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, lucene101CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene101CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testCompressionModes() { + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat(); + assertTrue(lucene101CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); + } + + public void testZstdNoDictCompressionModes() { + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( + Lucene101CustomCodec.Mode.ZSTD_NO_DICT + ); + assertTrue(lucene101CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); + } + +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java new file mode 100644 index 0000000..4d24b83 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.index.codec.customcodecs.QatCompressionMode; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.codec.customcodecs.Lucene101QatCodec; +import org.opensearch.index.codec.customcodecs.Lucene101QatStoredFieldsFormat; +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; + +public class Lucene101QatStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testLz4Lucene101QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode.QAT_LZ4); + assertEquals(Lucene101QatCodec.Mode.QAT_LZ4, lucene101QatStoredFieldsFormat.getMode()); + } + + public void testDeflateLucene101QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( + Lucene101QatCodec.Mode.QAT_DEFLATE + ); + assertEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, lucene101QatStoredFieldsFormat.getMode()); + } + + public void testLz4Lucene101QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( + Lucene101QatCodec.Mode.QAT_LZ4, + randomCompressionLevel + ); + assertEquals(Lucene101QatCodec.Mode.QAT_LZ4, lucene101QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene101QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testDeflateLucene101QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( + Lucene101QatCodec.Mode.QAT_DEFLATE, + randomCompressionLevel + ); + assertEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, lucene101QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene101QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testLz4CompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode.QAT_LZ4); + assertTrue(lucene101QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } + + public void testDeflateCompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( + Lucene101QatCodec.Mode.QAT_DEFLATE + ); + assertTrue(lucene101QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index 2eb9a06..feccf68 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -49,6 +49,8 @@ import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatStoredFieldsFormat; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.indices.mapper.MapperRegistry; diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormatTests.java similarity index 86% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormatTests.java index 1118cd1..81241d4 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormatTests.java @@ -6,8 +6,12 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912CustomCodec; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912CustomStoredFieldsFormat; import org.opensearch.test.OpenSearchTestCase; public class Lucene912CustomStoredFieldsFormatTests extends OpenSearchTestCase { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormatTests.java similarity index 89% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormatTests.java index 936e712..f1faf54 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormatTests.java @@ -6,8 +6,12 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatStoredFieldsFormat; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.is; From f797e68841fde06c4a3023efacba05e03cef3583 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Wed, 26 Feb 2025 23:15:48 +0530 Subject: [PATCH 04/16] Adding specific ubuntu platforms Signed-off-by: Mohit Kumar --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5393586..29beea9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: fail-fast: false matrix: jdk: [21] - platform: [ubuntu-latest, windows-latest] + platform: [ ubuntu-24.04-arm, ubuntu-24.04, windows-latest ] runs-on: ${{ matrix.platform }} steps: From 1cb8bb10015c49609e844c0807e03f23abf19a50 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Thu, 27 Feb 2025 02:44:07 +0530 Subject: [PATCH 05/16] BWC test command change Signed-off-by: Mohit Kumar --- .github/actions/create-bwc-build/action.yaml | 1 + .github/workflows/ci.yml | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/actions/create-bwc-build/action.yaml b/.github/actions/create-bwc-build/action.yaml index 624600e..a11152d 100644 --- a/.github/actions/create-bwc-build/action.yaml +++ b/.github/actions/create-bwc-build/action.yaml @@ -39,6 +39,7 @@ runs: arguments: assemble build-root-directory: ${{ inputs.plugin-branch }} + - id: get-opensearch-version uses: peternied/get-opensearch-version@v1 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29beea9..5c86d37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,18 +25,14 @@ jobs: uses: actions/checkout@v4 - name: Build BWC tests - uses: gradle/gradle-build-action@v3 - with: - cache-disabled: true - arguments: | - -p bwc-test build -x test -x integTest + run: ./gradlew -p bwc-test build -x test -x integTest backward-compatibility: strategy: fail-fast: false matrix: jdk: [21] - platform: [ ubuntu-24.04-arm, ubuntu-24.04, windows-latest ] + platform: [ ubuntu-latest, windows-latest ] runs-on: ${{ matrix.platform }} steps: From 6c9ccd16ef9bea303f964cb67b1f1f1f30b99cf9 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 26 Feb 2025 21:24:37 +0000 Subject: [PATCH 06/16] fix github actions Signed-off-by: Peter Zhu --- .github/actions/run-bwc-suite/action.yaml | 2 +- gradle/wrapper/gradle-wrapper.properties | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-bwc-suite/action.yaml b/.github/actions/run-bwc-suite/action.yaml index d66b7bc..426f852 100644 --- a/.github/actions/run-bwc-suite/action.yaml +++ b/.github/actions/run-bwc-suite/action.yaml @@ -49,7 +49,7 @@ runs: -Dbwc.version.previous=${{ steps.build-previous.outputs.built-version }} -Dbwc.version.next=${{ steps.build-next.outputs.built-version }} -i - - uses: alehechka/upload-tartifact@v2 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ inputs.report-artifact-name }} diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 723af56..6acc143 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,8 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip +distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=5b9c5eb3f9fc2c94abaea57d90bd78747ca117ddbbf96c859d3741181a12bf2a From aa800b31fbba69074a3ed5953f008225f78687c3 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Thu, 27 Feb 2025 03:05:39 +0530 Subject: [PATCH 07/16] Fixing syntax failures Signed-off-by: Mohit Kumar --- .../index/codec/customcodecs/ZstdNoDict101Codec.java | 3 +-- .../backward_codecs/lucene912/Lucene912CustomCodec.java | 3 ++- .../lucene912/Lucene912CustomStoredFieldsFormat.java | 6 ++++-- .../backward_codecs/lucene912/Lucene912QatCodec.java | 5 +++-- .../lucene912/Lucene912QatStoredFieldsFormat.java | 3 ++- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java index f5d694a..ea4ee5c 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java @@ -9,7 +9,6 @@ package org.opensearch.index.codec.customcodecs; import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; @@ -21,7 +20,7 @@ import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; /** ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ -public class ZstdNoDict101Codec extends Lucene101Codec implements CodecSettings, CodecAliases { +public class ZstdNoDict101Codec extends Lucene101CustomCodec implements CodecSettings, CodecAliases { /** Creates a new ZstdNoDictCodec instance with the default compression level. */ public ZstdNoDict101Codec() { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java index 9893c79..709909e 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java @@ -12,6 +12,7 @@ import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; @@ -98,7 +99,7 @@ public Lucene912CustomCodec(Mode mode, int compressionLevel) { * @param logger The logger. */ public Lucene912CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java index 880c0b5..a68b5dd 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java @@ -64,8 +64,10 @@ public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode) { public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; - zstdCompressionMode = new ZstdCompressionMode(compressionLevel); - zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + zstdCompressionMode = new ZstdCompressionMode(compressionLevel){ + }; + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel){ + }; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java index dab9329..dd788bb 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java @@ -12,6 +12,7 @@ import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; @@ -107,7 +108,7 @@ public Lucene912QatCodec(Mode mode, int compressionLevel, Supplier supplier ) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel, supplier); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java index c541401..b7e52e2 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java @@ -88,7 +88,8 @@ public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode, Supplier supplier) { this.mode = Objects.requireNonNull(mode); - qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier); + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier){ + }; } /** From 0299b5fea9919f6e68d4c6c578f79db04084f3f5 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 26 Feb 2025 21:41:21 +0000 Subject: [PATCH 08/16] Update spotless Signed-off-by: Peter Zhu --- .../codec/rest/CreateIndexWithCodecIT.java | 2 +- .../Lucene101CustomStoredFieldsFormat.java | 14 ++++---- .../codec/customcodecs/Lucene101QatCodec.java | 10 +++--- .../Lucene101QatStoredFieldsFormat.java | 33 +++++++++---------- .../customcodecs/QatDeflate101Codec.java | 1 - .../codec/customcodecs/QatLz4101Codec.java | 1 - .../lucene912/Lucene912CustomCodec.java | 2 +- .../Lucene912CustomStoredFieldsFormat.java | 4 +-- .../lucene912/Lucene912QatCodec.java | 2 +- .../Lucene912QatStoredFieldsFormat.java | 4 +-- .../codec/customcodecs/CustomCodecTests.java | 2 +- ...ucene101CustomStoredFieldsFormatTests.java | 16 ++++----- .../Lucene101QatStoredFieldsFormatTests.java | 16 ++++----- ...ucene912CustomStoredFieldsFormatTests.java | 2 -- .../Lucene912QatStoredFieldsFormatTests.java | 2 -- 15 files changed, 47 insertions(+), 64 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java index 806a471..7b9302a 100644 --- a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java +++ b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java @@ -26,8 +26,8 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.test.rest.OpenSearchRestTestCase; import javax.net.ssl.SSLEngine; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java index c7f7036..4e593ed 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java @@ -17,8 +17,6 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; -import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import java.io.IOException; import java.util.Objects; @@ -97,7 +95,7 @@ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOCo String previous = si.putAttribute(MODE_KEY, mode.name()); if (previous != null && previous.equals(mode.name()) == false) { throw new IllegalStateException( - "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() ); } return impl(mode).fieldsWriter(directory, si, context); @@ -116,11 +114,11 @@ StoredFieldsFormat impl(Lucene101CustomCodec.Mode mode) { private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { return new Lucene90CompressingStoredFieldsFormat( - formatName, - compressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT + formatName, + compressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT ); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java index 88170eb..0b71c32 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java @@ -123,11 +123,11 @@ public Lucene101QatCodec(Mode mode, int compressionLevel, MapperService mapperSe * @param supplier supplier for QAT mode. */ public Lucene101QatCodec( - Mode mode, - int compressionLevel, - MapperService mapperService, - Logger logger, - Supplier supplier + Mode mode, + int compressionLevel, + MapperService mapperService, + Logger logger, + Supplier supplier ) { super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel, supplier); diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java index c88183c..cd74463 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java @@ -23,7 +23,6 @@ import java.util.function.Supplier; import com.intel.qat.QatZipper; -import org.opensearch.index.codec.customcodecs.QatCompressionMode; import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_QAT_MODE; @@ -122,7 +121,7 @@ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOCo String previous = si.putAttribute(MODE_KEY, mode.name()); if (previous != null && previous.equals(mode.name()) == false) { throw new IllegalStateException( - "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() ); } return impl(mode).fieldsWriter(directory, si, context); @@ -132,19 +131,19 @@ private StoredFieldsFormat impl(Lucene101QatCodec.Mode mode) { switch (mode) { case QAT_LZ4: return getQatCompressingStoredFieldsFormat( - "QatStoredFieldsLz4", - qatCompressionMode, - QAT_LZ4_BLOCK_LENGTH, - QAT_LZ4_MAX_DOCS_PER_BLOCK, - QAT_LZ4_BLOCK_SHIFT + "QatStoredFieldsLz4", + qatCompressionMode, + QAT_LZ4_BLOCK_LENGTH, + QAT_LZ4_MAX_DOCS_PER_BLOCK, + QAT_LZ4_BLOCK_SHIFT ); case QAT_DEFLATE: return getQatCompressingStoredFieldsFormat( - "QatStoredFieldsDeflate", - qatCompressionMode, - QAT_DEFLATE_BLOCK_LENGTH, - QAT_DEFLATE_MAX_DOCS_PER_BLOCK, - QAT_DEFLATE_BLOCK_SHIFT + "QatStoredFieldsDeflate", + qatCompressionMode, + QAT_DEFLATE_BLOCK_LENGTH, + QAT_DEFLATE_MAX_DOCS_PER_BLOCK, + QAT_DEFLATE_BLOCK_SHIFT ); default: throw new IllegalStateException("Unsupported compression mode: " + mode); @@ -152,11 +151,11 @@ private StoredFieldsFormat impl(Lucene101QatCodec.Mode mode) { } private StoredFieldsFormat getQatCompressingStoredFieldsFormat( - String formatName, - CompressionMode compressionMode, - int blockSize, - int maxDocs, - int blockShift + String formatName, + CompressionMode compressionMode, + int blockSize, + int maxDocs, + int blockShift ) { return new Lucene90CompressingStoredFieldsFormat(formatName, compressionMode, blockSize, maxDocs, blockShift); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java index 9e6946e..8d36d37 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java @@ -12,7 +12,6 @@ import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; -import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.mapper.MapperService; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java index d0544f5..0f988e0 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java @@ -12,7 +12,6 @@ import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; -import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.mapper.MapperService; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java index 709909e..327bd97 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java @@ -9,9 +9,9 @@ package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.logging.log4j.Logger; +import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java index a68b5dd..e4f8f3a 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java @@ -64,9 +64,9 @@ public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode) { public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; - zstdCompressionMode = new ZstdCompressionMode(compressionLevel){ + zstdCompressionMode = new ZstdCompressionMode(compressionLevel) { }; - zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel){ + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel) { }; } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java index dd788bb..5d91353 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java @@ -9,9 +9,9 @@ package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.logging.log4j.Logger; +import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java index b7e52e2..7db1c96 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java @@ -17,13 +17,13 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; import java.io.IOException; import java.util.Objects; import java.util.function.Supplier; import com.intel.qat.QatZipper; -import org.opensearch.index.codec.customcodecs.QatCompressionMode; import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_QAT_MODE; @@ -88,7 +88,7 @@ public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode, Supplier supplier) { this.mode = Objects.requireNonNull(mode); - qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier){ + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier) { }; } diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java index 552f464..436d0a7 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -33,9 +33,9 @@ package org.opensearch.index.codec.customcodecs; import org.apache.logging.log4j.LogManager; +import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java index 981fdc1..b4088ab 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java @@ -8,10 +8,6 @@ package org.opensearch.index.codec.customcodecs; -import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; -import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; -import org.opensearch.index.codec.customcodecs.Lucene101CustomCodec; -import org.opensearch.index.codec.customcodecs.Lucene101CustomStoredFieldsFormat; import org.opensearch.test.OpenSearchTestCase; public class Lucene101CustomStoredFieldsFormatTests extends OpenSearchTestCase { @@ -23,7 +19,7 @@ public void testDefaultLucene101CustomCodecMode() { public void testZstdNoDictLucene101CustomCodecMode() { Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( - Lucene101CustomCodec.Mode.ZSTD_NO_DICT + Lucene101CustomCodec.Mode.ZSTD_NO_DICT ); assertEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, lucene101CustomStoredFieldsFormat.getMode()); } @@ -31,8 +27,8 @@ public void testZstdNoDictLucene101CustomCodecMode() { public void testZstdModeWithCompressionLevel() { int randomCompressionLevel = randomIntBetween(1, 6); Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( - Lucene101CustomCodec.Mode.ZSTD, - randomCompressionLevel + Lucene101CustomCodec.Mode.ZSTD, + randomCompressionLevel ); assertEquals(Lucene101CustomCodec.Mode.ZSTD, lucene101CustomStoredFieldsFormat.getMode()); assertEquals(randomCompressionLevel, lucene101CustomStoredFieldsFormat.getCompressionLevel()); @@ -41,8 +37,8 @@ public void testZstdModeWithCompressionLevel() { public void testZstdNoDictLucene101CustomCodecModeWithCompressionLevel() { int randomCompressionLevel = randomIntBetween(1, 6); Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( - Lucene101CustomCodec.Mode.ZSTD_NO_DICT, - randomCompressionLevel + Lucene101CustomCodec.Mode.ZSTD_NO_DICT, + randomCompressionLevel ); assertEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, lucene101CustomStoredFieldsFormat.getMode()); assertEquals(randomCompressionLevel, lucene101CustomStoredFieldsFormat.getCompressionLevel()); @@ -55,7 +51,7 @@ public void testCompressionModes() { public void testZstdNoDictCompressionModes() { Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( - Lucene101CustomCodec.Mode.ZSTD_NO_DICT + Lucene101CustomCodec.Mode.ZSTD_NO_DICT ); assertTrue(lucene101CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); } diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java index 4d24b83..b7af855 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java @@ -8,10 +8,6 @@ package org.opensearch.index.codec.customcodecs; -import org.opensearch.index.codec.customcodecs.QatCompressionMode; -import org.opensearch.index.codec.customcodecs.QatZipperFactory; -import org.opensearch.index.codec.customcodecs.Lucene101QatCodec; -import org.opensearch.index.codec.customcodecs.Lucene101QatStoredFieldsFormat; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.is; @@ -28,7 +24,7 @@ public void testLz4Lucene101QatCodecMode() { public void testDeflateLucene101QatCodecMode() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( - Lucene101QatCodec.Mode.QAT_DEFLATE + Lucene101QatCodec.Mode.QAT_DEFLATE ); assertEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, lucene101QatStoredFieldsFormat.getMode()); } @@ -37,8 +33,8 @@ public void testLz4Lucene101QatCodecModeWithCompressionLevel() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( - Lucene101QatCodec.Mode.QAT_LZ4, - randomCompressionLevel + Lucene101QatCodec.Mode.QAT_LZ4, + randomCompressionLevel ); assertEquals(Lucene101QatCodec.Mode.QAT_LZ4, lucene101QatStoredFieldsFormat.getMode()); assertEquals(randomCompressionLevel, lucene101QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); @@ -48,8 +44,8 @@ public void testDeflateLucene101QatCodecModeWithCompressionLevel() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( - Lucene101QatCodec.Mode.QAT_DEFLATE, - randomCompressionLevel + Lucene101QatCodec.Mode.QAT_DEFLATE, + randomCompressionLevel ); assertEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, lucene101QatStoredFieldsFormat.getMode()); assertEquals(randomCompressionLevel, lucene101QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); @@ -64,7 +60,7 @@ public void testLz4CompressionModes() { public void testDeflateCompressionModes() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( - Lucene101QatCodec.Mode.QAT_DEFLATE + Lucene101QatCodec.Mode.QAT_DEFLATE ); assertTrue(lucene101QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); } diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormatTests.java index 81241d4..1ee2391 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormatTests.java @@ -10,8 +10,6 @@ import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912CustomCodec; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912CustomStoredFieldsFormat; import org.opensearch.test.OpenSearchTestCase; public class Lucene912CustomStoredFieldsFormatTests extends OpenSearchTestCase { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormatTests.java index f1faf54..713bcde 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormatTests.java @@ -10,8 +10,6 @@ import org.opensearch.index.codec.customcodecs.QatCompressionMode; import org.opensearch.index.codec.customcodecs.QatZipperFactory; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatStoredFieldsFormat; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.is; From f5ae11af4644d485e345c088b6ae36526b4e683b Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 26 Feb 2025 22:06:01 +0000 Subject: [PATCH 09/16] Update version extraction Signed-off-by: Peter Zhu --- bwc-test/build.gradle | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bwc-test/build.gradle b/bwc-test/build.gradle index bed34fe..11cced0 100644 --- a/bwc-test/build.gradle +++ b/bwc-test/build.gradle @@ -84,15 +84,9 @@ String baseName = "customCodecsBwcCluster" String bwcFilePath = "src/test/resources/" String projectVersion = nextVersion -String previousOpenSearch = extractVersion(previousVersion) + "-SNAPSHOT"; -String nextOpenSearch = extractVersion(nextVersion) + "-SNAPSHOT"; - // Extracts the OpenSearch version from a plugin version string, 2.11.0.0 -> 2.11.0. -def String extractVersion(versionStr) { - def versionMatcher = versionStr =~ /(.+?)(\.\d+)$/ - versionMatcher.find() - return versionMatcher.group(1) -} +String previousOpenSearch = previousVersion.tokenize('-')[0][0..-3] + "-SNAPSHOT" +String nextOpenSearch = nextVersion.tokenize('-')[0][0..-3] + "-SNAPSHOT" 2.times {i -> testClusters { From 4e532cbba9e99e9f5ff030f8d9da10b2fbd9f5af Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 26 Feb 2025 22:11:30 +0000 Subject: [PATCH 10/16] Update results to include qualifier as well Signed-off-by: Peter Zhu --- bwc-test/build.gradle | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bwc-test/build.gradle b/bwc-test/build.gradle index 11cced0..76ae370 100644 --- a/bwc-test/build.gradle +++ b/bwc-test/build.gradle @@ -85,8 +85,10 @@ String bwcFilePath = "src/test/resources/" String projectVersion = nextVersion // Extracts the OpenSearch version from a plugin version string, 2.11.0.0 -> 2.11.0. -String previousOpenSearch = previousVersion.tokenize('-')[0][0..-3] + "-SNAPSHOT" -String nextOpenSearch = nextVersion.tokenize('-')[0][0..-3] + "-SNAPSHOT" +previousToken = previousVersion.tokenize('-') +nextToken = nextVersion.tokenize('-') +String previousOpenSearch = previousToken[1] ? previousToken[0][0..-3] + "-" + previousToken[1] + "-SNAPSHOT" : previousToken[0][0..-3] + "-SNAPSHOT" +String nextOpenSearch = nextToken[1] ? nextToken[0][0..-3] + "-" + nextToken[1] + "-SNAPSHOT" : nextToken[0][0..-3] + "-SNAPSHOT" 2.times {i -> testClusters { From 8c8b8c49a43fe732c69170908dc6ec0f4ec1e27b Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 26 Feb 2025 22:13:43 +0000 Subject: [PATCH 11/16] Add def previousToken Signed-off-by: Peter Zhu --- bwc-test/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bwc-test/build.gradle b/bwc-test/build.gradle index 76ae370..5f658d3 100644 --- a/bwc-test/build.gradle +++ b/bwc-test/build.gradle @@ -85,8 +85,8 @@ String bwcFilePath = "src/test/resources/" String projectVersion = nextVersion // Extracts the OpenSearch version from a plugin version string, 2.11.0.0 -> 2.11.0. -previousToken = previousVersion.tokenize('-') -nextToken = nextVersion.tokenize('-') +def previousToken = previousVersion.tokenize('-') +def nextToken = nextVersion.tokenize('-') String previousOpenSearch = previousToken[1] ? previousToken[0][0..-3] + "-" + previousToken[1] + "-SNAPSHOT" : previousToken[0][0..-3] + "-SNAPSHOT" String nextOpenSearch = nextToken[1] ? nextToken[0][0..-3] + "-" + nextToken[1] + "-SNAPSHOT" : nextToken[0][0..-3] + "-SNAPSHOT" From f3c4fb0d3c7de742224c6e1f88bf1cbc5cb01801 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 26 Feb 2025 22:18:41 +0000 Subject: [PATCH 12/16] Update new regex Signed-off-by: Peter Zhu --- bwc-test/build.gradle | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/bwc-test/build.gradle b/bwc-test/build.gradle index 5f658d3..302cd3a 100644 --- a/bwc-test/build.gradle +++ b/bwc-test/build.gradle @@ -84,11 +84,19 @@ String baseName = "customCodecsBwcCluster" String bwcFilePath = "src/test/resources/" String projectVersion = nextVersion +String previousOpenSearch = extractVersion(previousVersion) + "-SNAPSHOT"; +String nextOpenSearch = extractVersion(nextVersion) + "-SNAPSHOT"; + // Extracts the OpenSearch version from a plugin version string, 2.11.0.0 -> 2.11.0. -def previousToken = previousVersion.tokenize('-') -def nextToken = nextVersion.tokenize('-') -String previousOpenSearch = previousToken[1] ? previousToken[0][0..-3] + "-" + previousToken[1] + "-SNAPSHOT" : previousToken[0][0..-3] + "-SNAPSHOT" -String nextOpenSearch = nextToken[1] ? nextToken[0][0..-3] + "-" + nextToken[1] + "-SNAPSHOT" : nextToken[0][0..-3] + "-SNAPSHOT" +def String extractVersion(versionStr) { + def versionMatcher = versionStr =~ /(.+?)(\.\d+)(-(alpha|beta|rc)\d+)?$/ + versionMatcher.find() + if (versionMatcher.group(3) != null) { /* there is a qualifier */ + return versionMatcher.group(1) + versionMatcher.group(3) + } else { + return versionMatcher.group(1) + } +} 2.times {i -> testClusters { From ec959ca3aa2c5aa68f59ddf990b3b4cb143da255 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Thu, 27 Feb 2025 14:39:47 +0530 Subject: [PATCH 13/16] Addressing comments and resolving conflicts Signed-off-by: Mohit Kumar --- .../codec/customcodecs/CustomCodecPlugin.java | 7 +-- .../customcodecs/CustomCodecService.java | 20 +++---- .../lucene912/Lucene912CustomCodec.java | 18 ------- .../codec/customcodecs/CustomCodecTests.java | 52 +++++++++---------- .../codec/customcodecs/QatCodecTests.java | 33 ++++++------ 5 files changed, 50 insertions(+), 80 deletions(-) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index f2a7535..8f6c559 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -11,7 +11,6 @@ import org.opensearch.common.settings.Setting; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecServiceFactory; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.plugins.EnginePlugin; @@ -66,9 +65,7 @@ public List> getSettings() { } private static boolean isQatCodec(String codecName) { - return codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec()) - || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec()) - || codecName.equals(Lucene912QatCodec.Mode.QAT_LZ4.getCodec()) - || codecName.equals(Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec()); + return codecName.equals(Lucene101QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene101QatCodec.Mode.QAT_DEFLATE.getCodec()); } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java index 1b3085e..2c536fe 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -13,10 +13,6 @@ import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecService; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatDeflate912Codec; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatLz4912Codec; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Zstd912Codec; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.ZstdNoDict912Codec; import org.opensearch.index.mapper.MapperService; import java.util.Arrays; @@ -54,25 +50,25 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(ZSTD_CODEC, new Zstd912Codec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict912Codec(compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd101Codec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(compressionLevel)); if (QatZipperFactory.isQatAvailable()) { codecs.put( QAT_LZ4_CODEC, - new QatLz4912Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }) + new QatLz4101Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }) ); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate912Codec(compressionLevel, () -> { + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate101Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); } } else { - codecs.put(ZSTD_CODEC, new Zstd912Codec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict912Codec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd101Codec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(mapperService, logger, compressionLevel)); if (QatZipperFactory.isQatAvailable()) { - codecs.put(QAT_LZ4_CODEC, new QatLz4912Codec(mapperService, logger, compressionLevel, () -> { + codecs.put(QAT_LZ4_CODEC, new QatLz4101Codec(mapperService, logger, compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate912Codec(mapperService, logger, compressionLevel, () -> { + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate101Codec(mapperService, logger, compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java index 327bd97..5ef04a7 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java @@ -12,9 +12,6 @@ import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene101.Lucene101Codec; -import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; -import org.opensearch.index.mapper.MapperService; import java.util.Set; @@ -88,21 +85,6 @@ public Lucene912CustomCodec(Mode mode, int compressionLevel) { this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); } - /** - * Creates a new compression codec with the given compression level. We use - * lowercase letters when registering the codec so that we remain consistent with - * the other compression codecs: default, lucene_default, and best_compression. - * - * @param mode The compression codec (ZSTD or ZSTDNODICT). - * @param compressionLevel The compression level. - * @param mapperService The mapper service. - * @param logger The logger. - */ - public Lucene912CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); - } - @Override public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java index 436d0a7..88786cf 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -33,9 +33,9 @@ package org.opensearch.index.codec.customcodecs; import org.apache.logging.log4j.LogManager; -import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -52,10 +52,6 @@ import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.codec.CodecSettings; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912CustomCodec; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912CustomStoredFieldsFormat; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatDeflate912Codec; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatLz4912Codec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.similarity.SimilarityService; @@ -88,15 +84,15 @@ public void setup() { public void testZstd() throws Exception { Codec codec = createCodecService(false).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDict() throws Exception { Codec codec = createCodecService(false).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } @@ -111,16 +107,16 @@ public void testZstdDeprecatedCodec() { public void testZstdWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } @@ -161,25 +157,25 @@ public void testZstandardCompressionLevelSupport() throws Exception { public void testDefaultMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene101Codec.Mode.BEST_SPEED, codec); } public void testBestCompressionMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene101Codec.Mode.BEST_COMPRESSION, codec); } public void testZstdMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } @@ -188,11 +184,11 @@ public void testQatCodecsNotAvailable() throws IOException { assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_lz4")); assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_deflate")); - QatLz4912Codec qatLz4912Codec = new QatLz4912Codec(); - assertTrue(qatLz4912Codec.aliases().isEmpty()); + QatLz4101Codec qatLz4101Codec = new QatLz4101Codec(); + assertTrue(qatLz4101Codec.aliases().isEmpty()); - QatDeflate912Codec qatDeflate912Codec = new QatDeflate912Codec(); - assertTrue(qatDeflate912Codec.aliases().isEmpty()); + QatDeflate101Codec qatDeflate101Codec = new QatDeflate101Codec(); + assertTrue(qatDeflate101Codec.aliases().isEmpty()); } } @@ -208,18 +204,18 @@ public void testCodecServiceFactoryQatUnavailable() throws IOException { } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene912Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene101Codec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene912Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene101Codec.Mode.valueOf(v)); } - private void assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene912CustomStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene101CustomStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene912CustomCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene101CustomCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index feccf68..06be0b3 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -49,8 +49,7 @@ import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.codec.CodecSettings; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatStoredFieldsFormat; + import org.opensearch.index.mapper.MapperService; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.indices.mapper.MapperRegistry; @@ -82,16 +81,16 @@ public void setup() { public void testQatLz4() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_LZ4, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflate() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -99,8 +98,8 @@ public void testQatLz4WithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_lz4").codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_LZ4, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -108,8 +107,8 @@ public void testQatDeflateWithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_deflate").codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -125,25 +124,25 @@ public void testQatCompressionLevelSupport() throws Exception { public void testQatLz4MapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_LZ4, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflateMapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } - private void assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode expected, Codec actual) throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene912QatStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene101QatStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene912QatCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene101QatCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { From bfa001fb1dd33b9aeb56b196746804b39019ec8a Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Thu, 27 Feb 2025 14:39:47 +0530 Subject: [PATCH 14/16] Addressing comments and resolving conflicts Signed-off-by: Mohit Kumar --- .github/actions/create-bwc-build/action.yaml | 1 - .github/workflows/ci.yml | 2 +- .../codec/rest/CreateIndexWithCodecIT.java | 4 +- .../codec/customcodecs/CustomCodecPlugin.java | 7 +++- .../lucene912/Lucene912QatCodec.java | 39 ------------------- .../services/org.apache.lucene.codecs.Codec | 4 ++ 6 files changed, 13 insertions(+), 44 deletions(-) diff --git a/.github/actions/create-bwc-build/action.yaml b/.github/actions/create-bwc-build/action.yaml index a11152d..624600e 100644 --- a/.github/actions/create-bwc-build/action.yaml +++ b/.github/actions/create-bwc-build/action.yaml @@ -39,7 +39,6 @@ runs: arguments: assemble build-root-directory: ${{ inputs.plugin-branch }} - - id: get-opensearch-version uses: peternied/get-opensearch-version@v1 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c86d37..6e51250 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,7 @@ jobs: fail-fast: false matrix: jdk: [21] - platform: [ ubuntu-latest, windows-latest ] + platform: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java index 7b9302a..5aefc24 100644 --- a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java +++ b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java @@ -27,7 +27,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; import org.opensearch.index.codec.customcodecs.QatZipperFactory; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; +import org.opensearch.index.codec.customcodecs.Lucene101QatCodec; import org.opensearch.test.rest.OpenSearchRestTestCase; import javax.net.ssl.SSLEngine; @@ -105,7 +105,7 @@ public void testCreateIndexWithQatSPICodecWithQatHardwareUnavailable() throws IO .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put( "index.codec", - randomFrom(Lucene912QatCodec.Mode.QAT_LZ4.getCodec(), Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec()) + randomFrom(Lucene101QatCodec.Mode.QAT_LZ4.getCodec(), Lucene101QatCodec.Mode.QAT_DEFLATE.getCodec()) ) .put("index.codec.compression_level", randomIntBetween(1, 6)) .build() diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 8f6c559..7addcba 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -12,6 +12,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.Plugin; @@ -65,7 +66,11 @@ public List> getSettings() { } private static boolean isQatCodec(String codecName) { - return codecName.equals(Lucene101QatCodec.Mode.QAT_LZ4.getCodec()) + return codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec()) + || codecName.equals(Lucene912QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec()) + || codecName.equals(Lucene101QatCodec.Mode.QAT_LZ4.getCodec()) || codecName.equals(Lucene101QatCodec.Mode.QAT_DEFLATE.getCodec()); } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java index 5d91353..96adaaa 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java @@ -12,9 +12,6 @@ import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene101.Lucene101Codec; -import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; -import org.opensearch.index.mapper.MapperService; import java.util.Set; import java.util.function.Supplier; @@ -97,42 +94,6 @@ public Lucene912QatCodec(Mode mode, int compressionLevel, Supplier supplier - ) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel, supplier); - } @Override public StoredFieldsFormat storedFieldsFormat() { diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 33cdc5a..ba03119 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -9,3 +9,7 @@ org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Zstd912Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.ZstdNoDict912Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatDeflate912Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatLz4912Codec +org.opensearch.index.codec.customcodecs.Zstd101Codec +org.opensearch.index.codec.customcodecs.ZstdNoDict101Codec +org.opensearch.index.codec.customcodecs.QatDeflate101Codec +org.opensearch.index.codec.customcodecs.QatLz4101Codec From dff4221ad74babdb40e29c1c4a7f0a2171df60d9 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Thu, 27 Feb 2025 14:39:47 +0530 Subject: [PATCH 15/16] Addressing comments and resolving conflicts Signed-off-by: Mohit Kumar --- .../codec/rest/CreateIndexWithCodecIT.java | 2 +- .../codec/customcodecs/CustomCodecPlugin.java | 1 + .../lucene912/Lucene912CustomCodec.java | 1 - .../lucene912/Lucene912QatCodec.java | 2 -- .../lucene912/QatDeflate912Codec.java | 25 ------------------- .../lucene912/QatLz4912Codec.java | 25 ------------------- .../lucene912/Zstd912Codec.java | 13 ---------- .../lucene912/ZstdNoDict912Codec.java | 13 ---------- .../codec/customcodecs/CustomCodecTests.java | 2 +- .../codec/customcodecs/QatCodecTests.java | 1 - 10 files changed, 3 insertions(+), 82 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java index 5aefc24..3213f2d 100644 --- a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java +++ b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java @@ -26,8 +26,8 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.index.codec.customcodecs.Lucene101QatCodec; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.rest.OpenSearchRestTestCase; import javax.net.ssl.SSLEngine; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 7addcba..aef69e0 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -11,6 +11,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecServiceFactory; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec; import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.index.engine.EngineConfig; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java index 5ef04a7..c41b3d9 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java @@ -8,7 +8,6 @@ package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; -import org.apache.logging.log4j.Logger; import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java index 96adaaa..f594754 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java @@ -8,7 +8,6 @@ package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; -import org.apache.logging.log4j.Logger; import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; @@ -94,7 +93,6 @@ public Lucene912QatCodec(Mode mode, int compressionLevel, Supplier supplie super(Mode.QAT_DEFLATE, compressionLevel, supplier); } - /** - * Creates a new QatDeflate912Codec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - */ - public QatDeflate912Codec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger); - } - - /** - * Creates a new QatDeflate912Codec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - * @param supplier supplier for QAT acceleration mode. - */ - public QatDeflate912Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { - super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger, supplier); - } - /** The name for this codec. */ @Override public String toString() { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatLz4912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatLz4912Codec.java index 44e38a9..fa89900 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatLz4912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatLz4912Codec.java @@ -8,13 +8,11 @@ package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; -import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; import java.util.Set; import java.util.function.Supplier; @@ -53,29 +51,6 @@ public QatLz4912Codec(int compressionLevel, Supplier supplier) { super(Mode.QAT_LZ4, compressionLevel, supplier); } - /** - * Creates a new QatLz4912Codec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - */ - public QatLz4912Codec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.QAT_LZ4, compressionLevel, mapperService, logger); - } - - /** - * Creates a new QatLz4912Codec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - * @param supplier supplier for QAT acceleration mode. - */ - public QatLz4912Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { - super(Mode.QAT_LZ4, compressionLevel, mapperService, logger, supplier); - } - /** The name for this codec. */ @Override public String toString() { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Zstd912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Zstd912Codec.java index 1f9b566..76eb4ac 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Zstd912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Zstd912Codec.java @@ -8,12 +8,10 @@ package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; -import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; import java.util.Set; @@ -39,17 +37,6 @@ public Zstd912Codec(int compressionLevel) { super(Mode.ZSTD, compressionLevel); } - /** - * Creates a new ZstdCodec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - */ - public Zstd912Codec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.ZSTD, compressionLevel, mapperService, logger); - } - /** The name for this codec. */ @Override public String toString() { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/ZstdNoDict912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/ZstdNoDict912Codec.java index 191a48e..d434913 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/ZstdNoDict912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/ZstdNoDict912Codec.java @@ -8,12 +8,10 @@ package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; -import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; import java.util.Set; @@ -36,17 +34,6 @@ public ZstdNoDict912Codec(int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel); } - /** - * Creates a new ZstdNoDictCodec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - */ - public ZstdNoDict912Codec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger); - } - /** The name for this codec. */ @Override public String toString() { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java index 88786cf..fce8bdd 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -34,8 +34,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; import org.apache.lucene.codecs.lucene101.Lucene101Codec; +import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index 06be0b3..c4e7e84 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -49,7 +49,6 @@ import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.codec.CodecSettings; - import org.opensearch.index.mapper.MapperService; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.indices.mapper.MapperRegistry; From 5a07ef69ac78c79e5665ee807f810f574a904c87 Mon Sep 17 00:00:00 2001 From: Mohit Kumar Date: Thu, 27 Feb 2025 14:39:47 +0530 Subject: [PATCH 16/16] Addressed comments and resolving conflicts Signed-off-by: Mohit Kumar --- .../opensearch/index/codec/customcodecs/CustomCodecPlugin.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index aef69e0..1ba9e85 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -13,7 +13,6 @@ import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec; -import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.Plugin;