diff --git a/.github/actions/create-bwc-build/action.yaml b/.github/actions/create-bwc-build/action.yaml index e09a6ec..624600e 100644 --- a/.github/actions/create-bwc-build/action.yaml +++ b/.github/actions/create-bwc-build/action.yaml @@ -25,7 +25,7 @@ runs: with: path: ${{ inputs.plugin-branch }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 if: ${{ inputs.plugin-branch != 'current_branch' }} with: repository: opensearch-project/custom-codecs diff --git a/.github/actions/run-bwc-suite/action.yaml b/.github/actions/run-bwc-suite/action.yaml index d66b7bc..426f852 100644 --- a/.github/actions/run-bwc-suite/action.yaml +++ b/.github/actions/run-bwc-suite/action.yaml @@ -49,7 +49,7 @@ runs: -Dbwc.version.previous=${{ steps.build-previous.outputs.built-version }} -Dbwc.version.next=${{ steps.build-next.outputs.built-version }} -i - - uses: alehechka/upload-tartifact@v2 + - uses: actions/upload-artifact@v4 if: always() with: name: ${{ inputs.report-artifact-name }} diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 63f46ec..6cf41c4 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -44,7 +44,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.java }} distribution: temurin diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5393586..6e51250 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,11 +25,7 @@ jobs: uses: actions/checkout@v4 - name: Build BWC tests - uses: gradle/gradle-build-action@v3 - with: - cache-disabled: true - arguments: | - -p bwc-test build -x test -x integTest + run: ./gradlew -p bwc-test build -x test -x integTest backward-compatibility: strategy: diff --git a/.github/workflows/publish-maven-snapshots.yml b/.github/workflows/publish-maven-snapshots.yml index 6e5db19..1609860 100644 --- a/.github/workflows/publish-maven-snapshots.yml +++ b/.github/workflows/publish-maven-snapshots.yml @@ -17,11 +17,11 @@ jobs: contents: write steps: - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin java-version: 21 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} diff --git a/build.gradle b/build.gradle index 00a6f89..54025f9 100644 --- a/build.gradle +++ b/build.gradle @@ -31,9 +31,9 @@ buildscript { ext { opensearch_group = "org.opensearch" - opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.0.0-alpha1-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") - buildVersionQualifier = System.getProperty("build.version_qualifier", "") + buildVersionQualifier = System.getProperty("build.version_qualifier", "alpha1") } repositories { diff --git a/bwc-test/build.gradle b/bwc-test/build.gradle index 397cbf5..302cd3a 100644 --- a/bwc-test/build.gradle +++ b/bwc-test/build.gradle @@ -44,7 +44,7 @@ ext { buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "3.0.0-alpha1-SNAPSHOT") opensearch_group = "org.opensearch" } repositories { @@ -89,9 +89,13 @@ String nextOpenSearch = extractVersion(nextVersion) + "-SNAPSHOT"; // Extracts the OpenSearch version from a plugin version string, 2.11.0.0 -> 2.11.0. def String extractVersion(versionStr) { - def versionMatcher = versionStr =~ /(.+?)(\.\d+)$/ + def versionMatcher = versionStr =~ /(.+?)(\.\d+)(-(alpha|beta|rc)\d+)?$/ versionMatcher.find() - return versionMatcher.group(1) + if (versionMatcher.group(3) != null) { /* there is a qualifier */ + return versionMatcher.group(1) + versionMatcher.group(3) + } else { + return versionMatcher.group(1) + } } 2.times {i -> diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 723af56..6acc143 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,8 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip +distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=5b9c5eb3f9fc2c94abaea57d90bd78747ca117ddbbf96c859d3741181a12bf2a diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java index df2ef08..3213f2d 100644 --- a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java +++ b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java @@ -26,7 +26,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.codec.customcodecs.Lucene912QatCodec; +import org.opensearch.index.codec.customcodecs.Lucene101QatCodec; import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -105,7 +105,7 @@ public void testCreateIndexWithQatSPICodecWithQatHardwareUnavailable() throws IO .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put( "index.codec", - randomFrom(Lucene912QatCodec.Mode.QAT_LZ4.getCodec(), Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec()) + randomFrom(Lucene101QatCodec.Mode.QAT_LZ4.getCodec(), Lucene101QatCodec.Mode.QAT_DEFLATE.getCodec()) ) .put("index.codec.compression_level", randomIntBetween(1, 6)) .build() diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 7015704..1ba9e85 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -11,6 +11,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecServiceFactory; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec; import org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.plugins.EnginePlugin; @@ -68,6 +69,8 @@ private static boolean isQatCodec(String codecName) { return codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec()) || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec()) || codecName.equals(Lucene912QatCodec.Mode.QAT_LZ4.getCodec()) - || codecName.equals(Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec()); + || codecName.equals(Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec()) + || codecName.equals(Lucene101QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene101QatCodec.Mode.QAT_DEFLATE.getCodec()); } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java index f953505..2c536fe 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -50,25 +50,25 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(ZSTD_CODEC, new Zstd912Codec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict912Codec(compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd101Codec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(compressionLevel)); if (QatZipperFactory.isQatAvailable()) { codecs.put( QAT_LZ4_CODEC, - new QatLz4912Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }) + new QatLz4101Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }) ); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate912Codec(compressionLevel, () -> { + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate101Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); } } else { - codecs.put(ZSTD_CODEC, new Zstd912Codec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict912Codec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd101Codec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(mapperService, logger, compressionLevel)); if (QatZipperFactory.isQatAvailable()) { - codecs.put(QAT_LZ4_CODEC, new QatLz4912Codec(mapperService, logger, compressionLevel, () -> { + codecs.put(QAT_LZ4_CODEC, new QatLz4101Codec(mapperService, logger, compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate912Codec(mapperService, logger, compressionLevel, () -> { + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate101Codec(mapperService, logger, compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java similarity index 81% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java index 692360e..6c7f4fe 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java @@ -11,7 +11,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; @@ -23,22 +23,22 @@ * * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. * Supports two modes zstd and zstd_no_dict. - * Uses Lucene912 as the delegate codec + * Uses Lucene101 as the delegate codec * * @opensearch.internal */ -public abstract class Lucene912CustomCodec extends FilterCodec { +public abstract class Lucene101CustomCodec extends FilterCodec { /** Each mode represents a compression algorithm. */ public enum Mode { /** * ZStandard mode with dictionary */ - ZSTD("ZSTD912", Set.of("zstd")), + ZSTD("ZSTD101", Set.of("zstd")), /** * ZStandard mode without dictionary */ - ZSTD_NO_DICT("ZSTDNODICT912", Set.of("zstd_no_dict")); + ZSTD_NO_DICT("ZSTDNODICT101", Set.of("zstd_no_dict")); private final String codec; private final Set aliases; @@ -70,7 +70,7 @@ public Set getAliases() { * * @param mode The compression codec (ZSTD or ZSTDNODICT). */ - public Lucene912CustomCodec(Mode mode) { + public Lucene101CustomCodec(Mode mode) { this(mode, DEFAULT_COMPRESSION_LEVEL); } @@ -82,9 +82,9 @@ public Lucene912CustomCodec(Mode mode) { * @param mode The compression codec (ZSTD or ZSTDNODICT). * @param compressionLevel The compression level. */ - public Lucene912CustomCodec(Mode mode, int compressionLevel) { - super(mode.getCodec(), new Lucene912Codec()); - this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); + public Lucene101CustomCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene101Codec()); + this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel); } /** @@ -97,9 +97,9 @@ public Lucene912CustomCodec(Mode mode, int compressionLevel) { * @param mapperService The mapper service. * @param logger The logger. */ - public Lucene912CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); + public Lucene101CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel); } @Override diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java new file mode 100644 index 0000000..4e593ed --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java @@ -0,0 +1,140 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +import java.io.IOException; +import java.util.Objects; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** Stored field format used by pluggable codec */ +public class Lucene101CustomStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene101CustomStoredFieldsFormat.class.getSimpleName() + ".mode"; + + protected static final int ZSTD_BLOCK_LENGTH = 10 * 48 * 1024; + protected static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096; + protected static final int ZSTD_BLOCK_SHIFT = 10; + + private final CompressionMode zstdCompressionMode; + private final CompressionMode zstdNoDictCompressionMode; + + private final Lucene101CustomCodec.Mode mode; + private final int compressionLevel; + + /** default constructor */ + public Lucene101CustomStoredFieldsFormat() { + this(Lucene101CustomCodec.Mode.ZSTD, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents ZSTD or ZSTDNODICT + */ + public Lucene101CustomStoredFieldsFormat(Lucene101CustomCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents ZSTD or ZSTDNODICT + * @param compressionLevel The compression level for the mode. + */ + public Lucene101CustomStoredFieldsFormat(Lucene101CustomCodec.Mode mode, int compressionLevel) { + this.mode = Objects.requireNonNull(mode); + this.compressionLevel = compressionLevel; + zstdCompressionMode = new ZstdCompressionMode(compressionLevel); + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene101CustomCodec.Mode mode = Lucene101CustomCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + StoredFieldsFormat impl(Lucene101CustomCodec.Mode mode) { + switch (mode) { + case ZSTD: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode); + case ZSTD_NO_DICT: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } + + private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { + return new Lucene90CompressingStoredFieldsFormat( + formatName, + compressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT + ); + } + + public Lucene101CustomCodec.Mode getMode() { + return mode; + } + + /** + * Returns the compression level. + */ + public int getCompressionLevel() { + return compressionLevel; + } + + public CompressionMode getCompressionMode() { + return mode == Lucene101CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; + } + +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java similarity index 81% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java index 4010741..0b71c32 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java @@ -11,7 +11,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; @@ -27,14 +27,14 @@ * * @opensearch.internal */ -public abstract class Lucene912QatCodec extends FilterCodec { +public abstract class Lucene101QatCodec extends FilterCodec { /** Each mode represents a compression algorithm. */ public enum Mode { /** QAT lz4 mode. */ - QAT_LZ4("QATLZ4912", Set.of("qat_lz4")), + QAT_LZ4("QATLZ4101", Set.of("qat_lz4")), /** QAT deflate mode. */ - QAT_DEFLATE("QATDEFLATE912", Set.of("qat_deflate")); + QAT_DEFLATE("QATDEFLATE101", Set.of("qat_deflate")); private final String codec; private final Set aliases; @@ -65,7 +65,7 @@ public Set getAliases() { * * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). */ - public Lucene912QatCodec(Mode mode) { + public Lucene101QatCodec(Mode mode) { this(mode, DEFAULT_COMPRESSION_LEVEL); } @@ -77,9 +77,9 @@ public Lucene912QatCodec(Mode mode) { * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). * @param compressionLevel The compression level. */ - public Lucene912QatCodec(Mode mode, int compressionLevel) { - super(mode.getCodec(), new Lucene912Codec()); - this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel); + public Lucene101QatCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene101Codec()); + this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel); } /** @@ -91,9 +91,9 @@ public Lucene912QatCodec(Mode mode, int compressionLevel) { * @param compressionLevel The compression level. * @param supplier supplier for QAT mode. */ - public Lucene912QatCodec(Mode mode, int compressionLevel, Supplier supplier) { - super(mode.getCodec(), new Lucene912Codec()); - this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel, supplier); + public Lucene101QatCodec(Mode mode, int compressionLevel, Supplier supplier) { + super(mode.getCodec(), new Lucene101Codec()); + this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel, supplier); } /** @@ -106,9 +106,9 @@ public Lucene912QatCodec(Mode mode, int compressionLevel, Supplier supplier ) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel, supplier); + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel, supplier); } @Override diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java new file mode 100644 index 0000000..cd74463 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormat.java @@ -0,0 +1,188 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_QAT_MODE; + +/** Stored field format used by pluggable codec */ +public class Lucene101QatStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene101QatStoredFieldsFormat.class.getSimpleName() + ".mode"; + + private static final int QAT_DEFLATE_BLOCK_LENGTH = 10 * 48 * 1024; + private static final int QAT_DEFLATE_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_DEFLATE_BLOCK_SHIFT = 10; + + private static final int QAT_LZ4_BLOCK_LENGTH = 10 * 8 * 1024; + private static final int QAT_LZ4_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_LZ4_BLOCK_SHIFT = 10; + + private final QatCompressionMode qatCompressionMode; + private final Lucene101QatCodec.Mode mode; + + /** default constructor */ + public Lucene101QatStoredFieldsFormat() { + this(Lucene101QatCodec.DEFAULT_COMPRESSION_MODE, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + */ + public Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param compressionLevel The compression level for the mode. + */ + public Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode mode, Supplier supplier) { + this(mode, DEFAULT_COMPRESSION_LEVEL, supplier); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param compressionLevel The compression level for the mode. + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode mode, int compressionLevel, Supplier supplier) { + this.mode = Objects.requireNonNull(mode); + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene101QatCodec.Mode mode = Lucene101QatCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + private StoredFieldsFormat impl(Lucene101QatCodec.Mode mode) { + switch (mode) { + case QAT_LZ4: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsLz4", + qatCompressionMode, + QAT_LZ4_BLOCK_LENGTH, + QAT_LZ4_MAX_DOCS_PER_BLOCK, + QAT_LZ4_BLOCK_SHIFT + ); + case QAT_DEFLATE: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsDeflate", + qatCompressionMode, + QAT_DEFLATE_BLOCK_LENGTH, + QAT_DEFLATE_MAX_DOCS_PER_BLOCK, + QAT_DEFLATE_BLOCK_SHIFT + ); + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } + + private StoredFieldsFormat getQatCompressingStoredFieldsFormat( + String formatName, + CompressionMode compressionMode, + int blockSize, + int maxDocs, + int blockShift + ) { + return new Lucene90CompressingStoredFieldsFormat(formatName, compressionMode, blockSize, maxDocs, blockShift); + } + + /** + * Gets the mode of compression. + * + * @return either QAT_LZ4 or QAT_DEFLATE + */ + public Lucene101QatCodec.Mode getMode() { + return mode; + } + + /** + * + * @return the CompressionMode instance. + */ + public QatCompressionMode getCompressionMode() { + return qatCompressionMode; + } + + /** + * Returns {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene101QatCodec.Mode mode} + * @param mode codec's {@link Lucene101QatCodec.Mode mode} + * @return the {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene101QatCodec.Mode mode} + */ + private static QatZipper.Algorithm getAlgorithm(Lucene101QatCodec.Mode mode) { + return (mode == Lucene101QatCodec.Mode.QAT_LZ4) ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java index 86046c6..4b7162a 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java @@ -125,7 +125,7 @@ private void compress(byte[] bytes, int offset, int length, DataOutput out) thro @Override public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { - final int length = (int) buffersInput.size(); + final int length = (int) buffersInput.length(); byte[] bytes = new byte[length]; buffersInput.readBytes(bytes, 0, length); compress(bytes, 0, length, out); diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java similarity index 78% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java index 62365ad..8d36d37 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java @@ -23,55 +23,55 @@ import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; /** - * QatDeflate912Codec provides a DEFLATE compressor using the qat-java library. */ -public class QatDeflate912Codec extends Lucene912QatCodec implements CodecSettings, CodecAliases { +public class QatDeflate101Codec extends Lucene101QatCodec implements CodecSettings, CodecAliases { - /** Creates a new QatDeflate912Codec instance with the default compression level. */ - public QatDeflate912Codec() { + /** Creates a new QatDeflate101Codec instance with the default compression level. */ + public QatDeflate101Codec() { this(DEFAULT_COMPRESSION_LEVEL); } /** - * Creates a new QatDeflate912Codec instance. + * Creates a new QatDeflate101Codec instance. * * @param compressionLevel The compression level. */ - public QatDeflate912Codec(int compressionLevel) { + public QatDeflate101Codec(int compressionLevel) { super(Mode.QAT_DEFLATE, compressionLevel); } /** - * Creates a new QatDeflate912Codec instance with the default compression level. + * Creates a new QatDeflate101Codec instance with the default compression level. * * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatDeflate912Codec(int compressionLevel, Supplier supplier) { + public QatDeflate101Codec(int compressionLevel, Supplier supplier) { super(Mode.QAT_DEFLATE, compressionLevel, supplier); } /** - * Creates a new QatDeflate912Codec instance. + * Creates a new QatDeflate101Codec instance. * * @param mapperService The mapper service. * @param logger The logger. * @param compressionLevel The compression level. */ - public QatDeflate912Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public QatDeflate101Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger); } /** - * Creates a new QatDeflate912Codec instance. + * Creates a new QatDeflate101Codec instance. * * @param mapperService The mapper service. * @param logger The logger. * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatDeflate912Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { + public QatDeflate101Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger, supplier); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java similarity index 78% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java index 5a85aa4..0f988e0 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java @@ -23,55 +23,55 @@ import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; /** - * QatLz4912Codec provides an LZ4 compressor using the qat-java library. */ -public class QatLz4912Codec extends Lucene912QatCodec implements CodecSettings, CodecAliases { +public class QatLz4101Codec extends Lucene101QatCodec implements CodecSettings, CodecAliases { - /** Creates a new QatLz4912Codec instance with the default compression level. */ - public QatLz4912Codec() { + /** Creates a new QatLz4101Codec instance with the default compression level. */ + public QatLz4101Codec() { this(DEFAULT_COMPRESSION_LEVEL); } /** - * Creates a new QatLz4912Codec instance. + * Creates a new QatLz4101Codec instance. * * @param compressionLevel The compression level. */ - public QatLz4912Codec(int compressionLevel) { + public QatLz4101Codec(int compressionLevel) { super(Mode.QAT_LZ4, compressionLevel); } /** - * Creates a new QatLz4912Codec instance with the default compression level. + * Creates a new QatLz4101Codec instance with the default compression level. * * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatLz4912Codec(int compressionLevel, Supplier supplier) { + public QatLz4101Codec(int compressionLevel, Supplier supplier) { super(Mode.QAT_LZ4, compressionLevel, supplier); } /** - * Creates a new QatLz4912Codec instance. + * Creates a new QatLz4101Codec instance. * * @param mapperService The mapper service. * @param logger The logger. * @param compressionLevel The compression level. */ - public QatLz4912Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public QatLz4101Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.QAT_LZ4, compressionLevel, mapperService, logger); } /** - * Creates a new QatLz4912Codec instance. + * Creates a new QatLz4101Codec instance. * * @param mapperService The mapper service. * @param logger The logger. * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatLz4912Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { + public QatLz4101Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { super(Mode.QAT_LZ4, compressionLevel, mapperService, logger, supplier); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java new file mode 100644 index 0000000..c0ebc0e --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java @@ -0,0 +1,68 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * ZstdCodec provides ZSTD compressor using the zstd-jni library. + */ +public class Zstd101Codec extends Lucene101CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new ZstdCodec instance with the default compression level. */ + public Zstd101Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdCodec instance. + * + * @param compressionLevel The compression level. + */ + public Zstd101Codec(int compressionLevel) { + super(Mode.ZSTD, compressionLevel); + } + + /** + * Creates a new ZstdCodec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + */ + public Zstd101Codec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.ZSTD, compressionLevel, mapperService, logger); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java index 4fb53b7..5bb47a5 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java @@ -117,7 +117,7 @@ private void compress(byte[] bytes, int offset, int length, DataOutput out) thro @Override public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { - final int length = (int) buffersInput.size(); + final int length = (int) buffersInput.length(); byte[] bytes = new byte[length]; buffersInput.readBytes(bytes, 0, length); compress(bytes, 0, length, out); diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java similarity index 89% rename from src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java index b256e4d..ea4ee5c 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java @@ -20,10 +20,10 @@ import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; /** ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ -public class ZstdNoDict912Codec extends Lucene912CustomCodec implements CodecSettings, CodecAliases { +public class ZstdNoDict101Codec extends Lucene101CustomCodec implements CodecSettings, CodecAliases { /** Creates a new ZstdNoDictCodec instance with the default compression level. */ - public ZstdNoDict912Codec() { + public ZstdNoDict101Codec() { this(DEFAULT_COMPRESSION_LEVEL); } @@ -32,7 +32,7 @@ public ZstdNoDict912Codec() { * * @param compressionLevel The compression level. */ - public ZstdNoDict912Codec(int compressionLevel) { + public ZstdNoDict101Codec(int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel); } @@ -43,7 +43,7 @@ public ZstdNoDict912Codec(int compressionLevel) { * @param logger The logger. * @param compressionLevel The compression level. */ - public ZstdNoDict912Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public ZstdNoDict101Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java index fdee56d..688c902 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java @@ -105,7 +105,7 @@ private void compress(byte[] bytes, int offset, int length, DataOutput out) thro @Override public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { - final int length = (int) buffersInput.size(); + final int length = (int) buffersInput.length(); byte[] bytes = new byte[length]; buffersInput.readBytes(bytes, 0, length); compress(bytes, 0, length, out); diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java new file mode 100644 index 0000000..c41b3d9 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomCodec.java @@ -0,0 +1,96 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; + +import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; + +import java.util.Set; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * Supports two modes zstd and zstd_no_dict. + * Uses Lucene912 as the delegate codec + * + * @opensearch.internal + */ +public abstract class Lucene912CustomCodec extends FilterCodec { + + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** + * ZStandard mode with dictionary + */ + ZSTD("ZSTD912", Set.of("zstd")), + /** + * ZStandard mode without dictionary + */ + ZSTD_NO_DICT("ZSTDNODICT912", Set.of("zstd_no_dict")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** + * Returns the Codec that is registered with Lucene + */ + public String getCodec() { + return codec; + } + + /** + * Returns the aliases of the Codec + */ + public Set getAliases() { + return aliases; + } + } + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + */ + public Lucene912CustomCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param compressionLevel The compression level. + */ + public Lucene912CustomCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene912Codec()); + this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java similarity index 95% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java index 86005d5..e4f8f3a 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; @@ -17,6 +17,8 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import java.io.IOException; import java.util.Objects; @@ -62,8 +64,10 @@ public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode) { public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; - zstdCompressionMode = new ZstdCompressionMode(compressionLevel); - zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + zstdCompressionMode = new ZstdCompressionMode(compressionLevel) { + }; + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel) { + }; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java new file mode 100644 index 0000000..f594754 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatCodec.java @@ -0,0 +1,105 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; + +import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * + * @opensearch.internal + */ +public abstract class Lucene912QatCodec extends FilterCodec { + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** QAT lz4 mode. */ + QAT_LZ4("QATLZ4912", Set.of("qat_lz4")), + + /** QAT deflate mode. */ + QAT_DEFLATE("QATDEFLATE912", Set.of("qat_deflate")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** Returns the Codec that is registered with Lucene */ + public String getCodec() { + return codec; + } + + /** Returns the aliases of the Codec */ + public Set getAliases() { + return aliases; + } + } + + /** The default compression mode. */ + public static final Mode DEFAULT_COMPRESSION_MODE = Mode.QAT_LZ4; + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + */ + public Lucene912QatCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + */ + public Lucene912QatCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene912Codec()); + this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + * @param supplier supplier for QAT mode. + */ + public Lucene912QatCodec(Mode mode, int compressionLevel, Supplier supplier) { + super(mode.getCodec(), new Lucene912Codec()); + this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java similarity index 97% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java index 71d6b7e..7db1c96 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; @@ -17,6 +17,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; import java.io.IOException; import java.util.Objects; @@ -87,7 +88,8 @@ public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode, Supplier supplier) { this.mode = Objects.requireNonNull(mode); - qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier); + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier) { + }; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatDeflate912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatDeflate912Codec.java new file mode 100644 index 0000000..5b97d42 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatDeflate912Codec.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * QatDeflate912Codec provides a DEFLATE compressor using the qat-java library. + */ +public class QatDeflate912Codec extends Lucene912QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatDeflate912Codec instance with the default compression level. */ + public QatDeflate912Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatDeflate912Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatDeflate912Codec(int compressionLevel) { + super(Mode.QAT_DEFLATE, compressionLevel); + } + + /** + * Creates a new QatDeflate912Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatDeflate912Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_DEFLATE, compressionLevel, supplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_DEFLATE.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatLz4912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatLz4912Codec.java new file mode 100644 index 0000000..fa89900 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/QatLz4912Codec.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * QatLz4912Codec provides an LZ4 compressor using the qat-java library. + */ +public class QatLz4912Codec extends Lucene912QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatLz4912Codec instance with the default compression level. */ + public QatLz4912Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatLz4912Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatLz4912Codec(int compressionLevel) { + super(Mode.QAT_LZ4, compressionLevel); + } + + /** + * Creates a new QatLz4912Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatLz4912Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_LZ4, compressionLevel, supplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_LZ4.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Zstd912Codec.java similarity index 75% rename from src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Zstd912Codec.java index 6a643e7..76eb4ac 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Zstd912Codec.java @@ -6,14 +6,12 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; -import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Setting; import org.opensearch.index.codec.CodecAliases; import org.opensearch.index.codec.CodecSettings; import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; import java.util.Set; @@ -39,17 +37,6 @@ public Zstd912Codec(int compressionLevel) { super(Mode.ZSTD, compressionLevel); } - /** - * Creates a new ZstdCodec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - */ - public Zstd912Codec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.ZSTD, compressionLevel, mapperService, logger); - } - /** The name for this codec. */ @Override public String toString() { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/ZstdNoDict912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/ZstdNoDict912Codec.java new file mode 100644 index 0000000..d434913 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/ZstdNoDict912Codec.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ +public class ZstdNoDict912Codec extends Lucene912CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new ZstdNoDictCodec instance with the default compression level. */ + public ZstdNoDict912Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdNoDictCodec instance. + * + * @param compressionLevel The compression level. + */ + public ZstdNoDict912Codec(int compressionLevel) { + super(Mode.ZSTD_NO_DICT, compressionLevel); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD_NO_DICT.getAliases(); + } +} diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 4979cdf..ba03119 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -5,7 +5,11 @@ org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Zstd99Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.ZstdNoDict99Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.QatDeflate99Codec org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.QatLz499Codec -org.opensearch.index.codec.customcodecs.Zstd912Codec -org.opensearch.index.codec.customcodecs.ZstdNoDict912Codec -org.opensearch.index.codec.customcodecs.QatDeflate912Codec -org.opensearch.index.codec.customcodecs.QatLz4912Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Zstd912Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.ZstdNoDict912Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatDeflate912Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatLz4912Codec +org.opensearch.index.codec.customcodecs.Zstd101Codec +org.opensearch.index.codec.customcodecs.ZstdNoDict101Codec +org.opensearch.index.codec.customcodecs.QatDeflate101Codec +org.opensearch.index.codec.customcodecs.QatLz4101Codec diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java index 692cbca..fce8bdd 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -34,8 +34,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -84,15 +84,15 @@ public void setup() { public void testZstd() throws Exception { Codec codec = createCodecService(false).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDict() throws Exception { Codec codec = createCodecService(false).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } @@ -107,16 +107,16 @@ public void testZstdDeprecatedCodec() { public void testZstdWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } @@ -157,25 +157,25 @@ public void testZstandardCompressionLevelSupport() throws Exception { public void testDefaultMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene101Codec.Mode.BEST_SPEED, codec); } public void testBestCompressionMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene101Codec.Mode.BEST_COMPRESSION, codec); } public void testZstdMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene101CustomStoredFieldsFormat storedFieldsFormat = (Lucene101CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } @@ -184,11 +184,11 @@ public void testQatCodecsNotAvailable() throws IOException { assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_lz4")); assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_deflate")); - QatLz4912Codec qatLz4912Codec = new QatLz4912Codec(); - assertTrue(qatLz4912Codec.aliases().isEmpty()); + QatLz4101Codec qatLz4101Codec = new QatLz4101Codec(); + assertTrue(qatLz4101Codec.aliases().isEmpty()); - QatDeflate912Codec qatDeflate912Codec = new QatDeflate912Codec(); - assertTrue(qatDeflate912Codec.aliases().isEmpty()); + QatDeflate101Codec qatDeflate101Codec = new QatDeflate101Codec(); + assertTrue(qatDeflate101Codec.aliases().isEmpty()); } } @@ -204,18 +204,18 @@ public void testCodecServiceFactoryQatUnavailable() throws IOException { } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene912Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene101Codec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene912Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene101Codec.Mode.valueOf(v)); } - private void assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene101CustomCodec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene912CustomStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene101CustomStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene912CustomCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene101CustomCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java new file mode 100644 index 0000000..b4088ab --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormatTests.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +public class Lucene101CustomStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testDefaultLucene101CustomCodecMode() { + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat(); + assertEquals(Lucene101CustomCodec.Mode.ZSTD, lucene101CustomStoredFieldsFormat.getMode()); + } + + public void testZstdNoDictLucene101CustomCodecMode() { + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( + Lucene101CustomCodec.Mode.ZSTD_NO_DICT + ); + assertEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, lucene101CustomStoredFieldsFormat.getMode()); + } + + public void testZstdModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( + Lucene101CustomCodec.Mode.ZSTD, + randomCompressionLevel + ); + assertEquals(Lucene101CustomCodec.Mode.ZSTD, lucene101CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene101CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictLucene101CustomCodecModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( + Lucene101CustomCodec.Mode.ZSTD_NO_DICT, + randomCompressionLevel + ); + assertEquals(Lucene101CustomCodec.Mode.ZSTD_NO_DICT, lucene101CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene101CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testCompressionModes() { + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat(); + assertTrue(lucene101CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); + } + + public void testZstdNoDictCompressionModes() { + Lucene101CustomStoredFieldsFormat lucene101CustomStoredFieldsFormat = new Lucene101CustomStoredFieldsFormat( + Lucene101CustomCodec.Mode.ZSTD_NO_DICT + ); + assertTrue(lucene101CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); + } + +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java new file mode 100644 index 0000000..b7af855 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene101QatStoredFieldsFormatTests.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; + +public class Lucene101QatStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testLz4Lucene101QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode.QAT_LZ4); + assertEquals(Lucene101QatCodec.Mode.QAT_LZ4, lucene101QatStoredFieldsFormat.getMode()); + } + + public void testDeflateLucene101QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( + Lucene101QatCodec.Mode.QAT_DEFLATE + ); + assertEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, lucene101QatStoredFieldsFormat.getMode()); + } + + public void testLz4Lucene101QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( + Lucene101QatCodec.Mode.QAT_LZ4, + randomCompressionLevel + ); + assertEquals(Lucene101QatCodec.Mode.QAT_LZ4, lucene101QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene101QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testDeflateLucene101QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( + Lucene101QatCodec.Mode.QAT_DEFLATE, + randomCompressionLevel + ); + assertEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, lucene101QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene101QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testLz4CompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat(Lucene101QatCodec.Mode.QAT_LZ4); + assertTrue(lucene101QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } + + public void testDeflateCompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene101QatStoredFieldsFormat lucene101QatStoredFieldsFormat = new Lucene101QatStoredFieldsFormat( + Lucene101QatCodec.Mode.QAT_DEFLATE + ); + assertTrue(lucene101QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index 2eb9a06..c4e7e84 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -80,16 +80,16 @@ public void setup() { public void testQatLz4() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_LZ4, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflate() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -97,8 +97,8 @@ public void testQatLz4WithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_lz4").codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_LZ4, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -106,8 +106,8 @@ public void testQatDeflateWithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_deflate").codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -123,25 +123,25 @@ public void testQatCompressionLevelSupport() throws Exception { public void testQatLz4MapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_LZ4, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflateMapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); - Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode.QAT_DEFLATE, codec); + Lucene101QatStoredFieldsFormat storedFieldsFormat = (Lucene101QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } - private void assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene101QatCodec.Mode expected, Codec actual) throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene912QatStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene101QatStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene912QatCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene101QatCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormatTests.java similarity index 92% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormatTests.java index 1118cd1..1ee2391 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912CustomStoredFieldsFormatTests.java @@ -6,8 +6,10 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import org.opensearch.test.OpenSearchTestCase; public class Lucene912CustomStoredFieldsFormatTests extends OpenSearchTestCase { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormatTests.java similarity index 94% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormatTests.java index 936e712..713bcde 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene912/Lucene912QatStoredFieldsFormatTests.java @@ -6,8 +6,10 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene912; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.is;