Bumping custom-codecs plugin version to 3.0.0-alpha1 and lucene upgrade

.github/actions/create-bwc-build/action.yaml

@@ -25,7 +25,7 @@ runs:
       with:
         path: ${{ inputs.plugin-branch }}
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       if: ${{ inputs.plugin-branch != 'current_branch' }}
       with:
         repository: opensearch-project/custom-codecs
@@ -39,6 +39,7 @@ runs:
         arguments: assemble
         build-root-directory: ${{ inputs.plugin-branch }}
 
+
     - id: get-opensearch-version
       uses: peternied/get-opensearch-version@v1
       with:

.github/actions/run-bwc-suite/action.yaml

@@ -49,7 +49,7 @@ runs:
           -Dbwc.version.previous=${{ steps.build-previous.outputs.built-version }}
           -Dbwc.version.next=${{ steps.build-next.outputs.built-version }} -i
 
-    - uses: alehechka/upload-tartifact@v2
+    - uses: actions/upload-artifact@v4
       if: always()
       with:
         name: ${{ inputs.report-artifact-name }}

.github/workflows/check.yml

@@ -44,7 +44,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin

.github/workflows/ci.yml

@@ -25,18 +25,14 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Build BWC tests
-        uses: gradle/gradle-build-action@v3
-        with:
-          cache-disabled: true
-          arguments: |
-            -p bwc-test build -x test -x integTest
+        run: ./gradlew -p bwc-test build -x test -x integTest
 
   backward-compatibility:
     strategy:
       fail-fast: false
       matrix:
         jdk: [21]
-        platform: [ubuntu-latest, windows-latest]
+        platform: [ ubuntu-latest, windows-latest ]
     runs-on: ${{ matrix.platform }}
 
     steps:

.github/workflows/publish-maven-snapshots.yml

@@ -17,11 +17,11 @@ jobs:
       contents: write
 
     steps:
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: temurin
           java-version: 21
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: aws-actions/configure-aws-credentials@v1
         with:
           role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}

build.gradle

@@ -31,9 +31,9 @@
 buildscript {
   ext {
     opensearch_group = "org.opensearch"
-    opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
+    opensearch_version = System.getProperty("opensearch.version", "3.0.0-alpha1-SNAPSHOT")
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-    buildVersionQualifier = System.getProperty("build.version_qualifier", "")
+    buildVersionQualifier = System.getProperty("build.version_qualifier", "alpha1")
   }
 
   repositories {

bwc-test/build.gradle

@@ -44,7 +44,7 @@ ext {
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.0.0-alpha1-SNAPSHOT")
         opensearch_group = "org.opensearch"
     }
     repositories {
@@ -89,9 +89,13 @@ String nextOpenSearch  = extractVersion(nextVersion) + "-SNAPSHOT";
 
 // Extracts the OpenSearch version from a plugin version string, 2.11.0.0 -> 2.11.0.
 def String extractVersion(versionStr) {
-    def versionMatcher =  versionStr =~ /(.+?)(\.\d+)$/
+    def versionMatcher =  versionStr =~ /(.+?)(\.\d+)(-(alpha|beta|rc)\d+)?$/
     versionMatcher.find()
-    return versionMatcher.group(1)
+    if (versionMatcher.group(3) != null) { /* there is a qualifier */
+        return versionMatcher.group(1) + versionMatcher.group(3)
+    } else {
+        return versionMatcher.group(1)
+    }
 }
 
 2.times {i ->

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,8 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
+distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
+networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=5b9c5eb3f9fc2c94abaea57d90bd78747ca117ddbbf96c859d3741181a12bf2a

src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java

@@ -26,8 +26,8 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.Strings;
-import org.opensearch.index.codec.customcodecs.Lucene912QatCodec;
 import org.opensearch.index.codec.customcodecs.QatZipperFactory;
+import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 
 import javax.net.ssl.SSLEngine;

src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java

@@ -65,9 +65,7 @@ public List<Setting<?>> getSettings() {
     }
 
     private static boolean isQatCodec(String codecName) {
-        return codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec())
-            || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec())
-            || codecName.equals(Lucene912QatCodec.Mode.QAT_LZ4.getCodec())
-            || codecName.equals(Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec());
+        return  codecName.equals(Lucene101QatCodec.Mode.QAT_LZ4.getCodec())
+            || codecName.equals(Lucene101QatCodec.Mode.QAT_DEFLATE.getCodec());
     }
 }

src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java

@@ -50,25 +50,25 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin
         int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING);
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         if (mapperService == null) {
-            codecs.put(ZSTD_CODEC, new Zstd912Codec(compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict912Codec(compressionLevel));
+            codecs.put(ZSTD_CODEC, new Zstd101Codec(compressionLevel));
+            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(compressionLevel));
             if (QatZipperFactory.isQatAvailable()) {
                 codecs.put(
                     QAT_LZ4_CODEC,
-                    new QatLz4912Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })
+                    new QatLz4101Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })
                 );
-                codecs.put(QAT_DEFLATE_CODEC, new QatDeflate912Codec(compressionLevel, () -> {
+                codecs.put(QAT_DEFLATE_CODEC, new QatDeflate101Codec(compressionLevel, () -> {
                     return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING);
                 }));
             }
         } else {
-            codecs.put(ZSTD_CODEC, new Zstd912Codec(mapperService, logger, compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict912Codec(mapperService, logger, compressionLevel));
+            codecs.put(ZSTD_CODEC, new Zstd101Codec(mapperService, logger, compressionLevel));
+            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(mapperService, logger, compressionLevel));
             if (QatZipperFactory.isQatAvailable()) {
-                codecs.put(QAT_LZ4_CODEC, new QatLz4912Codec(mapperService, logger, compressionLevel, () -> {
+                codecs.put(QAT_LZ4_CODEC, new QatLz4101Codec(mapperService, logger, compressionLevel, () -> {
                     return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING);
                 }));
-                codecs.put(QAT_DEFLATE_CODEC, new QatDeflate912Codec(mapperService, logger, compressionLevel, () -> {
+                codecs.put(QAT_DEFLATE_CODEC, new QatDeflate101Codec(mapperService, logger, compressionLevel, () -> {
                     return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING);
                 }));
             }

src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java → src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java

@@ -11,7 +11,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.FilterCodec;
 import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
 import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
 import org.opensearch.index.mapper.MapperService;
 
@@ -23,22 +23,22 @@
  *
  * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec.
  * Supports two modes zstd and zstd_no_dict.
- * Uses Lucene912 as the delegate codec
+ * Uses Lucene101 as the delegate codec
  *
  * @opensearch.internal
  */
-public abstract class Lucene912CustomCodec extends FilterCodec {
+public abstract class Lucene101CustomCodec extends FilterCodec {
 
     /** Each mode represents a compression algorithm. */
     public enum Mode {
         /**
          * ZStandard mode with dictionary
          */
-        ZSTD("ZSTD912", Set.of("zstd")),
+        ZSTD("ZSTD101", Set.of("zstd")),
         /**
          * ZStandard mode without dictionary
          */
-        ZSTD_NO_DICT("ZSTDNODICT912", Set.of("zstd_no_dict"));
+        ZSTD_NO_DICT("ZSTDNODICT101", Set.of("zstd_no_dict"));
 
         private final String codec;
         private final Set<String> aliases;
@@ -70,7 +70,7 @@ public Set<String> getAliases() {
      *
      * @param mode The compression codec (ZSTD or ZSTDNODICT).
      */
-    public Lucene912CustomCodec(Mode mode) {
+    public Lucene101CustomCodec(Mode mode) {
         this(mode, DEFAULT_COMPRESSION_LEVEL);
     }
 
@@ -82,9 +82,9 @@ public Lucene912CustomCodec(Mode mode) {
      * @param mode The compression codec (ZSTD or ZSTDNODICT).
      * @param compressionLevel The compression level.
      */
-    public Lucene912CustomCodec(Mode mode, int compressionLevel) {
-        super(mode.getCodec(), new Lucene912Codec());
-        this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel);
+    public Lucene101CustomCodec(Mode mode, int compressionLevel) {
+        super(mode.getCodec(), new Lucene101Codec());
+        this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel);
     }
 
     /**
@@ -97,9 +97,9 @@ public Lucene912CustomCodec(Mode mode, int compressionLevel) {
      * @param mapperService The mapper service.
      * @param logger The logger.
      */
-    public Lucene912CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) {
-        super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger));
-        this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel);
+    public Lucene101CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) {
+        super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger));
+        this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel);
     }
 
     @Override

src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomStoredFieldsFormat.java

@@ -0,0 +1,140 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.customcodecs;
+
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.codecs.compressing.CompressionMode;
+import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL;
+
+/** Stored field format used by pluggable codec */
+public class Lucene101CustomStoredFieldsFormat extends StoredFieldsFormat {
+
+    /** A key that we use to map to a mode */
+    public static final String MODE_KEY = Lucene101CustomStoredFieldsFormat.class.getSimpleName() + ".mode";
+
+    protected static final int ZSTD_BLOCK_LENGTH = 10 * 48 * 1024;
+    protected static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096;
+    protected static final int ZSTD_BLOCK_SHIFT = 10;
+
+    private final CompressionMode zstdCompressionMode;
+    private final CompressionMode zstdNoDictCompressionMode;
+
+    private final Lucene101CustomCodec.Mode mode;
+    private final int compressionLevel;
+
+    /** default constructor */
+    public Lucene101CustomStoredFieldsFormat() {
+        this(Lucene101CustomCodec.Mode.ZSTD, DEFAULT_COMPRESSION_LEVEL);
+    }
+
+    /**
+     * Creates a new instance.
+     *
+     * @param mode The mode represents ZSTD or ZSTDNODICT
+     */
+    public Lucene101CustomStoredFieldsFormat(Lucene101CustomCodec.Mode mode) {
+        this(mode, DEFAULT_COMPRESSION_LEVEL);
+    }
+
+    /**
+     * Creates a new instance with the specified mode and compression level.
+     *
+     * @param mode The mode represents ZSTD or ZSTDNODICT
+     * @param compressionLevel The compression level for the mode.
+     */
+    public Lucene101CustomStoredFieldsFormat(Lucene101CustomCodec.Mode mode, int compressionLevel) {
+        this.mode = Objects.requireNonNull(mode);
+        this.compressionLevel = compressionLevel;
+        zstdCompressionMode = new ZstdCompressionMode(compressionLevel);
+        zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel);
+    }
+
+    /**
+     * Returns a {@link StoredFieldsReader} to load stored fields.
+     * @param directory The index directory.
+     * @param si The SegmentInfo that stores segment information.
+     * @param fn The fieldInfos.
+     * @param context The IOContext that holds additional details on the merge/search context.
+     */
+    @Override
+    public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
+        if (si.getAttribute(MODE_KEY) != null) {
+            String value = si.getAttribute(MODE_KEY);
+            Lucene101CustomCodec.Mode mode = Lucene101CustomCodec.Mode.valueOf(value);
+            return impl(mode).fieldsReader(directory, si, fn, context);
+        } else {
+            throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
+        }
+    }
+
+    /**
+     * Returns a {@link StoredFieldsReader} to write stored fields.
+     * @param directory The index directory.
+     * @param si The SegmentInfo that stores segment information.
+     * @param context The IOContext that holds additional details on the merge/search context.
+     */
+    @Override
+    public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException {
+        String previous = si.putAttribute(MODE_KEY, mode.name());
+        if (previous != null && previous.equals(mode.name()) == false) {
+            throw new IllegalStateException(
+                "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name()
+            );
+        }
+        return impl(mode).fieldsWriter(directory, si, context);
+    }
+
+    StoredFieldsFormat impl(Lucene101CustomCodec.Mode mode) {
+        switch (mode) {
+            case ZSTD:
+                return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode);
+            case ZSTD_NO_DICT:
+                return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode);
+            default:
+                throw new IllegalStateException("Unsupported compression mode: " + mode);
+        }
+    }
+
+    private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) {
+        return new Lucene90CompressingStoredFieldsFormat(
+            formatName,
+            compressionMode,
+            ZSTD_BLOCK_LENGTH,
+            ZSTD_MAX_DOCS_PER_BLOCK,
+            ZSTD_BLOCK_SHIFT
+        );
+    }
+
+    public Lucene101CustomCodec.Mode getMode() {
+        return mode;
+    }
+
+    /**
+     * Returns the compression level.
+     */
+    public int getCompressionLevel() {
+        return compressionLevel;
+    }
+
+    public CompressionMode getCompressionMode() {
+        return mode == Lucene101CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode;
+    }
+
+}