Bumping custom-codecs plugin version to 3.0.0-alpha1 and lucene upgrade

.github/actions/create-bwc-build/action.yaml

@@ -25,7 +25,7 @@ runs:
       with:
         path: ${{ inputs.plugin-branch }}
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       if: ${{ inputs.plugin-branch != 'current_branch' }}
       with:
         repository: opensearch-project/custom-codecs
@@ -39,6 +39,7 @@ runs:
         arguments: assemble
         build-root-directory: ${{ inputs.plugin-branch }}
 
+
     - id: get-opensearch-version
       uses: peternied/get-opensearch-version@v1
       with:

.github/actions/run-bwc-suite/action.yaml

@@ -49,7 +49,7 @@ runs:
           -Dbwc.version.previous=${{ steps.build-previous.outputs.built-version }}
           -Dbwc.version.next=${{ steps.build-next.outputs.built-version }} -i
 
-    - uses: alehechka/upload-tartifact@v2
+    - uses: actions/upload-artifact@v4
       if: always()
       with:
         name: ${{ inputs.report-artifact-name }}

.github/workflows/check.yml

@@ -44,7 +44,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin

.github/workflows/ci.yml

@@ -25,18 +25,14 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Build BWC tests
-        uses: gradle/gradle-build-action@v3
-        with:
-          cache-disabled: true
-          arguments: |
-            -p bwc-test build -x test -x integTest
+        run: ./gradlew -p bwc-test build -x test -x integTest
 
   backward-compatibility:
     strategy:
       fail-fast: false
       matrix:
         jdk: [21]
-        platform: [ubuntu-latest, windows-latest]
+        platform: [ ubuntu-latest, windows-latest ]
     runs-on: ${{ matrix.platform }}
 
     steps:

.github/workflows/publish-maven-snapshots.yml

@@ -17,11 +17,11 @@ jobs:
       contents: write
 
     steps:
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: temurin
           java-version: 21
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: aws-actions/configure-aws-credentials@v1
         with:
           role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}

build.gradle

@@ -31,9 +31,9 @@
 buildscript {
   ext {
     opensearch_group = "org.opensearch"
-    opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
+    opensearch_version = System.getProperty("opensearch.version", "3.0.0-alpha1-SNAPSHOT")
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
-    buildVersionQualifier = System.getProperty("build.version_qualifier", "")
+    buildVersionQualifier = System.getProperty("build.version_qualifier", "alpha1")
   }
 
   repositories {

bwc-test/build.gradle

@@ -44,7 +44,7 @@ ext {
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.0.0-alpha1-SNAPSHOT")
         opensearch_group = "org.opensearch"
     }
     repositories {
@@ -84,15 +84,11 @@ String baseName = "customCodecsBwcCluster"
 String bwcFilePath = "src/test/resources/"
 String projectVersion = nextVersion
 
-String previousOpenSearch  = extractVersion(previousVersion) + "-SNAPSHOT";
-String nextOpenSearch  = extractVersion(nextVersion) + "-SNAPSHOT";
-
 // Extracts the OpenSearch version from a plugin version string, 2.11.0.0 -> 2.11.0.
-def String extractVersion(versionStr) {
-    def versionMatcher =  versionStr =~ /(.+?)(\.\d+)$/
-    versionMatcher.find()
-    return versionMatcher.group(1)
-}
+def previousToken = previousVersion.tokenize('-')
+def nextToken = nextVersion.tokenize('-')
+String previousOpenSearch = previousToken[1] ? previousToken[0][0..-3] + "-" + previousToken[1] + "-SNAPSHOT" : previousToken[0][0..-3] + "-SNAPSHOT"
+String nextOpenSearch = nextToken[1] ? nextToken[0][0..-3] + "-" + nextToken[1] + "-SNAPSHOT" : nextToken[0][0..-3] + "-SNAPSHOT"
 
 2.times {i ->
     testClusters {

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,8 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
+distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
+networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=5b9c5eb3f9fc2c94abaea57d90bd78747ca117ddbbf96c859d3741181a12bf2a

src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java

@@ -26,8 +26,8 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.Strings;
-import org.opensearch.index.codec.customcodecs.Lucene912QatCodec;
 import org.opensearch.index.codec.customcodecs.QatZipperFactory;
+import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 
 import javax.net.ssl.SSLEngine;

src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java

@@ -11,6 +11,7 @@
 import org.opensearch.common.settings.Setting;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.codec.CodecServiceFactory;
+import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Lucene912QatCodec;
 import org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec;
 import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.plugins.EnginePlugin;

src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java

@@ -13,6 +13,10 @@
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatDeflate912Codec;
+import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.QatLz4912Codec;
+import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.Zstd912Codec;
+import org.opensearch.index.codec.customcodecs.backward_codecs.lucene912.ZstdNoDict912Codec;
 import org.opensearch.index.mapper.MapperService;
 
 import java.util.Arrays;

src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java

@@ -0,0 +1,114 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.customcodecs;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
+import org.opensearch.index.mapper.MapperService;
+
+import java.util.Set;
+
+import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL;
+
+/**
+ *
+ * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec.
+ * Supports two modes zstd and zstd_no_dict.
+ * Uses Lucene101 as the delegate codec
+ *
+ * @opensearch.internal
+ */
+public abstract class Lucene101CustomCodec extends FilterCodec {
+
+    /** Each mode represents a compression algorithm. */
+    public enum Mode {
+        /**
+         * ZStandard mode with dictionary
+         */
+        ZSTD("ZSTD101", Set.of("zstd")),
+        /**
+         * ZStandard mode without dictionary
+         */
+        ZSTD_NO_DICT("ZSTDNODICT101", Set.of("zstd_no_dict"));
+
+        private final String codec;
+        private final Set<String> aliases;
+
+        Mode(String codec, Set<String> aliases) {
+            this.codec = codec;
+            this.aliases = aliases;
+        }
+
+        /**
+         * Returns the Codec that is registered with Lucene
+         */
+        public String getCodec() {
+            return codec;
+        }
+
+        /**
+         * Returns the aliases of the Codec
+         */
+        public Set<String> getAliases() {
+            return aliases;
+        }
+    }
+
+    private final StoredFieldsFormat storedFieldsFormat;
+
+    /**
+     * Creates a new compression codec with the default compression level.
+     *
+     * @param mode The compression codec (ZSTD or ZSTDNODICT).
+     */
+    public Lucene101CustomCodec(Mode mode) {
+        this(mode, DEFAULT_COMPRESSION_LEVEL);
+    }
+
+    /**
+     * Creates a new compression codec with the given compression level. We use
+     * lowercase letters when registering the codec so that we remain consistent with
+     * the other compression codecs: default, lucene_default, and best_compression.
+     *
+     * @param mode The compression codec (ZSTD or ZSTDNODICT).
+     * @param compressionLevel The compression level.
+     */
+    public Lucene101CustomCodec(Mode mode, int compressionLevel) {
+        super(mode.getCodec(), new Lucene101Codec());
+        this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel);
+    }
+
+    /**
+     * Creates a new compression codec with the given compression level. We use
+     * lowercase letters when registering the codec so that we remain consistent with
+     * the other compression codecs: default, lucene_default, and best_compression.
+     *
+     * @param mode The compression codec (ZSTD or ZSTDNODICT).
+     * @param compressionLevel The compression level.
+     * @param mapperService The mapper service.
+     * @param logger The logger.
+     */
+    public Lucene101CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) {
+        super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger));
+        this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel);
+    }
+
+    @Override
+    public StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName();
+    }
+}