Merge branch 'lucene-on-faiss' into lucene-on-faiss-part1

Signed-off-by: Doo Yong Kim <0ctopus13prime@gmail.com>

Author: 0ctopus13prime

CHANGELOG.md

@@ -15,11 +15,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Maintenance
 * Update package name to fix compilation issue [#2513](https://github.com/opensearch-project/k-NN/pull/2513)
 ### Refactoring
+* Small Refactor Post Lucene 10.0.1 upgrade [#2541](https://github.com/opensearch-project/k-NN/pull/2541)
+* Refactor codec to leverage backwards_codecs [#2546](https://github.com/opensearch-project/k-NN/pull/2546)
 
 ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.19...2.x)
 ### Features
 ### Enhancements
 ### Bug Fixes
+* Fix derived source for binary and byte vectors [#2533](https://github.com/opensearch-project/k-NN/pull/2533/)
+* Fix the put mapping issue for already created index with flat mapper [#2542](https://github.com/opensearch-project/k-NN/pull/2542)
 ### Infrastructure
 ### Documentation
 ### Maintenance

qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java

@@ -13,7 +13,10 @@
 import java.util.Locale;
 import java.util.Optional;
 
-import static org.opensearch.knn.TestUtils.*;
+import static org.opensearch.knn.TestUtils.BWC_VERSION;
+import static org.opensearch.knn.TestUtils.CLIENT_TIMEOUT_VALUE;
+import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX;
+import static org.opensearch.knn.TestUtils.RESTART_UPGRADE_OLD_CLUSTER;
 
 public abstract class AbstractRestartUpgradeTestCase extends KNNRestTestCase {
     protected String testIndex;
@@ -58,6 +61,7 @@ protected static final boolean isRunningAgainstOldCluster() {
         return Boolean.parseBoolean(System.getProperty(RESTART_UPGRADE_OLD_CLUSTER));
     }
 
+    @Override
     protected final Optional<String> getBWCVersion() {
         return Optional.ofNullable(System.getProperty(BWC_VERSION, null));
     }

qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRollingUpgradeTestCase.java

@@ -6,14 +6,21 @@
 package org.opensearch.knn.bwc;
 
 import org.junit.Before;
-import org.opensearch.knn.KNNRestTestCase;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.knn.KNNRestTestCase;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 
 import java.util.Locale;
 import java.util.Optional;
 
-import static org.opensearch.knn.TestUtils.*;
+import static org.opensearch.knn.TestUtils.BWCSUITE_CLUSTER;
+import static org.opensearch.knn.TestUtils.BWC_VERSION;
+import static org.opensearch.knn.TestUtils.CLIENT_TIMEOUT_VALUE;
+import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX;
+import static org.opensearch.knn.TestUtils.MIXED_CLUSTER;
+import static org.opensearch.knn.TestUtils.OLD_CLUSTER;
+import static org.opensearch.knn.TestUtils.ROLLING_UPGRADE_FIRST_ROUND;
+import static org.opensearch.knn.TestUtils.UPGRADED_CLUSTER;
 
 public abstract class AbstractRollingUpgradeTestCase extends KNNRestTestCase {
     protected String testIndex;
@@ -81,6 +88,7 @@ protected final boolean isFirstMixedRound() {
         return Boolean.parseBoolean(System.getProperty(ROLLING_UPGRADE_FIRST_ROUND, "false"));
     }
 
+    @Override
     protected final Optional<String> getBWCVersion() {
         return Optional.ofNullable(System.getProperty(BWC_VERSION, null));
     }

scripts/build.sh

@@ -174,7 +174,7 @@ else
    ompPath=$(ldconfig -p | grep libgomp | cut -d ' ' -f 4)
    cp -v $ompPath $distributions/lib
 fi
-cp -v ./jni/release/${libPrefix}* $distributions/lib
+cp -v ./jni/build/release/${libPrefix}* $distributions/lib
 ls -l $distributions/lib
 
 # Add lib directory to the k-NN plugin zip

src/main/java/org/opensearch/knn/common/featureflags/KNNFeatureFlags.java

@@ -26,6 +26,7 @@ public class KNNFeatureFlags {
 
     // Feature flags
     private static final String KNN_FORCE_EVICT_CACHE_ENABLED = "knn.feature.cache.force_evict.enabled";
+    private static final String KNN_REMOTE_VECTOR_BUILD = "knn.feature.remote_index_build.enabled";
 
     @VisibleForTesting
     public static final Setting<Boolean> KNN_FORCE_EVICT_CACHE_ENABLED_SETTING = Setting.boolSetting(
@@ -35,8 +36,18 @@ public class KNNFeatureFlags {
         Dynamic
     );
 
+    /**
+     * Feature flag to control remote index build at the cluster level
+     */
+    public static final Setting<Boolean> KNN_REMOTE_VECTOR_BUILD_SETTING = Setting.boolSetting(
+        KNN_REMOTE_VECTOR_BUILD,
+        false,
+        NodeScope,
+        Dynamic
+    );
+
     public static List<Setting<?>> getFeatureFlags() {
-        return ImmutableList.of(KNN_FORCE_EVICT_CACHE_ENABLED_SETTING);
+        return ImmutableList.of(KNN_FORCE_EVICT_CACHE_ENABLED_SETTING, KNN_REMOTE_VECTOR_BUILD_SETTING);
     }
 
     /**
@@ -46,4 +57,11 @@ public static List<Setting<?>> getFeatureFlags() {
     public static boolean isForceEvictCacheEnabled() {
         return Booleans.parseBoolean(KNNSettings.state().getSettingValue(KNN_FORCE_EVICT_CACHE_ENABLED).toString(), false);
     }
+
+    /**
+     * @return true if remote vector index build feature flag is enabled
+     */
+    public static boolean isKNNRemoteVectorBuildEnabled() {
+        return Booleans.parseBooleanStrict(KNNSettings.state().getSettingValue(KNN_REMOTE_VECTOR_BUILD).toString(), false);
+    }
 }

src/main/java/org/opensearch/knn/index/KNNSettings.java

@@ -41,9 +41,9 @@
 
 import static java.util.stream.Collectors.toUnmodifiableMap;
 import static org.opensearch.common.settings.Setting.Property.Dynamic;
+import static org.opensearch.common.settings.Setting.Property.Final;
 import static org.opensearch.common.settings.Setting.Property.IndexScope;
 import static org.opensearch.common.settings.Setting.Property.NodeScope;
-import static org.opensearch.common.settings.Setting.Property.Final;
 import static org.opensearch.common.settings.Setting.Property.UnmodifiableOnRestore;
 import static org.opensearch.common.unit.MemorySizeValue.parseBytesSizeValueOrHeapRatio;
 import static org.opensearch.core.common.unit.ByteSizeValue.parseBytesSizeValue;
@@ -94,6 +94,8 @@ public class KNNSettings {
     public static final String KNN_FAISS_AVX512_SPR_DISABLED = "knn.faiss.avx512_spr.disabled";
     public static final String KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED = "index.knn.disk.vector.shard_level_rescoring_disabled";
     public static final String KNN_DERIVED_SOURCE_ENABLED = "index.knn.derived_source.enabled";
+    public static final String KNN_INDEX_REMOTE_VECTOR_BUILD = "index.knn.remote_index_build.enabled";
+    public static final String KNN_REMOTE_VECTOR_REPO = "knn.remote_index_build.vector_repo";
 
     /**
      * Default setting values
@@ -371,6 +373,21 @@ public class KNNSettings {
         NodeScope
     );
 
+    /**
+     * Index level setting to control whether remote index build is enabled or not.
+     */
+    public static final Setting<Boolean> KNN_INDEX_REMOTE_VECTOR_BUILD_SETTING = Setting.boolSetting(
+        KNN_INDEX_REMOTE_VECTOR_BUILD,
+        false,
+        Dynamic,
+        IndexScope
+    );
+
+    /**
+     * Cluster level setting which indicates the repository that the remote index build should write to.
+     */
+    public static final Setting<String> KNN_REMOTE_VECTOR_REPO_SETTING = Setting.simpleString(KNN_REMOTE_VECTOR_REPO, Dynamic, NodeScope);
+
     /**
      * Dynamic settings
      */
@@ -525,6 +542,14 @@ private Setting<?> getSetting(String key) {
             return KNN_DERIVED_SOURCE_ENABLED_SETTING;
         }
 
+        if (KNN_INDEX_REMOTE_VECTOR_BUILD.equals(key)) {
+            return KNN_INDEX_REMOTE_VECTOR_BUILD_SETTING;
+        }
+
+        if (KNN_REMOTE_VECTOR_REPO.equals(key)) {
+            return KNN_REMOTE_VECTOR_REPO_SETTING;
+        }
+
         throw new IllegalArgumentException("Cannot find setting by key [" + key + "]");
     }
 
@@ -550,7 +575,9 @@ public List<Setting<?>> getSettings() {
             QUANTIZATION_STATE_CACHE_SIZE_LIMIT_SETTING,
             QUANTIZATION_STATE_CACHE_EXPIRY_TIME_MINUTES_SETTING,
             KNN_DISK_VECTOR_SHARD_LEVEL_RESCORING_DISABLED_SETTING,
-            KNN_DERIVED_SOURCE_ENABLED_SETTING
+            KNN_DERIVED_SOURCE_ENABLED_SETTING,
+            KNN_INDEX_REMOTE_VECTOR_BUILD_SETTING,
+            KNN_REMOTE_VECTOR_REPO_SETTING
         );
         return Stream.concat(settings.stream(), Stream.concat(getFeatureFlags().stream(), dynamicCacheSettings.values().stream()))
             .collect(Collectors.toList());

src/main/java/org/opensearch/knn/index/KNNVectorScriptDocValues.java

@@ -74,26 +74,33 @@ public float[] get(int i) {
     /**
      * Creates a KNNVectorScriptDocValues object based on the provided parameters.
      *
-     * @param values          The DocIdSetIterator representing the vector values.
+     * @param knnVectorValues          The DocIdSetIterator representing the vector values.
      * @param fieldName       The name of the field.
      * @param vectorDataType  The data type of the vector.
      * @return A KNNVectorScriptDocValues object based on the type of the values.
      * @throws IllegalArgumentException If the type of values is unsupported.
      */
-    public static KNNVectorScriptDocValues create(Object values, String fieldName, VectorDataType vectorDataType) {
-        Objects.requireNonNull(values, "values must not be null");
-
-        if (values instanceof FloatVectorValues) {
-            return new KNNFloatVectorScriptDocValues((FloatVectorValues) values, fieldName, vectorDataType);
-        } else if (values instanceof ByteVectorValues) {
-            return new KNNByteVectorScriptDocValues((ByteVectorValues) values, fieldName, vectorDataType);
-        } else if (values instanceof BinaryDocValues) {
-            return new KNNNativeVectorScriptDocValues((BinaryDocValues) values, fieldName, vectorDataType);
+    public static KNNVectorScriptDocValues create(KnnVectorValues knnVectorValues, String fieldName, VectorDataType vectorDataType) {
+        Objects.requireNonNull(knnVectorValues, "values must not be null");
+        if (knnVectorValues instanceof FloatVectorValues) {
+            return new KNNFloatVectorScriptDocValues((FloatVectorValues) knnVectorValues, fieldName, vectorDataType);
+        } else if (knnVectorValues instanceof ByteVectorValues) {
+            return new KNNByteVectorScriptDocValues((ByteVectorValues) knnVectorValues, fieldName, vectorDataType);
         } else {
-            throw new IllegalArgumentException("Unsupported values type: " + values.getClass());
+            throw new IllegalArgumentException("Unsupported values type: " + knnVectorValues.getClass());
         }
     }
 
+    public static KNNVectorScriptDocValues create(DocIdSetIterator docIdSetIterator, String fieldName, VectorDataType vectorDataType) {
+        Objects.requireNonNull(docIdSetIterator, "values must not be null");
+        if (docIdSetIterator instanceof BinaryDocValues) {
+            return new KNNNativeVectorScriptDocValues((BinaryDocValues) docIdSetIterator, fieldName, vectorDataType);
+        } else {
+            throw new IllegalArgumentException("Unsupported values type: " + docIdSetIterator.getClass());
+        }
+
+    }
+
     private static final class KNNByteVectorScriptDocValues extends KNNVectorScriptDocValues {
         private final ByteVectorValues values;
         private final KnnVectorValues.DocIndexIterator iterator;

src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java

@@ -15,6 +15,7 @@
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.knn.index.KNNSettings;
 import org.opensearch.knn.index.codec.KNN990Codec.NativeEngines990KnnVectorsFormat;
+import org.opensearch.knn.index.codec.nativeindex.NativeIndexBuildStrategyFactory;
 import org.opensearch.knn.index.codec.params.KNNScalarQuantizedVectorsFormatParams;
 import org.opensearch.knn.index.codec.params.KNNVectorsFormatParams;
 import org.opensearch.knn.index.engine.KNNEngine;
@@ -44,6 +45,7 @@ public abstract class BasePerFieldKnnVectorsFormat extends PerFieldKnnVectorsFor
     private final Supplier<KnnVectorsFormat> defaultFormatSupplier;
     private final Function<KNNVectorsFormatParams, KnnVectorsFormat> vectorsFormatSupplier;
     private Function<KNNScalarQuantizedVectorsFormatParams, KnnVectorsFormat> scalarQuantizedVectorsFormatSupplier;
+    private final NativeIndexBuildStrategyFactory nativeIndexBuildStrategyFactory;
     private static final String MAX_CONNECTIONS = "max_connections";
     private static final String BEAM_WIDTH = "beam_width";
 
@@ -54,11 +56,26 @@ public BasePerFieldKnnVectorsFormat(
         Supplier<KnnVectorsFormat> defaultFormatSupplier,
         Function<KNNVectorsFormatParams, KnnVectorsFormat> vectorsFormatSupplier
     ) {
-        this.mapperService = mapperService;
-        this.defaultMaxConnections = defaultMaxConnections;
-        this.defaultBeamWidth = defaultBeamWidth;
-        this.defaultFormatSupplier = defaultFormatSupplier;
-        this.vectorsFormatSupplier = vectorsFormatSupplier;
+        this(mapperService, defaultMaxConnections, defaultBeamWidth, defaultFormatSupplier, vectorsFormatSupplier, null);
+    }
+
+    public BasePerFieldKnnVectorsFormat(
+        Optional<MapperService> mapperService,
+        int defaultMaxConnections,
+        int defaultBeamWidth,
+        Supplier<KnnVectorsFormat> defaultFormatSupplier,
+        Function<KNNVectorsFormatParams, KnnVectorsFormat> vectorsFormatSupplier,
+        Function<KNNScalarQuantizedVectorsFormatParams, KnnVectorsFormat> scalarQuantizedVectorsFormatSupplier
+    ) {
+        this(
+            mapperService,
+            defaultMaxConnections,
+            defaultBeamWidth,
+            defaultFormatSupplier,
+            vectorsFormatSupplier,
+            scalarQuantizedVectorsFormatSupplier,
+            new NativeIndexBuildStrategyFactory()
+        );
     }
 
     @Override
@@ -141,7 +158,8 @@ private NativeEngines990KnnVectorsFormat nativeEngineVectorsFormat() {
         int approximateThreshold = getApproximateThresholdValue();
         return new NativeEngines990KnnVectorsFormat(
             new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer()),
-            approximateThreshold
+            approximateThreshold,
+            nativeIndexBuildStrategyFactory
         );
     }
 

src/main/java/org/opensearch/knn/index/codec/KNN10010Codec/KNN10010Codec.java

@@ -12,31 +12,37 @@
 import org.apache.lucene.codecs.FilterCodec;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
 import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
 import org.opensearch.index.mapper.MapperService;
+import org.opensearch.knn.index.codec.KNN80Codec.KNN80CompoundFormat;
+import org.opensearch.knn.index.codec.KNN80Codec.KNN80DocValuesFormat;
 import org.opensearch.knn.index.codec.KNN9120Codec.DerivedSourceStoredFieldsFormat;
-import org.opensearch.knn.index.codec.KNNCodecVersion;
-import org.opensearch.knn.index.codec.KNNFormatFacade;
+import org.opensearch.knn.index.codec.KNN9120Codec.KNN9120PerFieldKnnVectorsFormat;
 import org.opensearch.knn.index.codec.derivedsource.DerivedSourceReadersSupplier;
 
+import java.util.Optional;
+
 /**
  * KNN Codec that wraps the Lucene Codec which is part of Lucene 10.0.1
  */
 
 public class KNN10010Codec extends FilterCodec {
 
-    private static final KNNCodecVersion VERSION = KNNCodecVersion.V_10_01_0;
-    private final KNNFormatFacade knnFormatFacade;
+    private static final String NAME = "KNN10010Codec";
+    public static final Codec DEFAULT_DELEGATE = new Lucene101Codec();
+    private static final PerFieldKnnVectorsFormat DEFAULT_KNN_VECTOR_FORMAT = new KNN9120PerFieldKnnVectorsFormat(Optional.empty());
+
     private final PerFieldKnnVectorsFormat perFieldKnnVectorsFormat;
     private final StoredFieldsFormat storedFieldsFormat;
 
     private final MapperService mapperService;
 
     /**
-     * No arg constructor that uses Lucene99 as the delegate
+     * No arg constructor that uses Lucene101Codec as the delegate
      */
     public KNN10010Codec() {
-        this(VERSION.getDefaultCodecDelegate(), VERSION.getPerFieldKnnVectorsFormat(), null);
+        this(DEFAULT_DELEGATE, DEFAULT_KNN_VECTOR_FORMAT, null);
     }
 
     /**
@@ -47,22 +53,21 @@ public KNN10010Codec() {
      * @param knnVectorsFormat per field format for KnnVector
      */
     @Builder
-    protected KNN10010Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat, MapperService mapperService) {
-        super(VERSION.getCodecName(), delegate);
-        knnFormatFacade = VERSION.getKnnFormatFacadeSupplier().apply(delegate);
+    public KNN10010Codec(Codec delegate, PerFieldKnnVectorsFormat knnVectorsFormat, MapperService mapperService) {
+        super(NAME, delegate);
         perFieldKnnVectorsFormat = knnVectorsFormat;
         this.mapperService = mapperService;
         this.storedFieldsFormat = getStoredFieldsFormat();
     }
 
     @Override
     public DocValuesFormat docValuesFormat() {
-        return knnFormatFacade.docValuesFormat();
+        return new KNN80DocValuesFormat(delegate.docValuesFormat());
     }
 
     @Override
     public CompoundFormat compoundFormat() {
-        return knnFormatFacade.compoundFormat();
+        return new KNN80CompoundFormat(delegate.compoundFormat());
     }
 
     @Override