Introduce compression and mode mapping parms (#2019) (#2029)

Introduces new params for mapping and training, called compression_level
and mode. These parameters are high level parameters that give the
plugin a hint as to what the user wants to configure their system like
without exposing algorithmic details. This change just adds these
parameters to the plugin as noops. In future change, we will add the
functionality for parameter resolution.

Along with this, I added a class to more easily manage the original
parameters that a user passes. This will help ensure our mapper
maintains good compatibility.

Signed-off-by: John Mazanec <jmazane@amazon.com>
(cherry picked from commit 920c819)

Author: opensearch-trigger-bot[bot]

src/main/java/org/opensearch/knn/common/KNNConstants.java

@@ -71,6 +71,7 @@ public class KNNConstants {
     public static final String VECTOR_DATA_TYPE_FIELD = "data_type";
     public static final String MODEL_VECTOR_DATA_TYPE_KEY = VECTOR_DATA_TYPE_FIELD;
     public static final VectorDataType DEFAULT_VECTOR_DATA_TYPE_FIELD = VectorDataType.FLOAT;
+    public static final String MINIMAL_MODE_AND_COMPRESSION_FEATURE = "mode_and_compression_feature";
 
     public static final String RADIAL_SEARCH_KEY = "radial_search";
 
@@ -149,4 +150,7 @@ public class KNNConstants {
     public static final Float DEFAULT_LUCENE_RADIAL_SEARCH_TRAVERSAL_SIMILARITY_RATIO = 0.95f;
     public static final String MIN_SCORE = "min_score";
     public static final String MAX_DISTANCE = "max_distance";
+
+    public static final String MODE_PARAMETER = "mode";
+    public static final String COMPRESSION_LEVEL_PARAMETER = "compression_level";
 }

src/main/java/org/opensearch/knn/index/engine/KNNMethodConfigContext.java

@@ -7,10 +7,9 @@
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
+import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.Setter;
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.opensearch.Version;
 import org.opensearch.knn.index.VectorDataType;
 
@@ -23,29 +22,10 @@
 @Getter
 @Builder
 @AllArgsConstructor
+@EqualsAndHashCode
 public final class KNNMethodConfigContext {
     private VectorDataType vectorDataType;
     private Integer dimension;
     private Version versionCreated;
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) return true;
-        if (obj == null || getClass() != obj.getClass()) return false;
-        KNNMethodConfigContext other = (KNNMethodConfigContext) obj;
-
-        EqualsBuilder equalsBuilder = new EqualsBuilder();
-        equalsBuilder.append(vectorDataType, other.vectorDataType);
-        equalsBuilder.append(dimension, other.dimension);
-        equalsBuilder.append(versionCreated, other.versionCreated);
-
-        return equalsBuilder.isEquals();
-    }
-
-    @Override
-    public int hashCode() {
-        return new HashCodeBuilder().append(vectorDataType).append(dimension).append(versionCreated).toHashCode();
-    }
-
     public static final KNNMethodConfigContext EMPTY = KNNMethodConfigContext.builder().build();
 }

src/main/java/org/opensearch/knn/index/mapper/CompressionLevel.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.knn.index.mapper;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import org.opensearch.core.common.Strings;
+
+import java.util.Arrays;
+import java.util.Locale;
+import java.util.stream.Collectors;
+
+/**
+ * Enum representing the compression level for float vectors. Compression in this sense refers to compressing a
+ * full precision value into a smaller number of bits. For instance. "16x" compression would mean that 2 bits would
+ * need to be used to represent a 32-bit floating point number.
+ */
+@AllArgsConstructor
+public enum CompressionLevel {
+    NOT_CONFIGURED(-1, ""),
+    x1(1, "1x"),
+    x2(2, "2x"),
+    x4(4, "4x"),
+    x8(8, "8x"),
+    x16(16, "16x"),
+    x32(32, "32x");
+
+    // Internally, an empty string is easier to deal with them null. However, from the mapping,
+    // we do not want users to pass in the empty string and instead want null. So we make the conversion herex
+    static final String[] NAMES_ARRAY = Arrays.stream(CompressionLevel.values())
+        .map(compressionLevel -> compressionLevel == NOT_CONFIGURED ? null : compressionLevel.getName())
+        .collect(Collectors.toList())
+        .toArray(new String[0]);
+
+    /**
+     * Default is set to 1x and is a noop
+     */
+    private static final CompressionLevel DEFAULT = x1;
+
+    /**
+     * Get the compression level from a string representation. The format for the string should be "Nx", where N is
+     * the factor by which compression should take place
+     *
+     * @param name String representation of the compression level
+     * @return CompressionLevel enum value
+     */
+    public static CompressionLevel fromName(String name) {
+        if (Strings.isEmpty(name)) {
+            return NOT_CONFIGURED;
+        }
+        for (CompressionLevel config : CompressionLevel.values()) {
+            if (config.getName() != null && config.getName().equals(name)) {
+                return config;
+            }
+        }
+        throw new IllegalArgumentException(String.format(Locale.ROOT, "Invalid compression level: \"[%s]\"", name));
+    }
+
+    private final int compressionLevel;
+    @Getter
+    private final String name;
+
+    /**
+     * Gets the number of bits used to represent a float in order to achieve this compression. For instance, for
+     * 32x compression, each float would need to be encoded in a single bit.
+     *
+     * @return number of bits to represent a float at this compression level
+     */
+    public int numBitsForFloat32() {
+        if (this == NOT_CONFIGURED) {
+            return DEFAULT.numBitsForFloat32();
+        }
+
+        return (Float.BYTES * Byte.SIZE) / compressionLevel;
+    }
+
+    /**
+     * Utility method that checks if compression is configured.
+     *
+     * @param compressionLevel Compression to check
+     * @return true if compression is configured, false otherwise
+     */
+    public static boolean isConfigured(CompressionLevel compressionLevel) {
+        return compressionLevel != null && compressionLevel != NOT_CONFIGURED;
+    }
+}

src/main/java/org/opensearch/knn/index/mapper/FlatVectorFieldMapper.java

@@ -31,7 +31,8 @@ public static FlatVectorFieldMapper createFieldMapper(
         CopyTo copyTo,
         Explicit<Boolean> ignoreMalformed,
         boolean stored,
-        boolean hasDocValues
+        boolean hasDocValues,
+        OriginalMappingParameters originalMappingParameters
     ) {
         final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(
             fullname,
@@ -47,7 +48,8 @@ public static FlatVectorFieldMapper createFieldMapper(
             ignoreMalformed,
             stored,
             hasDocValues,
-            knnMethodConfigContext.getVersionCreated()
+            knnMethodConfigContext.getVersionCreated(),
+            originalMappingParameters
         );
     }
 
@@ -59,9 +61,20 @@ private FlatVectorFieldMapper(
         Explicit<Boolean> ignoreMalformed,
         boolean stored,
         boolean hasDocValues,
-        Version indexCreatedVersion
+        Version indexCreatedVersion,
+        OriginalMappingParameters originalMappingParameters
     ) {
-        super(simpleName, mappedFieldType, multiFields, copyTo, ignoreMalformed, stored, hasDocValues, indexCreatedVersion, null);
+        super(
+            simpleName,
+            mappedFieldType,
+            multiFields,
+            copyTo,
+            ignoreMalformed,
+            stored,
+            hasDocValues,
+            indexCreatedVersion,
+            originalMappingParameters
+        );
         // setting it explicitly false here to ensure that when flatmapper is used Lucene based Vector field is not created.
         this.useLuceneBasedVectorField = false;
         this.perDimensionValidator = selectPerDimensionValidator(vectorDataType);