opensearch-project · mch2 · Aug 19, 2024 · Sep 12, 2024 · Sep 13, 2024
@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.12.0-snapshot-847316d
+lucene            = 9.12.0-snapshot-b08252b
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.4+7

@@ -0,0 +1 @@
+34b51d52a7766009ec90f8a3131378af43217c68
@@ -0,0 +1 @@
+37687e2c16b1ac93212459f9e5f31dea77d2dfbd
@@ -0,0 +1 @@
+2fdf36e849b93d0cd118fe8748bc005163c7620e
@@ -0,0 +1 @@
+718125a03a268e1da29076b7723e4fb6fa64add2
@@ -0,0 +1 @@
+8e54ada808f6e2ad7819e76cfed61de3dbc0fa5d
@@ -0,0 +1 @@
+e02bc647fd0711c780ab168535edda3f23353b2e
@@ -0,0 +1 @@
+c67f51af3a2fe15f14c9213a5ed7c1729b0c09b8
@@ -0,0 +1 @@
+98be510a2a532cfa1925bb3d020081729327c87d
@@ -0,0 +1 @@
+3439ff905d2e6802b6cfaed64716d3a662318b95
@@ -8,8 +8,8 @@
 
 package org.opensearch.plugin.correlation.core.index.codec;
 
+import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.plugin.correlation.core.index.codec.correlation990.CorrelationCodec;
 import org.opensearch.plugin.correlation.core.index.codec.correlation990.PerFieldCorrelationVectorsFormat;

@@ -0,0 +1 @@
+bd9e9c853c84d89af78c1cf925a306325d1b3259
@@ -0,0 +1 @@
+6c1d08db8f529c86416d08ab8f59dd54fcb7c41c
@@ -0,0 +1 @@
+34b51d52a7766009ec90f8a3131378af43217c68
@@ -0,0 +1 @@
+d897011a81a55d3d187173be61543b0415a7ee8a
@@ -0,0 +1 @@
+31829ce858f49b0c25604283e279d49257e75ae2
@@ -0,0 +1 @@
+253ff705cb15668c6553af903ffebb732711f7ed
@@ -0,0 +1 @@
+f8d36e21c27ba918fb25ab11abdf45737614923b
@@ -0,0 +1 @@
+09d591c82fb12e19c11629f030598ab7b5d4b142
@@ -0,0 +1 @@
+f1e7f5f4ef77d6334d45dd501a97f008961d3603
@@ -0,0 +1 @@
+f07ca34ccd9cee8fa5afffa6cc07169657b05f31
@@ -0,0 +1 @@
+ece014538e60b6beac07061b2cfd6d210ca2922d
@@ -0,0 +1 @@
+27ab6d42edb73fbfdf08f7525d67d9d5b4152c87
@@ -0,0 +1 @@
+8be205531dcaee072c5d9a980266f82560278bf7
@@ -0,0 +1 @@
+cc727ad7086288417f06ac891ec49289c676b85c
@@ -87,15 +87,7 @@ public int prevSetBit(int index) {
 
     @Override
     public int nextSetBit(int index) {
-        assert index >= 0 && index < length : "index=" + index + " numBits=" + length();
-        int next = first.nextSetBit(index);
-        while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) {
-            if (next == length() - 1) {
-                return DocIdSetIterator.NO_MORE_DOCS;
-            }
-            next = first.nextSetBit(next + 1);
-        }
-        return next;
+        return nextSetBit(index, length() - 1);
     }
 
     @Override
@@ -132,4 +124,20 @@ public void clear(int startIndex, int endIndex) {
     public boolean getAndSet(int i) {
         throw new UnsupportedOperationException("not implemented");
     }
+
+    @Override
+    public int nextSetBit(int start, int end) {
+        assert start >= 0 && start < length() : "start=" + start + " numBits=" + length();
+        assert end >= start && end < length() : "end=" + end + " numBits=" + length();
+
+        int next = first.nextSetBit(start);
+        while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) {
+            if (next >= end) {
+                return DocIdSetIterator.NO_MORE_DOCS;
+            }
+            next = first.nextSetBit(next + 1);
+        }
+        return next;
+
+    }
 }
@@ -110,7 +110,7 @@
  * @opensearch.internal
  */
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene99";
+    public static final String LATEST_CODEC = "Lucene912";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 
@@ -272,7 +272,7 @@ public static void checkSegmentInfoIntegrity(final Directory directory) throws I
 
             @Override
             protected Object doBody(String segmentFileName) throws IOException {
-                try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) {
+                try (IndexInput input = directory.openInput(segmentFileName, IOContext.READONCE)) {
                     CodecUtil.checksumEntireFile(input);
                 }
                 return null;

@@ -34,8 +34,8 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec.Mode;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec.Mode;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.IndexSettings;
@@ -70,10 +70,10 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         assert null != indexSettings;
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene99Codec());
-            codecs.put(LZ4, new Lucene99Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene99Codec(Mode.BEST_COMPRESSION));
-            codecs.put(ZLIB, new Lucene99Codec(Mode.BEST_COMPRESSION));
+            codecs.put(DEFAULT_CODEC, new Lucene912Codec());
+            codecs.put(LZ4, new Lucene912Codec());
+            codecs.put(BEST_COMPRESSION_CODEC, new Lucene912Codec(Mode.BEST_COMPRESSION));
+            codecs.put(ZLIB, new Lucene912Codec(Mode.BEST_COMPRESSION));
         } else {
             // CompositeCodec still delegates to PerFieldMappingPostingFormatCodec
             // We can still support all the compression codecs when composite index is present

@@ -37,7 +37,7 @@
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.index.codec.fuzzy.FuzzyFilterPostingsFormat;
 import org.opensearch.index.codec.fuzzy.FuzzySetFactory;
@@ -59,7 +59,7 @@
  *
  * @opensearch.internal
  */
-public class PerFieldMappingPostingFormatCodec extends Lucene99Codec {
+public class PerFieldMappingPostingFormatCodec extends Lucene912Codec {
     private final Logger logger;
     private final MapperService mapperService;
     private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();

@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
+import org.opensearch.index.codec.composite.composite99.Composite99DocValuesFormat;
+import org.opensearch.index.mapper.MapperService;
+
+/**
+ *  Extends the Codec to support new file formats for composite indices eg: star tree index
+ *  based on the mappings.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class Composite912Codec extends FilterCodec {
+    public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite912Codec";
+    private final MapperService mapperService;
+
+    // needed for SPI - this is used in reader path
+    public Composite912Codec() {
+        this(COMPOSITE_INDEX_CODEC_NAME, new Lucene912Codec(), null);
+    }
+
+    public Composite912Codec(Lucene912Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
+        this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService);
+    }
+
+    /**
+     * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to
+     * this ctor.
+     *
+     * @param name name of the codec
+     * @param delegate codec delegate
+     * @param mapperService mapper service instance
+     */
+    protected Composite912Codec(String name, Codec delegate, MapperService mapperService) {
+        super(name, delegate);
+        this.mapperService = mapperService;
+    }
+
+    @Override
+    public DocValuesFormat docValuesFormat() {
+        return new Composite99DocValuesFormat(mapperService);
+    }
+}
@@ -10,7 +10,7 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.codec.composite.composite99.Composite99Codec;
 import org.opensearch.index.mapper.MapperService;
@@ -38,10 +38,10 @@ public CompositeCodecFactory() {}
 
     public Map<String, Codec> getCompositeIndexCodecs(MapperService mapperService, Logger logger) {
         Map<String, Codec> codecs = new HashMap<>();
-        codecs.put(DEFAULT_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger));
-        codecs.put(LZ4, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger));
-        codecs.put(BEST_COMPRESSION_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger));
-        codecs.put(ZLIB, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+        codecs.put(DEFAULT_CODEC, new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger));
+        codecs.put(LZ4, new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger));
+        codecs.put(BEST_COMPRESSION_CODEC, new Composite912Codec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+        codecs.put(ZLIB, new Composite912Codec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, logger));
         return codecs;
     }
 }
@@ -8,13 +8,11 @@
 
 package org.opensearch.index.codec.composite.composite99;
 
-import org.apache.logging.log4j.Logger;
+import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
 import org.opensearch.common.annotation.ExperimentalApi;
-import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
 import org.opensearch.index.mapper.MapperService;
 
 /**
@@ -33,10 +31,6 @@ public Composite99Codec() {
         this(COMPOSITE_INDEX_CODEC_NAME, new Lucene99Codec(), null);
     }
 
-    public Composite99Codec(Lucene99Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
-        this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService);
-    }
-
     /**
      * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to
      * this ctor.

@@ -5214,7 +5214,7 @@ public void syncSegmentsFromGivenRemoteSegmentStore(
             } else if (segmentsNFile != null) {
                 try (
                     ChecksumIndexInput indexInput = new BufferedChecksumIndexInput(
-                        storeDirectory.openInput(segmentsNFile, IOContext.DEFAULT)
+                        storeDirectory.openInput(segmentsNFile, IOContext.READONCE)
                     )
                 ) {
                     long commitGeneration = SegmentInfos.generationFromSegmentsFileName(segmentsNFile);
@@ -5288,7 +5288,7 @@ private String copySegmentFiles(
 
     // Visible for testing
     boolean localDirectoryContains(Directory localDirectory, String file, long checksum) throws IOException {
-        try (IndexInput indexInput = localDirectory.openInput(file, IOContext.DEFAULT)) {
+        try (IndexInput indexInput = localDirectory.openInput(file, IOContext.READONCE)) {
             if (checksum == CodecUtil.retrieveChecksum(indexInput)) {
                 return true;
             } else {

@@ -488,7 +488,7 @@ private boolean skipUpload(String file) {
 
     private String getChecksumOfLocalFile(String file) throws IOException {
         if (!localSegmentChecksumMap.containsKey(file)) {
-            try (IndexInput indexInput = storeDirectory.openInput(file, IOContext.DEFAULT)) {
+            try (IndexInput indexInput = storeDirectory.openInput(file, IOContext.READONCE)) {
                 String checksum = Long.toString(CodecUtil.retrieveChecksum(indexInput));
                 localSegmentChecksumMap.put(file, checksum);
             }

@@ -759,7 +759,7 @@ private void tryAndDeleteLocalFile(String filename, Directory directory) {
     }
 
     private String getChecksumOfLocalFile(Directory directory, String file) throws IOException {
-        try (IndexInput indexInput = directory.openInput(file, IOContext.DEFAULT)) {
+        try (IndexInput indexInput = directory.openInput(file, IOContext.READONCE)) {
             return Long.toString(CodecUtil.retrieveChecksum(indexInput));
         }
     }