Merge remote-tracking branch 'upstream/main' into pipeline

Author: gaobinlong

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.10.0
+lucene            = 9.11.0-snapshot-8a555eb
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.2+13

libs/core/licenses/lucene-core-9.10.0.jar.sha1

@@ -0,0 +1 @@
+fe1cf5663be8bdb6aa757fd4101bc551684c90fb

libs/core/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1

@@ -101,7 +101,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_2);
     public static final Version V_2_12_1 = new Version(2120199, org.apache.lucene.util.Version.LUCENE_9_9_2);
     public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0);
-    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_10_0);
+    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0);
     public static final Version CURRENT = V_3_0_0;
 
     public static Version fromId(int id) {

libs/core/src/main/java/org/opensearch/Version.java

@@ -0,0 +1 @@
+00759eaff8f62b38ba66a05f26ab784c268908d3

modules/lang-expression/licenses/lucene-expressions-9.10.0.jar.sha1

@@ -0,0 +1 @@
+44a4e095d7e047a9452d81b224905b72c830f8ae

modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+7f57fe7322e6d3a9b4edcc3da0b1ee0791a814ec

plugins/analysis-icu/licenses/lucene-analysis-icu-9.10.0.jar.sha1

@@ -0,0 +1 @@
+9929da235100f8df323cfed165b8111fb2840093

plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+8564c86d880c6ce002250002e2fd0936cbfff61d

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.10.0.jar.sha1

@@ -0,0 +1 @@
+7e71777cfb5beb4ffd5b03030576d2f062eef13c

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+a597265bd6fb0a7e954e948a295d31507dd73cce

plugins/analysis-nori/licenses/lucene-analysis-nori-9.10.0.jar.sha1

@@ -0,0 +1 @@
+c9e534845bb08985d7fa21e2e71a14bc68c46089

plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+6f0cfa07a5e4b36423e398cd1fd51c6825773d9c

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.10.0.jar.sha1

@@ -0,0 +1 @@
+ab201b997c8449db1ecd2fa88bd42d2f457286fa

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+fe1cf5663be8bdb6aa757fd4101bc551684c90fb

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.10.0.jar.sha1

@@ -0,0 +1 @@
+a988f92842e48195c75a49377432533c9170d93d

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+7c669e2c01565d3bdf175cd61a1e4d0bdfc44311

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.10.0.jar.sha1

@@ -0,0 +1 @@
+633a6d44b4cde8e149daa3407e8b8f644eece951

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+25390259c9e5592354efbc2f250bb396402016b2

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.10.0.jar.sha1

@@ -0,0 +1 @@
+056f87a3d30c223b08d2f45fe465ddf11210b85f

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+e640f850b4fb13190be8422fe74c14c9d6603bb5

server/licenses/lucene-analysis-common-9.10.0.jar.sha1

@@ -0,0 +1 @@
+8eb57762bf408fa51d7511f5e3b917627be61d1d

server/licenses/lucene-analysis-common-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+615b4a90c5402084c2d5916a4c1fadc9d9177782

server/licenses/lucene-backward-codecs-9.10.0.jar.sha1

@@ -0,0 +1 @@
+19b42cdb5f76f63dece3ef5128207ebdd3741d48

server/licenses/lucene-backward-codecs-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+882691fe917e716fe6bcf8c0dd984b153495d015

server/licenses/lucene-core-9.10.0.jar.sha1

@@ -0,0 +1 @@
+ef6d483960f776d5dbdd1009863786ee09ba5707

server/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1

@@ -85,10 +85,10 @@ protected Settings.Builder randomRepositorySettings() {
         return settings;
     }
 
-    private Settings.Builder chunkedRepositorySettings() {
+    private Settings.Builder chunkedRepositorySettings(long chunkSize) {
         final Settings.Builder settings = Settings.builder();
         settings.put("location", randomRepoPath()).put("compress", randomBoolean());
-        settings.put("chunk_size", 2 << 23, ByteSizeUnit.BYTES);
+        settings.put("chunk_size", chunkSize, ByteSizeUnit.BYTES);
         return settings;
     }
 
@@ -194,18 +194,44 @@ public void testSnapshottingSearchableSnapshots() throws Exception {
     }
 
     /**
-     * Tests a chunked repository scenario for searchable snapshots by creating an index,
+     * Tests a default 8mib chunked repository scenario for searchable snapshots by creating an index,
      * taking a snapshot, restoring it as a searchable snapshot index.
      */
-    public void testCreateSearchableSnapshotWithChunks() throws Exception {
+    public void testCreateSearchableSnapshotWithDefaultChunks() throws Exception {
         final int numReplicasIndex = randomIntBetween(1, 4);
         final String indexName = "test-idx";
         final String restoredIndexName = indexName + "-copy";
         final String repoName = "test-repo";
         final String snapshotName = "test-snap";
         final Client client = client();
 
-        Settings.Builder repositorySettings = chunkedRepositorySettings();
+        Settings.Builder repositorySettings = chunkedRepositorySettings(2 << 23);
+
+        internalCluster().ensureAtLeastNumSearchAndDataNodes(numReplicasIndex + 1);
+        createIndexWithDocsAndEnsureGreen(numReplicasIndex, 1000, indexName);
+        createRepositoryWithSettings(repositorySettings, repoName);
+        takeSnapshot(client, snapshotName, repoName, indexName);
+
+        deleteIndicesAndEnsureGreen(client, indexName);
+        restoreSnapshotAndEnsureGreen(client, snapshotName, repoName);
+        assertRemoteSnapshotIndexSettings(client, restoredIndexName);
+
+        assertDocCount(restoredIndexName, 1000L);
+    }
+
+    /**
+     * Tests a small 1000 bytes chunked repository scenario for searchable snapshots by creating an index,
+     * taking a snapshot, restoring it as a searchable snapshot index.
+     */
+    public void testCreateSearchableSnapshotWithSmallChunks() throws Exception {
+        final int numReplicasIndex = randomIntBetween(1, 4);
+        final String indexName = "test-idx";
+        final String restoredIndexName = indexName + "-copy";
+        final String repoName = "test-repo";
+        final String snapshotName = "test-snap";
+        final Client client = client();
+
+        Settings.Builder repositorySettings = chunkedRepositorySettings(1000);
 
         internalCluster().ensureAtLeastNumSearchAndDataNodes(numReplicasIndex + 1);
         createIndexWithDocsAndEnsureGreen(numReplicasIndex, 1000, indexName);

server/licenses/lucene-grouping-9.10.0.jar.sha1

@@ -15,6 +15,8 @@
 import org.opensearch.index.store.remote.utils.TransferManager;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * This is an implementation of {@link OnDemandBlockIndexInput} where this class provides the main IndexInput using shard snapshot files.
@@ -136,25 +138,45 @@ protected IndexInput fetchBlock(int blockId) throws IOException {
         final long blockStart = getBlockStart(blockId);
         final long blockEnd = blockStart + getActualBlockSize(blockId);
 
-        // If the snapshot file is chunked, we must account for this by
-        // choosing the appropriate file part and updating the position
-        // accordingly.
-        final int part = (int) (blockStart / partSize);
-        final long partStart = part * partSize;
-
-        final long position = blockStart - partStart;
-        final long length = blockEnd - blockStart;
-
+        // Block may be present on multiple chunks of a file, so we need
+        // to fetch each chunk/blob part separately to fetch an entire block.
         BlobFetchRequest blobFetchRequest = BlobFetchRequest.builder()
-            .position(position)
-            .length(length)
-            .blobName(fileInfo.partName(part))
+            .blobParts(getBlobParts(blockStart, blockEnd))
             .directory(directory)
             .fileName(blockFileName)
             .build();
         return transferManager.fetchBlob(blobFetchRequest);
     }
 
+    /**
+     * Returns list of blob parts/chunks in a file for a given block.
+     */
+    protected List<BlobFetchRequest.BlobPart> getBlobParts(long blockStart, long blockEnd) {
+        // If the snapshot file is chunked, we must account for this by
+        // choosing the appropriate file part and updating the position
+        // accordingly.
+        int partNum = (int) (blockStart / partSize);
+        long pos = blockStart;
+        long diff = (blockEnd - blockStart);
+
+        List<BlobFetchRequest.BlobPart> blobParts = new ArrayList<>();
+        while (diff > 0) {
+            long partStart = pos % partSize;
+            long partEnd;
+            if ((partStart + diff) > partSize) {
+                partEnd = partSize;
+            } else {
+                partEnd = (partStart + diff);
+            }
+            long fetchBytes = partEnd - partStart;
+            blobParts.add(new BlobFetchRequest.BlobPart(fileInfo.partName(partNum), partStart, fetchBytes));
+            partNum++;
+            pos = pos + fetchBytes;
+            diff = (blockEnd - pos);
+        }
+        return blobParts;
+    }
+
     @Override
     public OnDemandBlockSnapshotIndexInput clone() {
         OnDemandBlockSnapshotIndexInput clone = buildSlice("clone", 0L, this.length);