Skip to content

Commit 611ecc2

Browse files
author
Bhumika Saini
authoredJul 21, 2023
[Remote Segment Store] Add Lucene major version to UploadedSegmentMetadata (#8088)
* Add Lucene version to UploadedSegmentMetadata --------- Signed-off-by: Bhumika Saini <sabhumik@amazon.com>
·
3.0.0-alpha1
1 parent b669533 commit 611ecc2

File tree

5 files changed

+225
-33
lines changed

5 files changed

+225
-33
lines changed
 

‎server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java

+21
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,25 @@ public static long invertLong(String str) {
4848
}
4949
return Long.MAX_VALUE - num;
5050
}
51+
52+
/**
53+
* Extracts the segment name from the provided segment file name
54+
* @param filename Segment file name to parse
55+
* @return Name of the segment that the segment file belongs to
56+
*/
57+
public static String getSegmentName(String filename) {
58+
// Segment file names follow patterns like "_0.cfe" or "_0_1_Lucene90_0.dvm".
59+
// Here, the segment name is "_0", which is the set of characters
60+
// starting with "_" until the next "_" or first ".".
61+
int endIdx = filename.indexOf('_', 1);
62+
if (endIdx == -1) {
63+
endIdx = filename.indexOf('.');
64+
}
65+
66+
if (endIdx == -1) {
67+
throw new IllegalArgumentException("Unable to infer segment name for segment file " + filename);
68+
}
69+
70+
return filename.substring(0, endIdx);
71+
}
5172
}

‎server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java

+79-3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
import org.apache.logging.log4j.message.ParameterizedMessage;
1515
import org.apache.lucene.codecs.CodecUtil;
1616
import org.apache.lucene.index.CorruptIndexException;
17+
import org.apache.lucene.index.SegmentCommitInfo;
18+
import org.apache.lucene.index.SegmentInfo;
1719
import org.apache.lucene.index.SegmentInfos;
1820
import org.apache.lucene.store.ByteBuffersDataOutput;
1921
import org.apache.lucene.store.ByteBuffersIndexOutput;
@@ -22,6 +24,7 @@
2224
import org.apache.lucene.store.IOContext;
2325
import org.apache.lucene.store.IndexInput;
2426
import org.apache.lucene.store.IndexOutput;
27+
import org.apache.lucene.util.Version;
2528
import org.opensearch.ExceptionsHelper;
2629
import org.opensearch.action.ActionListener;
2730
import org.opensearch.common.UUIDs;
@@ -217,6 +220,14 @@ public static class UploadedSegmentMetadata {
217220
private final String checksum;
218221
private final long length;
219222

223+
/**
224+
* The Lucene major version that wrote the original segment files.
225+
* As part of the Lucene version compatibility check, this version information stored in the metadata
226+
* will be used to skip downloading the segment files unnecessarily
227+
* if they were written by an incompatible Lucene version.
228+
*/
229+
private int writtenByMajor;
230+
220231
UploadedSegmentMetadata(String originalFilename, String uploadedFilename, String checksum, long length) {
221232
this.originalFilename = originalFilename;
222233
this.uploadedFilename = uploadedFilename;
@@ -226,7 +237,14 @@ public static class UploadedSegmentMetadata {
226237

227238
@Override
228239
public String toString() {
229-
return String.join(SEPARATOR, originalFilename, uploadedFilename, checksum, String.valueOf(length));
240+
return String.join(
241+
SEPARATOR,
242+
originalFilename,
243+
uploadedFilename,
244+
checksum,
245+
String.valueOf(length),
246+
String.valueOf(writtenByMajor)
247+
);
230248
}
231249

232250
public String getChecksum() {
@@ -239,12 +257,35 @@ public long getLength() {
239257

240258
public static UploadedSegmentMetadata fromString(String uploadedFilename) {
241259
String[] values = uploadedFilename.split(SEPARATOR);
242-
return new UploadedSegmentMetadata(values[0], values[1], values[2], Long.parseLong(values[3]));
260+
UploadedSegmentMetadata metadata = new UploadedSegmentMetadata(values[0], values[1], values[2], Long.parseLong(values[3]));
261+
if (values.length < 5) {
262+
logger.error("Lucene version is missing for UploadedSegmentMetadata: " + uploadedFilename);
263+
}
264+
265+
metadata.setWrittenByMajor(Integer.parseInt(values[4]));
266+
267+
return metadata;
243268
}
244269

245270
public String getOriginalFilename() {
246271
return originalFilename;
247272
}
273+
274+
public void setWrittenByMajor(int writtenByMajor) {
275+
if (writtenByMajor <= Version.LATEST.major && writtenByMajor >= Version.MIN_SUPPORTED_MAJOR) {
276+
this.writtenByMajor = writtenByMajor;
277+
} else {
278+
throw new IllegalArgumentException(
279+
"Lucene major version supplied ("
280+
+ writtenByMajor
281+
+ ") is incorrect. Should be between Version.LATEST ("
282+
+ Version.LATEST.major
283+
+ ") and Version.MIN_SUPPORTED_MAJOR ("
284+
+ Version.MIN_SUPPORTED_MAJOR
285+
+ ")."
286+
);
287+
}
288+
}
248289
}
249290

250291
/**
@@ -582,10 +623,13 @@ public void uploadMetadata(
582623
);
583624
try {
584625
try (IndexOutput indexOutput = storeDirectory.createOutput(metadataFilename, IOContext.DEFAULT)) {
626+
Map<String, Integer> segmentToLuceneVersion = getSegmentToLuceneVersion(segmentFiles, segmentInfosSnapshot);
585627
Map<String, String> uploadedSegments = new HashMap<>();
586628
for (String file : segmentFiles) {
587629
if (segmentsUploadedToRemoteStore.containsKey(file)) {
588-
uploadedSegments.put(file, segmentsUploadedToRemoteStore.get(file).toString());
630+
UploadedSegmentMetadata metadata = segmentsUploadedToRemoteStore.get(file);
631+
metadata.setWrittenByMajor(segmentToLuceneVersion.get(metadata.originalFilename));
632+
uploadedSegments.put(file, metadata.toString());
589633
} else {
590634
throw new NoSuchFileException(file);
591635
}
@@ -615,6 +659,38 @@ public void uploadMetadata(
615659
}
616660
}
617661

662+
/**
663+
* Parses the provided SegmentInfos to retrieve a mapping of the provided segment files to
664+
* the respective Lucene major version that wrote the segments
665+
* @param segmentFiles List of segment files for which the Lucene major version is needed
666+
* @param segmentInfosSnapshot SegmentInfos instance to parse
667+
* @return Map of the segment file to its Lucene major version
668+
*/
669+
private Map<String, Integer> getSegmentToLuceneVersion(Collection<String> segmentFiles, SegmentInfos segmentInfosSnapshot) {
670+
Map<String, Integer> segmentToLuceneVersion = new HashMap<>();
671+
for (SegmentCommitInfo segmentCommitInfo : segmentInfosSnapshot) {
672+
SegmentInfo info = segmentCommitInfo.info;
673+
Set<String> segFiles = info.files();
674+
for (String file : segFiles) {
675+
segmentToLuceneVersion.put(file, info.getVersion().major);
676+
}
677+
}
678+
679+
for (String file : segmentFiles) {
680+
if (segmentToLuceneVersion.containsKey(file) == false) {
681+
if (file.equals(segmentInfosSnapshot.getSegmentsFileName())) {
682+
segmentToLuceneVersion.put(file, segmentInfosSnapshot.getCommitLuceneVersion().major);
683+
} else {
684+
// Fallback to the Lucene major version of the respective segment's .si file
685+
String segmentInfoFileName = RemoteStoreUtils.getSegmentName(file) + ".si";
686+
segmentToLuceneVersion.put(file, segmentToLuceneVersion.get(segmentInfoFileName));
687+
}
688+
}
689+
}
690+
691+
return segmentToLuceneVersion;
692+
}
693+
618694
/**
619695
* Try to delete file from local store. Fails silently on failures
620696
* @param filename: name of the file to be deleted

‎server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java

+22
Original file line numberDiff line numberDiff line change
@@ -38,4 +38,26 @@ public void testinvert() {
3838
assertEquals(num, RemoteStoreUtils.invertLong(RemoteStoreUtils.invertLong(num)));
3939
}
4040
}
41+
42+
public void testGetSegmentNameForCfeFile() {
43+
assertEquals("_foo", RemoteStoreUtils.getSegmentName("_foo.cfe"));
44+
}
45+
46+
public void testGetSegmentNameForDvmFile() {
47+
assertEquals("_bar", RemoteStoreUtils.getSegmentName("_bar_1_Lucene90_0.dvm"));
48+
}
49+
50+
public void testGetSegmentNameWeirdSegmentNameOnlyUnderscore() {
51+
// Validate behaviour when segment name contains delimiters only
52+
assertEquals("_", RemoteStoreUtils.getSegmentName("_.dvm"));
53+
}
54+
55+
public void testGetSegmentNameUnderscoreDelimiterOverrides() {
56+
// Validate behaviour when segment name contains delimiters only
57+
assertEquals("_", RemoteStoreUtils.getSegmentName("___.dvm"));
58+
}
59+
60+
public void testGetSegmentNameException() {
61+
assertThrows(IllegalArgumentException.class, () -> RemoteStoreUtils.getSegmentName("dvd"));
62+
}
4163
}

‎server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java

+98-30
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.lucene.store.IndexOutput;
2222
import org.apache.lucene.store.OutputStreamIndexOutput;
2323
import org.apache.lucene.tests.util.LuceneTestCase;
24+
import org.apache.lucene.util.Version;
2425
import org.junit.After;
2526
import org.junit.Before;
2627
import org.mockito.Mockito;
@@ -124,14 +125,42 @@ public void testUploadedSegmentMetadataToString() {
124125
"123456",
125126
1234
126127
);
127-
assertEquals("abc::pqr::123456::1234", metadata.toString());
128+
metadata.setWrittenByMajor(Version.LATEST.major);
129+
assertEquals("abc::pqr::123456::1234::" + Version.LATEST.major, metadata.toString());
130+
}
131+
132+
public void testUploadedSegmentMetadataToStringExceptionTooNew() {
133+
RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = new RemoteSegmentStoreDirectory.UploadedSegmentMetadata(
134+
"abc",
135+
"pqr",
136+
"123456",
137+
1234
138+
);
139+
assertThrows(IllegalArgumentException.class, () -> metadata.setWrittenByMajor(Version.LATEST.major + 1));
140+
}
141+
142+
public void testUploadedSegmentMetadataToStringExceptionTooOld() {
143+
RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = new RemoteSegmentStoreDirectory.UploadedSegmentMetadata(
144+
"abc",
145+
"pqr",
146+
"123456",
147+
1234
148+
);
149+
assertThrows(IllegalArgumentException.class, () -> metadata.setWrittenByMajor(Version.LATEST.major - 2));
128150
}
129151

130152
public void testUploadedSegmentMetadataFromString() {
131153
RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = RemoteSegmentStoreDirectory.UploadedSegmentMetadata.fromString(
132-
"_0.cfe::_0.cfe__uuidxyz::4567::372000"
154+
"_0.cfe::_0.cfe__uuidxyz::4567::372000::" + Version.LATEST.major
155+
);
156+
assertEquals("_0.cfe::_0.cfe__uuidxyz::4567::372000::" + Version.LATEST.major, metadata.toString());
157+
}
158+
159+
public void testUploadedSegmentMetadataFromStringException() {
160+
assertThrows(
161+
ArrayIndexOutOfBoundsException.class,
162+
() -> RemoteSegmentStoreDirectory.UploadedSegmentMetadata.fromString("_0.cfe::_0.cfe__uuidxyz::4567::372000")
133163
);
134-
assertEquals("_0.cfe::_0.cfe__uuidxyz::4567::372000", metadata.toString());
135164
}
136165

137166
public void testGetPrimaryTermGenerationUuid() {
@@ -176,6 +205,8 @@ private Map<String, String> getDummyMetadata(String prefix, int commitGeneration
176205
+ randomIntBetween(1000, 5000)
177206
+ "::"
178207
+ randomIntBetween(512000, 1024000)
208+
+ "::"
209+
+ Version.MIN_SUPPORTED_MAJOR
179210
);
180211
metadata.put(
181212
prefix + ".cfs",
@@ -188,6 +219,8 @@ private Map<String, String> getDummyMetadata(String prefix, int commitGeneration
188219
+ randomIntBetween(1000, 5000)
189220
+ "::"
190221
+ randomIntBetween(512000, 1024000)
222+
+ "::"
223+
+ Version.MIN_SUPPORTED_MAJOR
191224
);
192225
metadata.put(
193226
prefix + ".si",
@@ -200,6 +233,8 @@ private Map<String, String> getDummyMetadata(String prefix, int commitGeneration
200233
+ randomIntBetween(1000, 5000)
201234
+ "::"
202235
+ randomIntBetween(512000, 1024000)
236+
+ "::"
237+
+ Version.LATEST.major
203238
);
204239
metadata.put(
205240
"segments_" + commitGeneration,
@@ -213,6 +248,8 @@ private Map<String, String> getDummyMetadata(String prefix, int commitGeneration
213248
+ randomIntBetween(1000, 5000)
214249
+ "::"
215250
+ randomIntBetween(1024, 5120)
251+
+ "::"
252+
+ Version.LATEST.major
216253
);
217254
return metadata;
218255
}
@@ -611,8 +648,8 @@ public void testContainsFile() throws IOException {
611648
).thenReturn(metadataFiles);
612649

613650
Map<String, String> metadata = new HashMap<>();
614-
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512");
615-
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024");
651+
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512::" + Version.LATEST.major);
652+
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024::" + Version.LATEST.major);
616653

617654
when(remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)).thenReturn(createMetadataFileBytes(metadata, 1, 5));
618655

@@ -641,37 +678,53 @@ public void testUploadMetadataEmpty() throws IOException {
641678
IndexOutput indexOutput = mock(IndexOutput.class);
642679
when(storeDirectory.createOutput(startsWith("metadata__12__o"), eq(IOContext.DEFAULT))).thenReturn(indexOutput);
643680

644-
Collection<String> segmentFiles = List.of("s1", "s2", "s3");
681+
Collection<String> segmentFiles = List.of("_s1.si", "_s1.cfe", "_s3.cfs");
645682
assertThrows(
646683
NoSuchFileException.class,
647684
() -> remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, segmentInfos, storeDirectory, 12L, 34L)
648685
);
649686
}
650687

651688
public void testUploadMetadataNonEmpty() throws IOException {
652-
populateMetadata();
689+
indexDocs(142364, 5);
690+
flushShard(indexShard, true);
691+
SegmentInfos segInfos = indexShard.store().readLastCommittedSegmentsInfo();
692+
long primaryTerm = 12;
693+
String primaryTermLong = RemoteStoreUtils.invertLong(primaryTerm);
694+
long generation = segInfos.getGeneration();
695+
String generationLong = RemoteStoreUtils.invertLong(generation);
696+
String latestMetadataFileName = "metadata__" + primaryTermLong + "__" + generationLong + "__abc";
697+
List<String> metadataFiles = List.of(latestMetadataFileName);
698+
when(
699+
remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
700+
RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
701+
1
702+
)
703+
).thenReturn(metadataFiles);
704+
Map<String, Map<String, String>> metadataFilenameContentMapping = Map.of(
705+
latestMetadataFileName,
706+
getDummyMetadata("_0", (int) generation)
707+
);
708+
when(remoteMetadataDirectory.openInput(latestMetadataFileName, IOContext.DEFAULT)).thenReturn(
709+
createMetadataFileBytes(metadataFilenameContentMapping.get(latestMetadataFileName), generation, primaryTerm)
710+
);
711+
653712
remoteSegmentStoreDirectory.init();
654713

655714
Directory storeDirectory = mock(Directory.class);
656715
BytesStreamOutput output = new BytesStreamOutput();
657716
IndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096);
717+
when(storeDirectory.createOutput(startsWith("metadata__" + primaryTermLong + "__" + generationLong), eq(IOContext.DEFAULT)))
718+
.thenReturn(indexOutput);
658719

659-
String generation = RemoteStoreUtils.invertLong(segmentInfos.getGeneration());
660-
String primaryTerm = RemoteStoreUtils.invertLong(12);
661-
when(storeDirectory.createOutput(startsWith("metadata__" + primaryTerm + "__" + generation), eq(IOContext.DEFAULT))).thenReturn(
662-
indexOutput
663-
);
664-
665-
Collection<String> segmentFiles = List.of("_0.si", "_0.cfe", "_0.cfs", "segments_1");
666-
remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, segmentInfos, storeDirectory, 12L, 34L);
720+
remoteSegmentStoreDirectory.uploadMetadata(segInfos.files(true), segInfos, storeDirectory, primaryTerm, generation);
667721

668722
verify(remoteMetadataDirectory).copyFrom(
669723
eq(storeDirectory),
670-
startsWith("metadata__" + primaryTerm + "__" + generation),
671-
startsWith("metadata__" + primaryTerm + "__" + generation),
724+
startsWith("metadata__" + primaryTermLong + "__" + generationLong),
725+
startsWith("metadata__" + primaryTermLong + "__" + generationLong),
672726
eq(IOContext.DEFAULT)
673727
);
674-
675728
VersionedCodecStreamWrapper<RemoteSegmentMetadata> streamWrapper = new VersionedCodecStreamWrapper<>(
676729
new RemoteSegmentMetadataHandler(),
677730
RemoteSegmentMetadata.CURRENT_VERSION,
@@ -680,16 +733,25 @@ public void testUploadMetadataNonEmpty() throws IOException {
680733
RemoteSegmentMetadata remoteSegmentMetadata = streamWrapper.readStream(
681734
new ByteArrayIndexInput("expected", BytesReference.toBytes(output.bytes()))
682735
);
683-
684736
Map<String, RemoteSegmentStoreDirectory.UploadedSegmentMetadata> actual = remoteSegmentStoreDirectory
685737
.getSegmentsUploadedToRemoteStore();
686738
Map<String, RemoteSegmentStoreDirectory.UploadedSegmentMetadata> expected = remoteSegmentMetadata.getMetadata();
687-
688739
for (String filename : expected.keySet()) {
689740
assertEquals(expected.get(filename).toString(), actual.get(filename).toString());
690741
}
691742
}
692743

744+
public void testUploadMetadataNoSegmentCommitInfos() throws IOException {
745+
SegmentInfos segInfos = indexShard.store().readLastCommittedSegmentsInfo();
746+
int numSegCommitInfos = segInfos.size();
747+
assertEquals(
748+
"For a fresh index, the number of SegmentCommitInfo instances associated with the SegmentInfos instance should be 0, but were found to be "
749+
+ numSegCommitInfos,
750+
0,
751+
numSegCommitInfos
752+
);
753+
}
754+
693755
public void testNoMetadataHeaderCorruptIndexException() throws IOException {
694756
List<String> metadataFiles = List.of(metadataFilename);
695757
when(
@@ -700,8 +762,8 @@ public void testNoMetadataHeaderCorruptIndexException() throws IOException {
700762
).thenReturn(metadataFiles);
701763

702764
Map<String, String> metadata = new HashMap<>();
703-
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234");
704-
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345");
765+
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::" + Version.LATEST.major);
766+
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::" + Version.LATEST.major);
705767

706768
BytesStreamOutput output = new BytesStreamOutput();
707769
OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096);
@@ -723,8 +785,8 @@ public void testInvalidCodecHeaderCorruptIndexException() throws IOException {
723785
).thenReturn(metadataFiles);
724786

725787
Map<String, String> metadata = new HashMap<>();
726-
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234");
727-
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345");
788+
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::" + Version.LATEST.major);
789+
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::" + Version.LATEST.major);
728790

729791
BytesStreamOutput output = new BytesStreamOutput();
730792
OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096);
@@ -748,8 +810,8 @@ public void testHeaderMinVersionCorruptIndexException() throws IOException {
748810
).thenReturn(metadataFiles);
749811

750812
Map<String, String> metadata = new HashMap<>();
751-
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234");
752-
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345");
813+
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::" + Version.LATEST.major);
814+
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::" + Version.LATEST.major);
753815

754816
BytesStreamOutput output = new BytesStreamOutput();
755817
OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096);
@@ -773,8 +835,8 @@ public void testHeaderMaxVersionCorruptIndexException() throws IOException {
773835
).thenReturn(metadataFiles);
774836

775837
Map<String, String> metadata = new HashMap<>();
776-
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234");
777-
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345");
838+
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::" + Version.LATEST.major);
839+
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::" + Version.LATEST.major);
778840

779841
BytesStreamOutput output = new BytesStreamOutput();
780842
OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096);
@@ -798,8 +860,8 @@ public void testIncorrectChecksumCorruptIndexException() throws IOException {
798860
).thenReturn(metadataFiles);
799861

800862
Map<String, String> metadata = new HashMap<>();
801-
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512");
802-
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024");
863+
metadata.put("_0.cfe", "_0.cfe::_0.cfe__" + UUIDs.base64UUID() + "::1234::512::" + Version.LATEST.major);
864+
metadata.put("_0.cfs", "_0.cfs::_0.cfs__" + UUIDs.base64UUID() + "::2345::1024::" + Version.LATEST.major);
803865

804866
BytesStreamOutput output = new BytesStreamOutput();
805867
IndexOutput indexOutput = new OutputStreamIndexOutput("segment metadata", "metadata output stream", output, 4096);
@@ -947,6 +1009,12 @@ public void testSegmentMetadataCurrentVersion() {
9471009
assertEquals(RemoteSegmentMetadata.CURRENT_VERSION, 1);
9481010
}
9491011

1012+
private void indexDocs(int startDocId, int numberOfDocs) throws IOException {
1013+
for (int i = startDocId; i < startDocId + numberOfDocs; i++) {
1014+
indexDoc(indexShard, "_doc", Integer.toString(i));
1015+
}
1016+
}
1017+
9501018
public void testMetadataFileNameOrder() {
9511019
String file1 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 21, 23, 1, 1);
9521020
String file2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(15, 38, 38, 1, 1);

‎server/src/test/java/org/opensearch/index/store/remote/metadata/RemoteSegmentMetadataHandlerTests.java

+5
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.apache.lucene.store.ByteBuffersDataOutput;
1313
import org.apache.lucene.store.ByteBuffersIndexOutput;
1414
import org.apache.lucene.store.OutputStreamIndexOutput;
15+
import org.apache.lucene.util.Version;
1516
import org.junit.After;
1617
import org.junit.Before;
1718
import org.opensearch.cluster.metadata.IndexMetadata;
@@ -134,6 +135,8 @@ private Map<String, String> getDummyData() {
134135
+ randomIntBetween(1000, 5000)
135136
+ "::"
136137
+ randomIntBetween(1024, 2048)
138+
+ "::"
139+
+ Version.LATEST.major
137140
);
138141
expectedOutput.put(
139142
prefix + ".cfs",
@@ -146,6 +149,8 @@ private Map<String, String> getDummyData() {
146149
+ randomIntBetween(1000, 5000)
147150
+ "::"
148151
+ randomIntBetween(1024, 2048)
152+
+ "::"
153+
+ Version.LATEST.major
149154
);
150155
return expectedOutput;
151156
}

0 commit comments

Comments
 (0)
Please sign in to comment.