Skip to content

Commit a918530

Browse files
Add changes to build star tree in off heap (#14817)
--------- Signed-off-by: Bharathwaj G <bharath78910@gmail.com>
1 parent 212597e commit a918530

24 files changed

+2028
-90
lines changed

server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ public void testValidCompositeIndex() {
275275
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
276276
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
277277
assertEquals(
278-
StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP,
278+
StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP,
279279
starTreeFieldType.getStarTreeConfig().getBuildMode()
280280
);
281281
assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims());
@@ -359,7 +359,7 @@ public void testUpdateIndexWhenMappingIsSame() {
359359
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
360360
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
361361
assertEquals(
362-
StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP,
362+
StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP,
363363
starTreeFieldType.getStarTreeConfig().getBuildMode()
364364
);
365365
assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims());
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.common.util;
10+
11+
import org.apache.lucene.store.IndexInput;
12+
import org.apache.lucene.store.IndexOutput;
13+
import org.apache.lucene.store.RandomAccessInput;
14+
15+
import java.io.IOException;
16+
17+
/**
18+
* A bitset backed by a byte array. This will initialize and set bits in the byte array based on the index.
19+
*/
20+
public class ByteArrayBackedBitset {
21+
private final byte[] byteArray;
22+
23+
/**
24+
* Constructor which uses an on heap list. This should be using during construction of the bitset.
25+
*/
26+
public ByteArrayBackedBitset(int capacity) {
27+
byteArray = new byte[capacity];
28+
}
29+
30+
/**
31+
* Constructor which set the Lucene's RandomAccessInput to read the bitset into a read-only buffer.
32+
*/
33+
public ByteArrayBackedBitset(RandomAccessInput in, long offset, int length) throws IOException {
34+
byteArray = new byte[length];
35+
int i = 0;
36+
while (i < length) {
37+
byteArray[i] = in.readByte(offset + i);
38+
i++;
39+
}
40+
}
41+
42+
/**
43+
* Constructor which set the Lucene's IndexInput to read the bitset into a read-only buffer.
44+
*/
45+
public ByteArrayBackedBitset(IndexInput in, int length) throws IOException {
46+
byteArray = new byte[length];
47+
int i = 0;
48+
while (i < length) {
49+
byteArray[i] = in.readByte();
50+
i++;
51+
}
52+
}
53+
54+
/**
55+
* Sets the bit at the given index to 1.
56+
* Each byte can indicate 8 bits, so the index is divided by 8 to get the byte array index.
57+
* @param index the index to set the bit
58+
*/
59+
public void set(int index) {
60+
int byteArrIndex = index >> 3;
61+
byteArray[byteArrIndex] |= (byte) (1 << (index & 7));
62+
}
63+
64+
public int write(IndexOutput output) throws IOException {
65+
int numBytes = 0;
66+
for (Byte bitSet : byteArray) {
67+
output.writeByte(bitSet);
68+
numBytes += Byte.BYTES;
69+
}
70+
return numBytes;
71+
}
72+
73+
/**
74+
* Retrieves whether the bit is set or not at the given index.
75+
* @param index the index to look up for the bit
76+
* @return true if bit is set, false otherwise
77+
*/
78+
public boolean get(int index) throws IOException {
79+
int byteArrIndex = index >> 3;
80+
return (byteArray[byteArrIndex] & (1 << (index & 7))) != 0;
81+
}
82+
83+
public int getCurrBytesRead() {
84+
return byteArray.length;
85+
}
86+
}

server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@
88

99
package org.opensearch.index.codec.composite;
1010

11-
import org.apache.logging.log4j.LogManager;
12-
import org.apache.logging.log4j.Logger;
1311
import org.apache.lucene.codecs.DocValuesConsumer;
1412
import org.apache.lucene.codecs.DocValuesProducer;
1513
import org.apache.lucene.index.DocValues;
@@ -50,9 +48,9 @@ public class Composite99DocValuesWriter extends DocValuesConsumer {
5048
private final Set<CompositeMappedFieldType> compositeMappedFieldTypes;
5149
private final Set<String> compositeFieldSet;
5250
private final Set<String> segmentFieldSet;
51+
private final boolean segmentHasCompositeFields;
5352

5453
private final Map<String, DocValuesProducer> fieldProducerMap = new HashMap<>();
55-
private static final Logger logger = LogManager.getLogger(Composite99DocValuesWriter.class);
5654

5755
public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) {
5856

@@ -70,6 +68,8 @@ public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState
7068
for (CompositeMappedFieldType type : compositeMappedFieldTypes) {
7169
compositeFieldSet.addAll(type.fields());
7270
}
71+
// check if there are any composite fields which are part of the segment
72+
segmentHasCompositeFields = Collections.disjoint(segmentFieldSet, compositeFieldSet) == false;
7373
}
7474

7575
@Override
@@ -91,7 +91,7 @@ public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) th
9191
public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
9292
delegate.addSortedNumericField(field, valuesProducer);
9393
// Perform this only during flush flow
94-
if (mergeState.get() == null) {
94+
if (mergeState.get() == null && segmentHasCompositeFields) {
9595
createCompositeIndicesIfPossible(valuesProducer, field);
9696
}
9797
}

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/CountValueAggregator.java

+5
Original file line numberDiff line numberDiff line change
@@ -68,4 +68,9 @@ public Long toLongValue(Long value) {
6868
public Long toStarTreeNumericTypeValue(Long value) {
6969
return value;
7070
}
71+
72+
@Override
73+
public Long getIdentityMetricValue() {
74+
return 0L;
75+
}
7176
}

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/SumValueAggregator.java

+5
Original file line numberDiff line numberDiff line change
@@ -103,4 +103,9 @@ public Double toStarTreeNumericTypeValue(Long value) {
103103
throw new IllegalStateException("Cannot convert " + value + " to sortable aggregation type", e);
104104
}
105105
}
106+
107+
@Override
108+
public Double getIdentityMetricValue() {
109+
return 0D;
110+
}
106111
}

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/aggregators/ValueAggregator.java

+5
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,9 @@ public interface ValueAggregator<A> {
6161
* Converts an aggregated value from a Long type.
6262
*/
6363
A toStarTreeNumericTypeValue(Long rawValue);
64+
65+
/**
66+
* Fetches a value that does not alter the result of aggregations
67+
*/
68+
A getIdentityMetricValue();
6469
}

0 commit comments

Comments
 (0)