Skip to content

Commit 23f5c2f

Browse files
sarthakaggarwal97sarthakaggarwal97
authored
[Star Tree] Scaled Float Support (opensearch-project#15442)
--------- Signed-off-by: Sarthak Aggarwal <sarthagg@amazon.com>
1 parent 48cf5f0 commit 23f5c2f

File tree

43 files changed

+575
-483
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+575
-483
lines changed

‎modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java

+21-2
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.opensearch.common.xcontent.support.XContentMapValues;
5050
import org.opensearch.core.xcontent.XContentParser;
5151
import org.opensearch.core.xcontent.XContentParser.Token;
52+
import org.opensearch.index.compositeindex.datacube.DimensionType;
5253
import org.opensearch.index.fielddata.FieldData;
5354
import org.opensearch.index.fielddata.IndexFieldData;
5455
import org.opensearch.index.fielddata.IndexNumericFieldData;
@@ -71,10 +72,12 @@
7172
import java.util.Collections;
7273
import java.util.List;
7374
import java.util.Map;
75+
import java.util.Optional;
7476
import java.util.function.Supplier;
7577

7678
/** A {@link FieldMapper} for scaled floats. Values are internally multiplied
77-
* by a scaling factor and rounded to the closest long. */
79+
* by a scaling factor and rounded to the closest long.
80+
*/
7881
public class ScaledFloatFieldMapper extends ParametrizedFieldMapper {
7982

8083
public static final String CONTENT_TYPE = "scaled_float";
@@ -162,11 +165,21 @@ public ScaledFloatFieldMapper build(BuilderContext context) {
162165
);
163166
return new ScaledFloatFieldMapper(name, type, multiFieldsBuilder.build(this, context), copyTo.build(), this);
164167
}
168+
169+
@Override
170+
public Optional<DimensionType> getSupportedDataCubeDimensionType() {
171+
return Optional.of(DimensionType.NUMERIC);
172+
}
173+
174+
@Override
175+
public boolean isDataCubeMetricSupported() {
176+
return true;
177+
}
165178
}
166179

167180
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings()));
168181

169-
public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder {
182+
public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder, FieldValueConverter {
170183

171184
private final double scalingFactor;
172185
private final Double nullValue;
@@ -340,6 +353,12 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
340353
private double scale(Object input) {
341354
return new BigDecimal(Double.toString(parse(input))).multiply(BigDecimal.valueOf(scalingFactor)).doubleValue();
342355
}
356+
357+
@Override
358+
public double toDoubleValue(long value) {
359+
double inverseScalingFactor = 1d / scalingFactor;
360+
return value * inverseScalingFactor;
361+
}
343362
}
344363

345364
private final Explicit<Boolean> ignoreMalformed;

‎modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java

+101-7
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,24 @@
3434

3535
import org.apache.lucene.index.DocValuesType;
3636
import org.apache.lucene.index.IndexableField;
37+
import org.opensearch.common.settings.Settings;
38+
import org.opensearch.common.util.FeatureFlags;
3739
import org.opensearch.common.xcontent.XContentFactory;
3840
import org.opensearch.core.common.bytes.BytesReference;
3941
import org.opensearch.core.xcontent.MediaTypeRegistry;
4042
import org.opensearch.core.xcontent.XContentBuilder;
43+
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
4144
import org.opensearch.plugins.Plugin;
45+
import org.junit.AfterClass;
46+
import org.junit.BeforeClass;
4247

4348
import java.io.IOException;
4449
import java.util.Arrays;
4550
import java.util.Collection;
4651
import java.util.List;
4752

4853
import static java.util.Collections.singletonList;
54+
import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX;
4955
import static org.hamcrest.Matchers.containsString;
5056

5157
public class ScaledFloatFieldMapperTests extends MapperTestCase {
@@ -91,24 +97,112 @@ public void testExistsQueryDocValuesDisabled() throws IOException {
9197
assertParseMinimalWarnings();
9298
}
9399

94-
public void testDefaults() throws Exception {
95-
XContentBuilder mapping = fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0));
100+
@BeforeClass
101+
public static void createMapper() {
102+
FeatureFlags.initializeFeatureFlags(Settings.builder().put(STAR_TREE_INDEX, "true").build());
103+
}
104+
105+
@AfterClass
106+
public static void clearMapper() {
107+
FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
108+
}
109+
110+
public void testScaledFloatWithStarTree() throws Exception {
111+
112+
double scalingFactorField1 = randomDouble() * 100;
113+
double scalingFactorField2 = randomDouble() * 100;
114+
double scalingFactorField3 = randomDouble() * 100;
115+
116+
XContentBuilder mapping = getStarTreeMappingWithScaledFloat(scalingFactorField1, scalingFactorField2, scalingFactorField3);
96117
DocumentMapper mapper = createDocumentMapper(mapping);
97-
assertEquals(mapping.toString(), mapper.mappingSource().toString());
118+
assertTrue(mapping.toString().contains("startree"));
98119

99-
ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123)));
100-
IndexableField[] fields = doc.rootDoc().getFields("field");
120+
long randomLongField1 = randomLong();
121+
long randomLongField2 = randomLong();
122+
long randomLongField3 = randomLong();
123+
ParsedDocument doc = mapper.parse(
124+
source(b -> b.field("field1", randomLongField1).field("field2", randomLongField2).field("field3", randomLongField3))
125+
);
126+
validateScaledFloatFields(doc, "field1", randomLongField1, scalingFactorField1);
127+
validateScaledFloatFields(doc, "field2", randomLongField2, scalingFactorField2);
128+
validateScaledFloatFields(doc, "field3", randomLongField3, scalingFactorField3);
129+
}
130+
131+
@Override
132+
protected Settings getIndexSettings() {
133+
return Settings.builder()
134+
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
135+
.put(super.getIndexSettings())
136+
.build();
137+
}
138+
139+
private static void validateScaledFloatFields(ParsedDocument doc, String field, long value, double scalingFactor) {
140+
IndexableField[] fields = doc.rootDoc().getFields(field);
101141
assertEquals(2, fields.length);
102142
IndexableField pointField = fields[0];
103143
assertEquals(1, pointField.fieldType().pointDimensionCount());
104144
assertFalse(pointField.fieldType().stored());
105-
assertEquals(1230, pointField.numericValue().longValue());
145+
assertEquals((long) (value * scalingFactor), pointField.numericValue().longValue());
106146
IndexableField dvField = fields[1];
107147
assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
108-
assertEquals(1230, dvField.numericValue().longValue());
148+
assertEquals((long) (value * scalingFactor), dvField.numericValue().longValue());
109149
assertFalse(dvField.fieldType().stored());
110150
}
111151

152+
private XContentBuilder getStarTreeMappingWithScaledFloat(
153+
double scalingFactorField1,
154+
double scalingFactorField2,
155+
double scalingFactorField3
156+
) throws IOException {
157+
return topMapping(b -> {
158+
b.startObject("composite");
159+
b.startObject("startree");
160+
b.field("type", "star_tree");
161+
b.startObject("config");
162+
b.field("max_leaf_docs", 100);
163+
b.startArray("ordered_dimensions");
164+
b.startObject();
165+
b.field("name", "field1");
166+
b.endObject();
167+
b.startObject();
168+
b.field("name", "field2");
169+
b.endObject();
170+
b.endArray();
171+
b.startArray("metrics");
172+
b.startObject();
173+
b.field("name", "field3");
174+
b.startArray("stats");
175+
b.value("sum");
176+
b.value("value_count");
177+
b.endArray();
178+
b.endObject();
179+
b.endArray();
180+
b.endObject();
181+
b.endObject();
182+
b.endObject();
183+
b.startObject("properties");
184+
b.startObject("field1");
185+
b.field("type", "scaled_float").field("scaling_factor", scalingFactorField1);
186+
b.endObject();
187+
b.startObject("field2");
188+
b.field("type", "scaled_float").field("scaling_factor", scalingFactorField2);
189+
b.endObject();
190+
b.startObject("field3");
191+
b.field("type", "scaled_float").field("scaling_factor", scalingFactorField3);
192+
b.endObject();
193+
b.endObject();
194+
});
195+
}
196+
197+
public void testDefaults() throws Exception {
198+
XContentBuilder mapping = fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0));
199+
DocumentMapper mapper = createDocumentMapper(mapping);
200+
assertEquals(mapping.toString(), mapper.mappingSource().toString());
201+
202+
ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123)));
203+
validateScaledFloatFields(doc, "field", 123, 10.0);
204+
}
205+
112206
public void testMissingScalingFactor() {
113207
Exception e = expectThrows(
114208
MapperParsingException.class,

‎server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

+27-46
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import org.opensearch.action.index.IndexResponse;
1313
import org.opensearch.action.search.SearchResponse;
1414
import org.opensearch.action.support.master.AcknowledgedResponse;
15-
import org.opensearch.common.Rounding;
1615
import org.opensearch.common.settings.Settings;
1716
import org.opensearch.common.util.FeatureFlags;
1817
import org.opensearch.core.common.unit.ByteSizeUnit;
@@ -23,7 +22,6 @@
2322
import org.opensearch.index.IndexService;
2423
import org.opensearch.index.IndexSettings;
2524
import org.opensearch.index.compositeindex.CompositeIndexSettings;
26-
import org.opensearch.index.compositeindex.datacube.DateDimension;
2725
import org.opensearch.index.compositeindex.datacube.MetricStat;
2826
import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
2927
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
@@ -62,7 +60,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
6260
.startObject("config")
6361
.startArray("ordered_dimensions")
6462
.startObject()
65-
.field("name", "timestamp")
63+
.field("name", "numeric_dv_1")
64+
.endObject()
65+
.startObject()
66+
.field("name", "numeric_dv_2")
6667
.endObject()
6768
.startObject()
6869
.field("name", getDim(invalidDim, keywordDim))
@@ -84,6 +85,14 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
8485
.field("type", "integer")
8586
.field("doc_values", true)
8687
.endObject()
88+
.startObject("numeric_dv_1")
89+
.field("type", "integer")
90+
.field("doc_values", true)
91+
.endObject()
92+
.startObject("numeric_dv_2")
93+
.field("type", "integer")
94+
.field("doc_values", true)
95+
.endObject()
8796
.startObject("numeric")
8897
.field("type", "integer")
8998
.field("doc_values", false)
@@ -112,11 +121,7 @@ private static XContentBuilder createMaxDimTestMapping() {
112121
.startObject("config")
113122
.startArray("ordered_dimensions")
114123
.startObject()
115-
.field("name", "timestamp")
116-
.startArray("calendar_intervals")
117-
.value("day")
118-
.value("month")
119-
.endArray()
124+
.field("name", "dim4")
120125
.endObject()
121126
.startObject()
122127
.field("name", "dim2")
@@ -201,7 +206,7 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea
201206
.startObject("config")
202207
.startArray("ordered_dimensions")
203208
.startObject()
204-
.field("name", "timestamp")
209+
.field("name", "numeric_dv1")
205210
.endObject()
206211
.startObject()
207212
.field("name", changeDim ? "numeric_new" : getDim(false, false))
@@ -223,6 +228,10 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea
223228
.field("type", "integer")
224229
.field("doc_values", true)
225230
.endObject()
231+
.startObject("numeric_dv1")
232+
.field("type", "integer")
233+
.field("doc_values", true)
234+
.endObject()
226235
.startObject("numeric")
227236
.field("type", "integer")
228237
.field("doc_values", false)
@@ -256,7 +265,7 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo
256265
.startObject("config")
257266
.startArray("ordered_dimensions")
258267
.startObject()
259-
.field("name", "timestamp")
268+
.field("name", "numeric_dv2")
260269
.endObject()
261270
.startObject()
262271
.field("name", "numeric_dv")
@@ -284,6 +293,10 @@ private XContentBuilder getMappingWithDuplicateFields(boolean isDuplicateDim, bo
284293
.field("type", "integer")
285294
.field("doc_values", true)
286295
.endObject()
296+
.startObject("numeric_dv2")
297+
.field("type", "integer")
298+
.field("doc_values", true)
299+
.endObject()
287300
.startObject("numeric_dv1")
288301
.field("type", "integer")
289302
.field("doc_values", true)
@@ -328,15 +341,8 @@ public void testValidCompositeIndex() {
328341
for (CompositeMappedFieldType ft : fts) {
329342
assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType);
330343
StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft;
331-
assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField());
332-
assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
333-
DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
334-
List<Rounding.DateTimeUnit> expectedTimeUnits = Arrays.asList(
335-
Rounding.DateTimeUnit.MINUTES_OF_HOUR,
336-
Rounding.DateTimeUnit.HOUR_OF_DAY
337-
);
338-
assertEquals(expectedTimeUnits, dateDim.getIntervals());
339-
assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
344+
assertEquals("numeric_dv_1", starTreeFieldType.getDimensions().get(0).getField());
345+
assertEquals("numeric_dv_2", starTreeFieldType.getDimensions().get(1).getField());
340346
assertEquals(2, starTreeFieldType.getMetrics().size());
341347
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
342348

@@ -496,15 +502,8 @@ public void testUpdateIndexWhenMappingIsSame() {
496502
for (CompositeMappedFieldType ft : fts) {
497503
assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType);
498504
StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft;
499-
assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField());
500-
assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
501-
DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
502-
List<Rounding.DateTimeUnit> expectedTimeUnits = Arrays.asList(
503-
Rounding.DateTimeUnit.MINUTES_OF_HOUR,
504-
Rounding.DateTimeUnit.HOUR_OF_DAY
505-
);
506-
assertEquals(expectedTimeUnits, dateDim.getIntervals());
507-
assertEquals("numeric_dv", starTreeFieldType.getDimensions().get(1).getField());
505+
assertEquals("numeric_dv_1", starTreeFieldType.getDimensions().get(0).getField());
506+
assertEquals("numeric_dv_2", starTreeFieldType.getDimensions().get(1).getField());
508507
assertEquals("numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
509508

510509
// Assert default metrics
@@ -570,24 +569,6 @@ public void testMaxMetricsCompositeIndex() {
570569
);
571570
}
572571

573-
public void testMaxCalendarIntervalsCompositeIndex() {
574-
MapperParsingException ex = expectThrows(
575-
MapperParsingException.class,
576-
() -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping())
577-
.setSettings(
578-
Settings.builder()
579-
.put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1)
580-
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
581-
.put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(512, ByteSizeUnit.MB))
582-
)
583-
.get()
584-
);
585-
assertEquals(
586-
"Failed to parse mapping [_doc]: At most [1] calendar intervals are allowed in dimension [timestamp]",
587-
ex.getMessage()
588-
);
589-
}
590-
591572
public void testUnsupportedDim() {
592573
MapperParsingException ex = expectThrows(
593574
MapperParsingException.class,

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,7 @@
1212
import org.opensearch.common.annotation.ExperimentalApi;
1313
import org.opensearch.common.xcontent.support.XContentMapValues;
1414
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
15-
import org.opensearch.index.mapper.DateFieldMapper;
1615
import org.opensearch.index.mapper.Mapper;
17-
import org.opensearch.index.mapper.NumberFieldMapper;
1816

1917
import java.util.ArrayList;
2018
import java.util.List;
@@ -55,11 +53,13 @@ public static Dimension parseAndCreateDimension(
5553
Map<String, Object> dimensionMap,
5654
Mapper.TypeParser.ParserContext c
5755
) {
58-
if (builder instanceof DateFieldMapper.Builder) {
56+
if (builder.getSupportedDataCubeDimensionType().isPresent()
57+
&& builder.getSupportedDataCubeDimensionType().get().equals(DimensionType.DATE)) {
5958
return parseAndCreateDateDimension(name, dimensionMap, c);
60-
} else if (builder instanceof NumberFieldMapper.Builder) {
61-
return new NumericDimension(name);
62-
}
59+
} else if (builder.getSupportedDataCubeDimensionType().isPresent()
60+
&& builder.getSupportedDataCubeDimensionType().get().equals(DimensionType.NUMERIC)) {
61+
return new NumericDimension(name);
62+
}
6363
throw new IllegalArgumentException(
6464
String.format(Locale.ROOT, "unsupported field type associated with star tree dimension [%s]", name)
6565
);

0 commit comments

Comments
 (0)