Skip to content

Commit 547e3ab

Browse files
backslashtbackslasht
committed
Merge branch 'main' into shard_routing_fix
2 parents cc24120 + ef2a9e5 commit 547e3ab

File tree

17 files changed

+691
-9
lines changed

17 files changed

+691
-9
lines changed

‎CHANGELOG.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
109109
- Allow setting KEYSTORE_PASSWORD through env variable ([#12865](https://github.com/opensearch-project/OpenSearch/pull/12865))
110110
- [Concurrent Segment Search] Perform buildAggregation concurrently and support Composite Aggregations ([#12697](https://github.com/opensearch-project/OpenSearch/pull/12697))
111111
- [Concurrent Segment Search] Disable concurrent segment search for system indices and throttled requests ([#12954](https://github.com/opensearch-project/OpenSearch/pull/12954))
112+
- Derived fields support to derive field values at query time without indexing ([#12569](https://github.com/opensearch-project/OpenSearch/pull/12569))
112113
- Detect breaking changes on pull requests ([#9044](https://github.com/opensearch-project/OpenSearch/pull/9044))
113114
- Add cluster primary balance contraint for rebalancing with buffer ([#12656](https://github.com/opensearch-project/OpenSearch/pull/12656))
114115

@@ -117,7 +118,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
117118
- Bump `asm` from 9.6 to 9.7 ([#12908](https://github.com/opensearch-project/OpenSearch/pull/12908))
118119
- Bump `net.minidev:json-smart` from 2.5.0 to 2.5.1 ([#12893](https://github.com/opensearch-project/OpenSearch/pull/12893))
119120
- Bump `netty` from 4.1.107.Final to 4.1.108.Final ([#12924](https://github.com/opensearch-project/OpenSearch/pull/12924))
120-
- Bump `commons-io:commons-io` from 2.15.1 to 2.16.0 ([#12996](https://github.com/opensearch-project/OpenSearch/pull/12996))
121+
- Bump `commons-io:commons-io` from 2.15.1 to 2.16.0 ([#12996](https://github.com/opensearch-project/OpenSearch/pull/12996), [#12998](https://github.com/opensearch-project/OpenSearch/pull/12998), [#12999](https://github.com/opensearch-project/OpenSearch/pull/12999))
121122

122123
### Changed
123124
- [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))

‎plugins/discovery-azure-classic/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ dependencies {
5353
api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
5454
api "commons-codec:commons-codec:${versions.commonscodec}"
5555
api "commons-lang:commons-lang:2.6"
56-
api "commons-io:commons-io:2.15.1"
56+
api "commons-io:commons-io:2.16.0"
5757
api 'javax.mail:mail:1.4.7'
5858
api 'javax.inject:javax.inject:1'
5959
api "com.sun.jersey:jersey-client:${versions.jersey}"

‎plugins/discovery-azure-classic/licenses/commons-io-2.15.1.jar.sha1

-1
This file was deleted.

‎plugins/discovery-azure-classic/licenses/commons-io-2.16.0.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
27875a7935f1ddcc13267eb6fae1f719e0409572

‎plugins/ingest-attachment/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ dependencies {
5757
runtimeOnly "com.google.guava:guava:${versions.guava}"
5858
// Other dependencies
5959
api 'org.tukaani:xz:1.9'
60-
api 'commons-io:commons-io:2.15.1'
60+
api 'commons-io:commons-io:2.16.0'
6161
api "org.slf4j:slf4j-api:${versions.slf4j}"
6262

6363
// character set detection

‎plugins/ingest-attachment/licenses/commons-io-2.15.1.jar.sha1

-1
This file was deleted.

‎plugins/ingest-attachment/licenses/commons-io-2.16.0.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
27875a7935f1ddcc13267eb6fae1f719e0409572

‎server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java

+129
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.index.mapper;
10+
11+
import org.apache.lucene.index.IndexableField;
12+
import org.opensearch.core.xcontent.XContentBuilder;
13+
import org.opensearch.script.Script;
14+
15+
import java.io.IOException;
16+
import java.util.Arrays;
17+
import java.util.List;
18+
import java.util.function.Function;
19+
20+
/**
21+
* A field mapper for derived fields
22+
*
23+
* @opensearch.internal
24+
*/
25+
public class DerivedFieldMapper extends ParametrizedFieldMapper {
26+
27+
public static final String CONTENT_TYPE = "derived";
28+
29+
private static DerivedFieldMapper toType(FieldMapper in) {
30+
return (DerivedFieldMapper) in;
31+
}
32+
33+
/**
34+
* Builder for this field mapper
35+
*
36+
* @opensearch.internal
37+
*/
38+
public static class Builder extends ParametrizedFieldMapper.Builder {
39+
// TODO: The type of parameter may change here if the actual underlying FieldType object is needed
40+
private final Parameter<String> type = Parameter.stringParam("type", false, m -> toType(m).type, "text");
41+
42+
private final Parameter<Script> script = new Parameter<>(
43+
"script",
44+
false,
45+
() -> null,
46+
(n, c, o) -> o == null ? null : Script.parse(o),
47+
m -> toType(m).script
48+
).setSerializerCheck((id, ic, value) -> value != null);
49+
50+
public Builder(String name) {
51+
super(name);
52+
}
53+
54+
@Override
55+
protected List<Parameter<?>> getParameters() {
56+
return Arrays.asList(type, script);
57+
}
58+
59+
@Override
60+
public DerivedFieldMapper build(BuilderContext context) {
61+
FieldMapper fieldMapper = DerivedFieldSupportedTypes.getFieldMapperFromType(type.getValue(), name, context);
62+
Function<Object, IndexableField> fieldFunction = DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(
63+
type.getValue(),
64+
name
65+
);
66+
DerivedFieldType ft = new DerivedFieldType(
67+
buildFullName(context),
68+
type.getValue(),
69+
script.getValue(),
70+
fieldMapper,
71+
fieldFunction
72+
);
73+
return new DerivedFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this);
74+
}
75+
}
76+
77+
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
78+
private final String type;
79+
private final Script script;
80+
81+
protected DerivedFieldMapper(
82+
String simpleName,
83+
MappedFieldType mappedFieldType,
84+
MultiFields multiFields,
85+
CopyTo copyTo,
86+
Builder builder
87+
) {
88+
super(simpleName, mappedFieldType, multiFields, copyTo);
89+
this.type = builder.type.getValue();
90+
this.script = builder.script.getValue();
91+
}
92+
93+
@Override
94+
public DerivedFieldType fieldType() {
95+
return (DerivedFieldType) super.fieldType();
96+
}
97+
98+
@Override
99+
protected void parseCreateField(ParseContext context) throws IOException {
100+
// Leaving this empty as the parsing should be handled via the Builder when root object is parsed.
101+
// The context would not contain anything in this case since the DerivedFieldMapper is not indexed or stored.
102+
throw new UnsupportedOperationException("should not be invoked");
103+
}
104+
105+
@Override
106+
public ParametrizedFieldMapper.Builder getMergeBuilder() {
107+
return new Builder(simpleName()).init(this);
108+
}
109+
110+
@Override
111+
protected String contentType() {
112+
return CONTENT_TYPE;
113+
}
114+
115+
@Override
116+
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
117+
getMergeBuilder().toXContent(builder, includeDefaults);
118+
multiFields.toXContent(builder, params);
119+
copyTo.toXContent(builder, params);
120+
}
121+
122+
public String getType() {
123+
return type;
124+
}
125+
126+
public Script getScript() {
127+
return script;
128+
}
129+
}

‎server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java

+84-1
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,15 @@ protected static boolean parseObjectOrDocumentTypeProperties(
291291
} else if (fieldName.equals("enabled")) {
292292
builder.enabled(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".enabled"));
293293
return true;
294+
} else if (fieldName.equals("derived")) {
295+
if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
296+
// nothing to do here, empty (to support "derived: []" case)
297+
} else if (fieldNode instanceof Map) {
298+
parseDerived(builder, (Map<String, Object>) fieldNode, parserContext);
299+
} else {
300+
throw new OpenSearchParseException("derived must be a map type");
301+
}
302+
return true;
294303
} else if (fieldName.equals("properties")) {
295304
if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
296305
// nothing to do here, empty (to support "properties: []" case)
@@ -349,6 +358,55 @@ protected static void parseNested(
349358
}
350359
}
351360

361+
protected static void parseDerived(ObjectMapper.Builder objBuilder, Map<String, Object> derivedNode, ParserContext parserContext) {
362+
Iterator<Map.Entry<String, Object>> iterator = derivedNode.entrySet().iterator();
363+
while (iterator.hasNext()) {
364+
Map.Entry<String, Object> entry = iterator.next();
365+
String fieldName = entry.getKey();
366+
// Should accept empty arrays, as a work around for when the
367+
// user can't provide an empty Map. (PHP for example)
368+
boolean isEmptyList = entry.getValue() instanceof List && ((List<?>) entry.getValue()).isEmpty();
369+
370+
if (entry.getValue() instanceof Map) {
371+
@SuppressWarnings("unchecked")
372+
Map<String, Object> node = (Map<String, Object>) entry.getValue();
373+
374+
// Derived fields are a bit unique in that the 'type' attribute does not map to the TypeParser
375+
// like it would for traditional fields in properties.
376+
// So in this case, the DerivedFieldMapper's TypeParser will explicitly be used
377+
Mapper.TypeParser typeParser = parserContext.typeParser(DerivedFieldMapper.CONTENT_TYPE);
378+
String[] fieldNameParts = fieldName.split("\\.");
379+
// field name is just ".", which is invalid
380+
if (fieldNameParts.length < 1) {
381+
throw new MapperParsingException("Invalid field name " + fieldName);
382+
}
383+
String realFieldName = fieldNameParts[fieldNameParts.length - 1];
384+
Mapper.Builder<?> fieldBuilder = typeParser.parse(realFieldName, node, parserContext);
385+
for (int i = fieldNameParts.length - 2; i >= 0; --i) {
386+
ObjectMapper.Builder<?> intermediate = new ObjectMapper.Builder<>(fieldNameParts[i]);
387+
intermediate.add(fieldBuilder);
388+
fieldBuilder = intermediate;
389+
}
390+
objBuilder.add(fieldBuilder);
391+
node.remove("type");
392+
DocumentMapperParser.checkNoRemainingFields(fieldName, node, parserContext.indexVersionCreated());
393+
iterator.remove();
394+
} else if (isEmptyList) {
395+
iterator.remove();
396+
} else {
397+
throw new MapperParsingException(
398+
"Expected map for property [derived_fields] on field [" + fieldName + "] but got a " + fieldName.getClass()
399+
);
400+
}
401+
}
402+
403+
DocumentMapperParser.checkNoRemainingFields(
404+
derivedNode,
405+
parserContext.indexVersionCreated(),
406+
"DocType mapping definition has unsupported parameters: "
407+
);
408+
}
409+
352410
protected static void parseProperties(ObjectMapper.Builder objBuilder, Map<String, Object> propsNode, ParserContext parserContext) {
353411
Iterator<Map.Entry<String, Object>> iterator = propsNode.entrySet().iterator();
354412
while (iterator.hasNext()) {
@@ -663,7 +721,21 @@ public void toXContent(XContentBuilder builder, Params params, ToXContent custom
663721
doXContent(builder, params);
664722

665723
// sort the mappers so we get consistent serialization format
666-
Mapper[] sortedMappers = mappers.values().stream().toArray(size -> new Mapper[size]);
724+
Mapper[] derivedSortedMappers = mappers.values()
725+
.stream()
726+
.filter(m -> m instanceof DerivedFieldMapper)
727+
.toArray(size -> new Mapper[size]);
728+
Arrays.sort(derivedSortedMappers, new Comparator<Mapper>() {
729+
@Override
730+
public int compare(Mapper o1, Mapper o2) {
731+
return o1.name().compareTo(o2.name());
732+
}
733+
});
734+
735+
Mapper[] sortedMappers = mappers.values()
736+
.stream()
737+
.filter(m -> !(m instanceof DerivedFieldMapper))
738+
.toArray(size -> new Mapper[size]);
667739
Arrays.sort(sortedMappers, new Comparator<Mapper>() {
668740
@Override
669741
public int compare(Mapper o1, Mapper o2) {
@@ -672,6 +744,17 @@ public int compare(Mapper o1, Mapper o2) {
672744
});
673745

674746
int count = 0;
747+
for (Mapper mapper : derivedSortedMappers) {
748+
if (count++ == 0) {
749+
builder.startObject("derived");
750+
}
751+
mapper.toXContent(builder, params);
752+
}
753+
if (count > 0) {
754+
builder.endObject();
755+
}
756+
757+
count = 0;
675758
for (Mapper mapper : sortedMappers) {
676759
if (!(mapper instanceof MetadataFieldMapper)) {
677760
if (count++ == 0) {

‎server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -670,7 +670,11 @@ public final void parse(String name, ParserContext parserContext, Map<String, Ob
670670
deprecatedParamsMap.put(deprecatedName, param);
671671
}
672672
}
673-
String type = (String) fieldNode.remove("type");
673+
String type = (String) fieldNode.get("type");
674+
if (paramsMap.get("type") == null) {
675+
fieldNode.remove("type");
676+
}
677+
674678
for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
675679
Map.Entry<String, Object> entry = iterator.next();
676680
final String propName = entry.getKey();

‎server/src/main/java/org/opensearch/indices/IndicesModule.java

+2
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.opensearch.index.mapper.ConstantKeywordFieldMapper;
5050
import org.opensearch.index.mapper.DataStreamFieldMapper;
5151
import org.opensearch.index.mapper.DateFieldMapper;
52+
import org.opensearch.index.mapper.DerivedFieldMapper;
5253
import org.opensearch.index.mapper.DocCountFieldMapper;
5354
import org.opensearch.index.mapper.FieldAliasMapper;
5455
import org.opensearch.index.mapper.FieldNamesFieldMapper;
@@ -170,6 +171,7 @@ public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mappe
170171
mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
171172
mappers.put(FlatObjectFieldMapper.CONTENT_TYPE, FlatObjectFieldMapper.PARSER);
172173
mappers.put(ConstantKeywordFieldMapper.CONTENT_TYPE, new ConstantKeywordFieldMapper.TypeParser());
174+
mappers.put(DerivedFieldMapper.CONTENT_TYPE, DerivedFieldMapper.PARSER);
173175

174176
for (MapperPlugin mapperPlugin : mapperPlugins) {
175177
for (Map.Entry<String, Mapper.TypeParser> entry : mapperPlugin.getMappers().entrySet()) {

‎server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java

-1
Original file line numberDiff line numberDiff line change
@@ -1026,7 +1026,6 @@ public void testWeightedRoutingShardState() {
10261026
shardRouting = shardIterator.nextOrNull();
10271027
assertNotNull(shardRouting);
10281028
requestCount.put(shardRouting.currentNodeId(), requestCount.getOrDefault(shardRouting.currentNodeId(), 0) + 1);
1029-
10301029
}
10311030
assertEquals(3, requestCount.get("node1").intValue());
10321031
assertEquals(2, requestCount.get("node2").intValue());

0 commit comments

Comments
 (0)