From 31fc5a32ac94cadc1b989727a622beb83ff2c7b8 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]" <github-actions[bot]@users.noreply.github.com>
Date: Thu, 11 Apr 2024 17:01:43 +0000
Subject: [PATCH] [Derived Fields] PR4: Capability to define derived fields in
search request (#12850)
* Support derived fields definition in search request
* adds support for fetch phase on derived fields
* adds support for highlighting on derived fields
---------
Signed-off-by: Rishabh Maurya <rishabhmaurya05@gmail.com>
(cherry picked from commit 645b1f1b6c0738ef2b4a0de364b6bcd42af239b9)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
---
.../java/org/opensearch/client/SearchIT.java | 229 +++++++++++
.../search/fields/SearchFieldsIT.java | 157 ++++++++
.../action/search/SearchRequestBuilder.java | 15 +
.../opensearch/index/mapper/DerivedField.java | 90 +++++
.../index/mapper/DerivedFieldMapper.java | 37 +-
.../mapper/DerivedFieldSupportedTypes.java | 40 +-
.../index/mapper/DerivedFieldType.java | 197 +++-------
.../mapper/DerivedFieldValueFetcher.java | 38 +-
.../index/mapper/DocumentMapperParser.java | 2 +-
.../index/query/DerivedFieldQuery.java | 29 +-
.../index/query/QueryShardContext.java | 30 +-
.../VectorGeoPointShapeQueryProcessor.java | 5 +-
.../opensearch/script/DerivedFieldScript.java | 7 -
.../org/opensearch/search/SearchService.java | 25 ++
.../search/builder/SearchSourceBuilder.java | 76 +++-
.../subphase/highlight/HighlightPhase.java | 8 +-
.../subphase/highlight/HighlightUtils.java | 4 +
.../highlight/UnifiedHighlighter.java | 5 +
.../mapper/DerivedFieldMapperQueryTests.java | 48 ++-
.../index/mapper/DerivedFieldTypeTests.java | 7 +-
.../index/query/DerivedFieldQueryTests.java | 8 +-
.../index/query/QueryShardContextTests.java | 31 ++
.../opensearch/search/SearchServiceTests.java | 44 +++
.../builder/SearchSourceBuilderTests.java | 46 +++
.../DerivedFieldFetchAndHighlightTests.java | 366 ++++++++++++++++++
.../opensearch/script/MockScriptEngine.java | 44 ++-
26 files changed, 1373 insertions(+), 215 deletions(-)
create mode 100644 server/src/main/java/org/opensearch/index/mapper/DerivedField.java
create mode 100644 server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
index bb705aebd2fcf..b962fa8ff415e 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
@@ -54,15 +54,19 @@
import org.opensearch.action.search.SearchScrollRequest;
import org.opensearch.client.core.CountRequest;
import org.opensearch.client.core.CountResponse;
+import org.opensearch.common.geo.ShapeRelation;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.geometry.Rectangle;
+import org.opensearch.index.query.GeoShapeQueryBuilder;
import org.opensearch.index.query.MatchQueryBuilder;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.query.RangeQueryBuilder;
import org.opensearch.index.query.ScriptQueryBuilder;
import org.opensearch.index.query.TermsQueryBuilder;
import org.opensearch.join.aggregations.Children;
@@ -102,6 +106,8 @@
import org.opensearch.search.suggest.Suggest;
import org.opensearch.search.suggest.SuggestBuilder;
import org.opensearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
import org.hamcrest.Matchers;
import org.junit.Before;
@@ -116,6 +122,7 @@
import java.util.concurrent.TimeUnit;
import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.opensearch.index.query.QueryBuilders.geoShapeQuery;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent;
import static org.hamcrest.Matchers.arrayContaining;
import static org.hamcrest.Matchers.both;
@@ -764,6 +771,228 @@ public void testSearchWithWeirdScriptFields() throws Exception {
}
}
+ public void testSearchWithDerivedFields() throws Exception {
+ // Just testing DerivedField definition from SearchSourceBuilder derivedField()
+ // We are not testing the full functionality here
+ Request doc = new Request("PUT", "test/_doc/1");
+ doc.setJsonEntity("{\"field\":\"value\"}");
+ client().performRequest(doc);
+ client().performRequest(new Request("POST", "/test/_refresh"));
+ // Keyword field
+ {
+ SearchRequest searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "keyword", new Script("emit(params._source[\"field\"])"))
+ .fetchField("result")
+ .query(new TermsQueryBuilder("result", "value"))
+ );
+ SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ SearchHit searchHit = searchResponse.getHits().getAt(0);
+ List<Object> values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(1, values.size());
+ assertEquals("value", values.get(0));
+
+ // multi valued
+ searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField(
+ "result",
+ "keyword",
+ new Script("emit(params._source[\"field\"]);emit(params._source[\"field\"] + \"_2\")")
+ )
+ .query(new TermsQueryBuilder("result", "value_2"))
+ .fetchField("result")
+ );
+ searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ searchHit = searchResponse.getHits().getAt(0);
+ values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(2, values.size());
+ assertEquals("value", values.get(0));
+ assertEquals("value_2", values.get(1));
+ }
+ // Boolean field
+ {
+ SearchRequest searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "boolean", new Script("emit(((String)params._source[\"field\"]).equals(\"value\"))"))
+ .query(new TermsQueryBuilder("result", "true"))
+ .fetchField("result")
+ );
+ SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ SearchHit searchHit = searchResponse.getHits().getAt(0);
+ List<Object> values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(1, values.size());
+ assertEquals(true, values.get(0));
+ }
+ // Long field
+ {
+ SearchRequest searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "long", new Script("emit(Long.MAX_VALUE)"))
+ .query(new RangeQueryBuilder("result").from(Long.MAX_VALUE - 1).to(Long.MAX_VALUE))
+ .fetchField("result")
+ );
+
+ SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ SearchHit searchHit = searchResponse.getHits().getAt(0);
+ List<Object> values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(1, values.size());
+ assertEquals(Long.MAX_VALUE, values.get(0));
+
+ // multi-valued
+ searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "long", new Script("emit(Long.MAX_VALUE); emit(Long.MIN_VALUE);"))
+ .query(new RangeQueryBuilder("result").from(Long.MIN_VALUE).to(Long.MIN_VALUE + 1))
+ .fetchField("result")
+ );
+
+ searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ searchHit = searchResponse.getHits().getAt(0);
+ values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(2, values.size());
+ assertEquals(Long.MAX_VALUE, values.get(0));
+ assertEquals(Long.MIN_VALUE, values.get(1));
+ }
+ // Double field
+ {
+ SearchRequest searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "double", new Script("emit(Double.MAX_VALUE)"))
+ .query(new RangeQueryBuilder("result").from(Double.MAX_VALUE - 1).to(Double.MAX_VALUE))
+ .fetchField("result")
+ );
+ SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ SearchHit searchHit = searchResponse.getHits().getAt(0);
+ List<Object> values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(1, values.size());
+ assertEquals(Double.MAX_VALUE, values.get(0));
+
+ // multi-valued
+ searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "double", new Script("emit(Double.MAX_VALUE); emit(Double.MIN_VALUE);"))
+ .query(new RangeQueryBuilder("result").from(Double.MIN_VALUE).to(Double.MIN_VALUE + 1))
+ .fetchField("result")
+ );
+
+ searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ searchHit = searchResponse.getHits().getAt(0);
+ values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(2, values.size());
+ assertEquals(Double.MAX_VALUE, values.get(0));
+ assertEquals(Double.MIN_VALUE, values.get(1));
+ }
+ // Date field
+ {
+ DateTime date1 = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC);
+ DateTime date2 = new DateTime(1990, 12, 30, 0, 0, DateTimeZone.UTC);
+ SearchRequest searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L)"))
+ .query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
+ .fetchField("result")
+ );
+
+ SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ SearchHit searchHit = searchResponse.getHits().getAt(0);
+ List<Object> values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(1, values.size());
+ assertEquals(date1.toString(), values.get(0));
+
+ // multi-valued
+ searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L); " + "emit(" + date2.getMillis() + "L)"))
+ .query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
+ .fetchField("result")
+ );
+
+ searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ searchHit = searchResponse.getHits().getAt(0);
+ values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(2, values.size());
+ assertEquals(date1.toString(), values.get(0));
+ assertEquals(date2.toString(), values.get(1));
+ }
+ // Geo field
+ {
+ GeoShapeQueryBuilder qb = geoShapeQuery("result", new Rectangle(-35, 35, 35, -35));
+ qb.relation(ShapeRelation.INTERSECTS);
+ SearchRequest searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "geo_point", new Script("emit(10.0, 20.0)"))
+ .query(qb)
+ .fetchField("result")
+ );
+
+ SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ SearchHit searchHit = searchResponse.getHits().getAt(0);
+ List<Object> values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(1, values.size());
+ assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
+ assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));
+
+ // multi-valued
+ searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "geo_point", new Script("emit(10.0, 20.0); emit(20.0, 30.0);"))
+ .query(qb)
+ .fetchField("result")
+ );
+
+ searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ searchHit = searchResponse.getHits().getAt(0);
+ values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(2, values.size());
+ assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
+ assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));
+ assertEquals(20.0, ((HashMap) values.get(1)).get("lat"));
+ assertEquals(30.0, ((HashMap) values.get(1)).get("lon"));
+ }
+ // IP field
+ {
+ SearchRequest searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource().derivedField("result", "ip", new Script("emit(\"10.0.0.1\")")).fetchField("result")
+ );
+
+ SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ SearchHit searchHit = searchResponse.getHits().getAt(0);
+ List<Object> values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(1, values.size());
+ assertEquals("10.0.0.1", values.get(0));
+
+ // multi-valued
+ searchRequest = new SearchRequest("test").source(
+ SearchSourceBuilder.searchSource()
+ .derivedField("result", "ip", new Script("emit(\"10.0.0.1\"); emit(\"10.0.0.2\");"))
+ .fetchField("result")
+ );
+
+ searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+ searchHit = searchResponse.getHits().getAt(0);
+ values = searchHit.getFields().get("result").getValues();
+ assertNotNull(values);
+ assertEquals(2, values.size());
+ assertEquals("10.0.0.1", values.get(0));
+ assertEquals("10.0.0.2", values.get(1));
+
+ }
+
+ }
+
public void testSearchScroll() throws Exception {
for (int i = 0; i < 100; i++) {
XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java
index 906d45ef84b3f..2ce96092203e8 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java
@@ -40,6 +40,7 @@
import org.opensearch.common.Numbers;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.common.document.DocumentField;
+import org.opensearch.common.geo.GeoPoint;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.time.DateFormatter;
import org.opensearch.common.time.DateUtils;
@@ -51,6 +52,7 @@
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.fielddata.ScriptDocValues;
+import org.opensearch.index.mapper.DateFieldMapper;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.plugins.Plugin;
@@ -189,6 +191,20 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
scripts.put("doc['s']", vars -> docScript(vars, "s"));
scripts.put("doc['ms']", vars -> docScript(vars, "ms"));
+ scripts.put("doc['keyword_field']", vars -> sourceScript(vars, "keyword_field"));
+ scripts.put("doc['multi_keyword_field']", vars -> sourceScript(vars, "multi_keyword_field"));
+ scripts.put("doc['long_field']", vars -> sourceScript(vars, "long_field"));
+ scripts.put("doc['multi_long_field']", vars -> sourceScript(vars, "multi_long_field"));
+ scripts.put("doc['double_field']", vars -> sourceScript(vars, "double_field"));
+ scripts.put("doc['multi_double_field']", vars -> sourceScript(vars, "multi_double_field"));
+ scripts.put("doc['date_field']", vars -> sourceScript(vars, "date_field"));
+ scripts.put("doc['multi_date_field']", vars -> sourceScript(vars, "multi_date_field"));
+ scripts.put("doc['ip_field']", vars -> sourceScript(vars, "ip_field"));
+ scripts.put("doc['multi_ip_field']", vars -> sourceScript(vars, "multi_ip_field"));
+ scripts.put("doc['boolean_field']", vars -> sourceScript(vars, "boolean_field"));
+ scripts.put("doc['geo_field']", vars -> sourceScript(vars, "geo_field"));
+ scripts.put("doc['multi_geo_field']", vars -> sourceScript(vars, "multi_geo_field"));
+
return scripts;
}
@@ -1299,6 +1315,147 @@ public void testScriptFields() throws Exception {
}
}
+ public void testDerivedFields() throws Exception {
+ assertAcked(
+ prepareCreate("index").setMapping(
+ "keyword_field",
+ "type=keyword",
+ "multi_keyword_field",
+ "type=keyword",
+ "long_field",
+ "type=long",
+ "multi_long_field",
+ "type=long",
+ "double_field",
+ "type=double",
+ "multi_double_field",
+ "type=double",
+ "date_field",
+ "type=date",
+ "multi_date_field",
+ "type=date",
+ "ip_field",
+ "type=ip",
+ "multi_ip_field",
+ "type=ip",
+ "boolean_field",
+ "type=boolean",
+ "geo_field",
+ "type=geo_point",
+ "multi_geo_field",
+ "type=geo_point"
+ ).get()
+ );
+ final int numDocs = randomIntBetween(3, 8);
+ List<IndexRequestBuilder> reqs = new ArrayList<>();
+
+ DateTime date1 = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC);
+ DateTime date2 = new DateTime(1990, 12, 30, 0, 0, DateTimeZone.UTC);
+
+ for (int i = 0; i < numDocs; ++i) {
+ reqs.add(
+ client().prepareIndex("index")
+ .setId(Integer.toString(i))
+ .setSource(
+ "keyword_field",
+ Integer.toString(i),
+ "multi_keyword_field",
+ new String[] { Integer.toString(i), Integer.toString(i + 1) },
+ "long_field",
+ (long) i,
+ "multi_long_field",
+ new long[] { i, i + 1 },
+ "double_field",
+ (double) i,
+ "multi_double_field",
+ new double[] { i, i + 1 },
+ "date_field",
+ date1.getMillis(),
+ "multi_date_field",
+ new Long[] { date1.getMillis(), date2.getMillis() },
+ "ip_field",
+ "172.16.1.10",
+ "multi_ip_field",
+ new String[] { "172.16.1.10", "172.16.1.11" },
+ "boolean_field",
+ true,
+ "geo_field",
+ new GeoPoint(12.0, 10.0),
+ "multi_geo_field",
+ new GeoPoint[] { new GeoPoint(12.0, 10.0), new GeoPoint(13.0, 10.0) }
+ )
+ );
+ }
+ indexRandom(true, reqs);
+ indexRandomForConcurrentSearch("index");
+ ensureSearchable();
+ SearchRequestBuilder req = client().prepareSearch("index");
+ String[][] fieldLookup = new String[][] {
+ { "keyword_field", "keyword" },
+ { "multi_keyword_field", "keyword" },
+ { "long_field", "long" },
+ { "multi_long_field", "long" },
+ { "double_field", "double" },
+ { "multi_double_field", "double" },
+ { "date_field", "date" },
+ { "multi_date_field", "date" },
+ { "ip_field", "ip" },
+ { "multi_ip_field", "ip" },
+ { "boolean_field", "boolean" },
+ { "geo_field", "geo_point" },
+ { "multi_geo_field", "geo_point" } };
+ for (String[] field : fieldLookup) {
+ req.addDerivedField(
+ "derived_" + field[0],
+ field[1],
+ new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['" + field[0] + "']", Collections.emptyMap())
+ );
+ }
+ req.addFetchField("derived_*");
+ SearchResponse resp = req.get();
+ assertSearchResponse(resp);
+ for (SearchHit hit : resp.getHits().getHits()) {
+ final int id = Integer.parseInt(hit.getId());
+ Map<String, DocumentField> fields = hit.getFields();
+
+ assertEquals(fields.get("derived_keyword_field").getValues().get(0), Integer.toString(id));
+ assertEquals(fields.get("derived_multi_keyword_field").getValues().get(0), Integer.toString(id));
+ assertEquals(fields.get("derived_multi_keyword_field").getValues().get(1), Integer.toString(id + 1));
+
+ assertEquals(fields.get("derived_long_field").getValues().get(0), id);
+ assertEquals(fields.get("derived_multi_long_field").getValues().get(0), id);
+ assertEquals(fields.get("derived_multi_long_field").getValues().get(1), (id + 1));
+
+ assertEquals(fields.get("derived_double_field").getValues().get(0), (double) id);
+ assertEquals(fields.get("derived_multi_double_field").getValues().get(0), (double) id);
+ assertEquals(fields.get("derived_multi_double_field").getValues().get(1), (double) (id + 1));
+
+ assertEquals(
+ fields.get("derived_date_field").getValues().get(0),
+ DateFieldMapper.getDefaultDateTimeFormatter().formatJoda(date1)
+ );
+ assertEquals(
+ fields.get("derived_multi_date_field").getValues().get(0),
+ DateFieldMapper.getDefaultDateTimeFormatter().formatJoda(date1)
+ );
+ assertEquals(
+ fields.get("derived_multi_date_field").getValues().get(1),
+ DateFieldMapper.getDefaultDateTimeFormatter().formatJoda(date2)
+ );
+
+ assertEquals(fields.get("derived_ip_field").getValues().get(0), "172.16.1.10");
+ assertEquals(fields.get("derived_multi_ip_field").getValues().get(0), "172.16.1.10");
+ assertEquals(fields.get("derived_multi_ip_field").getValues().get(1), "172.16.1.11");
+
+ assertEquals(fields.get("derived_boolean_field").getValues().get(0), true);
+
+ assertEquals(fields.get("derived_geo_field").getValues().get(0), new GeoPoint(12.0, 10.0));
+ assertEquals(fields.get("derived_multi_geo_field").getValues().get(0), new GeoPoint(12.0, 10.0));
+ assertEquals(fields.get("derived_multi_geo_field").getValues().get(1), new GeoPoint(13.0, 10.0));
+
+ }
+ }
+
public void testDocValueFieldsWithFieldAlias() throws Exception {
XContentBuilder mapping = XContentFactory.jsonBuilder()
.startObject()
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
index 9dac827e7d518..4a547ee2c82bd 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
@@ -363,6 +363,21 @@ public SearchRequestBuilder addScriptField(String name, Script script) {
return this;
}
+ /**
+ * Adds a derived field of a given type. The script provided will be used to derive the value
+ * of a given type. Thereafter, it can be treated as regular field of a given type to perform
+ * query on them.
+ *
+ * @param name The name of the field to be used in various parts of the query. The name will also represent
+ * the field value in the return hit.
+ * @param type The type of derived field. All values emitted by script must be of this type
+ * @param script The script to use
+ */
+ public SearchRequestBuilder addDerivedField(String name, String type, Script script) {
+ sourceBuilder().derivedField(name, type, script);
+ return this;
+ }
+
/**
* Adds a sort against the given field name and the sort ordering.
*
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedField.java b/server/src/main/java/org/opensearch/index/mapper/DerivedField.java
new file mode 100644
index 0000000000000..7ebe4e5f0b0e8
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedField.java
@@ -0,0 +1,90 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.script.Script;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * DerivedField representation: expects a name, type and script.
+ */
+@PublicApi(since = "2.14.0")
+public class DerivedField implements Writeable, ToXContentFragment {
+
+ private final String name;
+ private final String type;
+ private final Script script;
+
+ public DerivedField(String name, String type, Script script) {
+ this.name = name;
+ this.type = type;
+ this.script = script;
+ }
+
+ public DerivedField(StreamInput in) throws IOException {
+ name = in.readString();
+ type = in.readString();
+ script = new Script(in);
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeString(name);
+ out.writeString(type);
+ script.writeTo(out);
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+ builder.startObject(name);
+ builder.field("type", type);
+ builder.field("script", script);
+ builder.endObject();
+ return builder;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public Script getScript() {
+ return script;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, type, script);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ DerivedField other = (DerivedField) obj;
+ return Objects.equals(name, other.name) && Objects.equals(type, other.type) && Objects.equals(script, other.script);
+ }
+
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
index b448487a4f810..c6ae71320c35c 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
@@ -14,7 +14,9 @@
import java.io.IOException;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.function.Function;
/**
@@ -37,7 +39,7 @@ private static DerivedFieldMapper toType(FieldMapper in) {
*/
public static class Builder extends ParametrizedFieldMapper.Builder {
// TODO: The type of parameter may change here if the actual underlying FieldType object is needed
- private final Parameter<String> type = Parameter.stringParam("type", false, m -> toType(m).type, "text");
+ private final Parameter<String> type = Parameter.stringParam("type", false, m -> toType(m).type, "");
private final Parameter<Script> script = new Parameter<>(
"script",
@@ -51,6 +53,12 @@ public Builder(String name) {
super(name);
}
+ public Builder(DerivedField derivedField) {
+ super(derivedField.getName());
+ this.type.setValue(derivedField.getType());
+ this.script.setValue(derivedField.getScript());
+ }
+
@Override
protected List<Parameter<?>> getParameters() {
return Arrays.asList(type, script);
@@ -64,9 +72,7 @@ public DerivedFieldMapper build(BuilderContext context) {
name
);
DerivedFieldType ft = new DerivedFieldType(
- buildFullName(context),
- type.getValue(),
- script.getValue(),
+ new DerivedField(buildFullName(context), type.getValue(), script.getValue()),
fieldMapper,
fieldFunction
);
@@ -126,4 +132,27 @@ public String getType() {
public Script getScript() {
return script;
}
+
+ public static Map<String, DerivedFieldType> getAllDerivedFieldTypeFromObject(
+ Map<String, Object> derivedFieldObject,
+ MapperService mapperService
+ ) {
+ Map<String, DerivedFieldType> derivedFieldTypes = new HashMap<>();
+ DocumentMapper documentMapper = mapperService.documentMapperParser().parse(DerivedFieldMapper.CONTENT_TYPE, derivedFieldObject);
+ if (documentMapper != null && documentMapper.mappers() != null) {
+ for (Mapper mapper : documentMapper.mappers()) {
+ if (mapper instanceof DerivedFieldMapper) {
+ DerivedFieldType derivedFieldType = ((DerivedFieldMapper) mapper).fieldType();
+ derivedFieldTypes.put(derivedFieldType.name(), derivedFieldType);
+ }
+ }
+ }
+ return derivedFieldTypes;
+ }
+
+ public static DerivedFieldType getDerivedFieldType(DerivedField derivedField, MapperService mapperService) {
+ BuilderContext builderContext = new Mapper.BuilderContext(mapperService.getIndexSettings().getSettings(), new ContentPath(1));
+ Builder builder = new Builder(derivedField);
+ return builder.build(builderContext).fieldType();
+ }
}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java
index 10b5c4a0f7157..aa6936bf6529a 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java
@@ -20,12 +20,13 @@
import org.apache.lucene.index.IndexableField;
import org.opensearch.Version;
import org.opensearch.common.Booleans;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.geo.GeoPoint;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.common.network.InetAddresses;
import java.net.InetAddress;
import java.util.Arrays;
-import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
import java.util.function.Function;
@@ -36,7 +37,7 @@
* it is used to create an IndexableField for the provided type and object. It is useful when indexing into
* lucene MemoryIndex in {@link org.opensearch.index.query.DerivedFieldQuery}.
*/
-enum DerivedFieldSupportedTypes {
+public enum DerivedFieldSupportedTypes {
BOOLEAN("boolean", (name, context) -> {
BooleanFieldMapper.Builder builder = new BooleanFieldMapper.Builder(name);
@@ -51,7 +52,7 @@ enum DerivedFieldSupportedTypes {
value = Booleans.parseBooleanStrict(textValue, false);
}
return new Field(name, value ? "T" : "F", BooleanFieldMapper.Defaults.FIELD_TYPE);
- }),
+ }, o -> o),
DATE("date", (name, context) -> {
// TODO: should we support mapping settings exposed by a given field type from derived fields too?
// for example, support `format` for date type?
@@ -63,17 +64,17 @@ enum DerivedFieldSupportedTypes {
Version.CURRENT
);
return builder.build(context);
- }, name -> o -> new LongPoint(name, (long) o)),
+ }, name -> o -> new LongPoint(name, (long) o), o -> DateFieldMapper.getDefaultDateTimeFormatter().formatMillis((long) o)),
GEO_POINT("geo_point", (name, context) -> {
GeoPointFieldMapper.Builder builder = new GeoPointFieldMapper.Builder(name);
return builder.build(context);
}, name -> o -> {
// convert o to array of double
- if (!(o instanceof List) || ((List<?>) o).size() != 2 || !(((List<?>) o).get(0) instanceof Double)) {
+ if (!(o instanceof Tuple) || !(((Tuple<?, ?>) o).v1() instanceof Double || !(((Tuple<?, ?>) o).v2() instanceof Double))) {
throw new ClassCastException("geo_point should be in format emit(double lat, double lon) for derived fields");
}
- return new LatLonPoint(name, (Double) ((List<?>) o).get(0), (Double) ((List<?>) o).get(1));
- }),
+ return new LatLonPoint(name, (double) ((Tuple<?, ?>) o).v1(), (double) ((Tuple<?, ?>) o).v2());
+ }, o -> new GeoPoint((double) ((Tuple) o).v1(), (double) ((Tuple) o).v2())),
IP("ip", (name, context) -> {
IpFieldMapper.Builder builder = new IpFieldMapper.Builder(name, false, Version.CURRENT);
return builder.build(context);
@@ -85,7 +86,7 @@ enum DerivedFieldSupportedTypes {
address = InetAddresses.forString(o.toString());
}
return new InetAddressPoint(name, address);
- }),
+ }, o -> o),
KEYWORD("keyword", (name, context) -> {
FieldType dummyFieldType = new FieldType();
dummyFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
@@ -100,29 +101,33 @@ enum DerivedFieldSupportedTypes {
keywordBuilder.copyTo.build(),
keywordBuilder
);
- }, name -> o -> new KeywordField(name, (String) o, Field.Store.NO)),
+ }, name -> o -> new KeywordField(name, (String) o, Field.Store.NO), o -> o),
LONG("long", (name, context) -> {
NumberFieldMapper.Builder longBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.LONG, false, false);
return longBuilder.build(context);
- }, name -> o -> new LongField(name, Long.parseLong(o.toString()), Field.Store.NO)),
+ }, name -> o -> new LongField(name, Long.parseLong(o.toString()), Field.Store.NO), o -> o),
DOUBLE("double", (name, context) -> {
NumberFieldMapper.Builder doubleBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.DOUBLE, false, false);
return doubleBuilder.build(context);
- }, name -> o -> new DoubleField(name, Double.parseDouble(o.toString()), Field.Store.NO));
+ }, name -> o -> new DoubleField(name, Double.parseDouble(o.toString()), Field.Store.NO), o -> o);
final String name;
private final BiFunction<String, Mapper.BuilderContext, FieldMapper> builder;
private final Function<String, Function<Object, IndexableField>> indexableFieldBuilder;
+ private final Function<Object, Object> valueForDisplay;
+
DerivedFieldSupportedTypes(
String name,
BiFunction<String, Mapper.BuilderContext, FieldMapper> builder,
- Function<String, Function<Object, IndexableField>> indexableFieldBuilder
+ Function<String, Function<Object, IndexableField>> indexableFieldBuilder,
+ Function<Object, Object> valueForDisplay
) {
this.name = name;
this.builder = builder;
this.indexableFieldBuilder = indexableFieldBuilder;
+ this.valueForDisplay = valueForDisplay;
}
public String getName() {
@@ -137,6 +142,10 @@ private Function<Object, IndexableField> getIndexableFieldGenerator(String name)
return indexableFieldBuilder.apply(name);
}
+ private Function<Object, Object> getValueForDisplayGenerator() {
+ return valueForDisplay;
+ }
+
private static final Map<String, DerivedFieldSupportedTypes> enumMap = Arrays.stream(DerivedFieldSupportedTypes.values())
.collect(Collectors.toMap(DerivedFieldSupportedTypes::getName, enumValue -> enumValue));
@@ -153,4 +162,11 @@ public static Function<Object, IndexableField> getIndexableFieldGeneratorType(St
}
return enumMap.get(type).getIndexableFieldGenerator(name);
}
+
+ public static Function<Object, Object> getValueForDisplayGenerator(String type) {
+ if (!enumMap.containsKey(type)) {
+ throw new IllegalArgumentException("Type [" + type + "] isn't supported in Derived field context.");
+ }
+ return enumMap.get(type).getValueForDisplayGenerator();
+ }
}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java
index abdca7879cc94..8b480819acd0e 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java
@@ -18,10 +18,11 @@
import org.opensearch.common.geo.ShapeRelation;
import org.opensearch.common.time.DateMathParser;
import org.opensearch.common.unit.Fuzziness;
+import org.opensearch.geometry.Geometry;
+import org.opensearch.index.analysis.NamedAnalyzer;
import org.opensearch.index.query.DerivedFieldQuery;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.script.DerivedFieldScript;
-import org.opensearch.script.Script;
import org.opensearch.search.lookup.SearchLookup;
import java.io.IOException;
@@ -36,19 +37,16 @@
* Contains logic to execute different type of queries on a derived field of given type.
* @opensearch.internal
*/
-public final class DerivedFieldType extends MappedFieldType {
- private final String type;
+public final class DerivedFieldType extends MappedFieldType implements GeoShapeQueryable {
- private final Script script;
+ private final DerivedField derivedField;
FieldMapper typeFieldMapper;
final Function<Object, IndexableField> indexableFieldGenerator;
public DerivedFieldType(
- String name,
- String type,
- Script script,
+ DerivedField derivedField,
boolean isIndexed,
boolean isStored,
boolean hasDocValues,
@@ -56,21 +54,14 @@ public DerivedFieldType(
FieldMapper typeFieldMapper,
Function<Object, IndexableField> fieldFunction
) {
- super(name, isIndexed, isStored, hasDocValues, typeFieldMapper.fieldType().getTextSearchInfo(), meta);
- this.type = type;
- this.script = script;
+ super(derivedField.getName(), isIndexed, isStored, hasDocValues, typeFieldMapper.fieldType().getTextSearchInfo(), meta);
+ this.derivedField = derivedField;
this.typeFieldMapper = typeFieldMapper;
this.indexableFieldGenerator = fieldFunction;
}
- public DerivedFieldType(
- String name,
- String type,
- Script script,
- FieldMapper typeFieldMapper,
- Function<Object, IndexableField> fieldFunction
- ) {
- this(name, type, script, false, false, false, Collections.emptyMap(), typeFieldMapper, fieldFunction);
+ public DerivedFieldType(DerivedField derivedField, FieldMapper typeFieldMapper, Function<Object, IndexableField> fieldFunction) {
+ this(derivedField, false, false, false, Collections.emptyMap(), typeFieldMapper, fieldFunction);
}
@Override
@@ -79,7 +70,15 @@ public String typeName() {
}
public String getType() {
- return type;
+ return derivedField.getType();
+ }
+
+ public MappedFieldType getTypeMappedFieldType() {
+ return typeFieldMapper.mappedFieldType;
+ }
+
+ public NamedAnalyzer getIndexAnalyzer() {
+ return typeFieldMapper.mappedFieldType.indexAnalyzer();
}
@Override
@@ -87,46 +86,33 @@ public DerivedFieldValueFetcher valueFetcher(QueryShardContext context, SearchLo
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
- return new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
+ Function<Object, Object> valueForDisplay = DerivedFieldSupportedTypes.getValueForDisplayGenerator(getType());
+ return new DerivedFieldValueFetcher(
+ getDerivedFieldLeafFactory(context, searchLookup == null ? context.lookup() : searchLookup),
+ valueForDisplay,
+ indexableFieldGenerator
+ );
}
@Override
public Query termQuery(Object value, QueryShardContext context) {
Query query = typeFieldMapper.mappedFieldType.termQuery(value, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
public Query termQueryCaseInsensitive(Object value, @Nullable QueryShardContext context) {
Query query = typeFieldMapper.mappedFieldType.termQueryCaseInsensitive(value, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
public Query termsQuery(List<?> values, @Nullable QueryShardContext context) {
Query query = typeFieldMapper.mappedFieldType.termsQuery(values, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
@@ -150,14 +136,8 @@ public Query rangeQuery(
parser,
context
);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
@@ -170,14 +150,8 @@ public Query fuzzyQuery(
QueryShardContext context
) {
Query query = typeFieldMapper.mappedFieldType.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
@@ -199,14 +173,8 @@ public Query fuzzyQuery(
method,
context
);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
@@ -217,14 +185,8 @@ public Query prefixQuery(
QueryShardContext context
) {
Query query = typeFieldMapper.mappedFieldType.prefixQuery(value, method, caseInsensitive, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
@@ -235,27 +197,15 @@ public Query wildcardQuery(
QueryShardContext context
) {
Query query = typeFieldMapper.mappedFieldType.wildcardQuery(value, method, caseInsensitive, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
public Query normalizedWildcardQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) {
Query query = typeFieldMapper.mappedFieldType.normalizedWildcardQuery(value, method, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
@@ -268,54 +218,30 @@ public Query regexpQuery(
QueryShardContext context
) {
Query query = typeFieldMapper.mappedFieldType.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements, QueryShardContext context) throws IOException {
Query query = typeFieldMapper.mappedFieldType.phraseQuery(stream, slop, enablePositionIncrements, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements, QueryShardContext context)
throws IOException {
Query query = typeFieldMapper.mappedFieldType.multiPhraseQuery(stream, slop, enablePositionIncrements, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, QueryShardContext context) throws IOException {
Query query = typeFieldMapper.mappedFieldType.phrasePrefixQuery(stream, slop, maxExpansions, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
@@ -328,14 +254,15 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew
@Override
public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) {
Query query = typeFieldMapper.mappedFieldType.distanceFeatureQuery(origin, pivot, boost, context);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context));
- return new DerivedFieldQuery(
- query,
- valueFetcher,
- context.lookup(),
- indexableFieldGenerator,
- typeFieldMapper.mappedFieldType.indexAnalyzer()
- );
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+ }
+
+ @Override
+ public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relation, QueryShardContext context) {
+ Query query = ((GeoShapeQueryable) (typeFieldMapper.mappedFieldType)).geoShapeQuery(shape, fieldName, relation, context);
+ DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+ return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
}
@Override
@@ -348,7 +275,7 @@ public boolean isAggregatable() {
return false;
}
- private DerivedFieldScript.LeafFactory getDerivedFieldLeafFactory(QueryShardContext context) {
+ private DerivedFieldScript.LeafFactory getDerivedFieldLeafFactory(QueryShardContext context, SearchLookup searchLookup) {
if (!context.documentMapper("").sourceMapper().enabled()) {
throw new IllegalArgumentException(
"DerivedFieldQuery error: unable to fetch fields from _source field: _source is disabled in the mappings "
@@ -357,7 +284,7 @@ private DerivedFieldScript.LeafFactory getDerivedFieldLeafFactory(QueryShardCont
+ "]"
);
}
- DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
- return factory.newFactory(script.getParams(), context.lookup());
+ DerivedFieldScript.Factory factory = context.compile(derivedField.getScript(), DerivedFieldScript.CONTEXT);
+ return factory.newFactory(derivedField.getScript().getParams(), searchLookup);
}
}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java
index 40aa2f9890965..2d9379e04c512 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java
@@ -8,33 +8,69 @@
package org.opensearch.index.mapper;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
+import org.opensearch.common.annotation.PublicApi;
import org.opensearch.script.DerivedFieldScript;
import org.opensearch.search.lookup.SourceLookup;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
+import java.util.function.Function;
/**
* The value fetcher contains logic to execute script and fetch the value in form of list of object.
* It expects DerivedFieldScript.LeafFactory as an input and sets the contract with consumer to call
* {@link #setNextReader(LeafReaderContext)} whenever a segment is switched.
*/
+@PublicApi(since = "2.14.0")
public final class DerivedFieldValueFetcher implements ValueFetcher {
private DerivedFieldScript derivedFieldScript;
private final DerivedFieldScript.LeafFactory derivedFieldScriptFactory;
- public DerivedFieldValueFetcher(DerivedFieldScript.LeafFactory derivedFieldScriptFactory) {
+ private final Function<Object, Object> valueForDisplay;
+ private final Function<Object, IndexableField> indexableFieldFunction;
+
+ public DerivedFieldValueFetcher(
+ DerivedFieldScript.LeafFactory derivedFieldScriptFactory,
+ Function<Object, Object> valueForDisplay,
+ Function<Object, IndexableField> indexableFieldFunction
+ ) {
this.derivedFieldScriptFactory = derivedFieldScriptFactory;
+ this.valueForDisplay = valueForDisplay;
+ this.indexableFieldFunction = indexableFieldFunction;
}
@Override
public List<Object> fetchValues(SourceLookup lookup) {
+ List<Object> values = fetchValuesInternal(lookup);
+ if (values.isEmpty()) {
+ return values;
+ }
+ List<Object> result = new ArrayList<>();
+ for (Object v : values) {
+ result.add(valueForDisplay.apply(v));
+ }
+ return result;
+ }
+
+ private List<Object> fetchValuesInternal(SourceLookup lookup) {
derivedFieldScript.setDocument(lookup.docId());
derivedFieldScript.execute();
return derivedFieldScript.getEmittedValues();
}
+ public List<IndexableField> getIndexableField(SourceLookup lookup) {
+ List<Object> values = fetchValuesInternal(lookup);
+ List<IndexableField> indexableFields = new ArrayList<>();
+ for (Object v : values) {
+ indexableFields.add(indexableFieldFunction.apply(v));
+ }
+ return indexableFields;
+ }
+
+ @Override
public void setNextReader(LeafReaderContext context) {
try {
derivedFieldScript = derivedFieldScriptFactory.newInstance(context);
diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
index 2e85cdccb6b0d..72cb370d2d362 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
@@ -136,7 +136,7 @@ public DocumentMapper parse(@Nullable String type, CompressedXContent source) th
}
@SuppressWarnings({ "unchecked" })
- private DocumentMapper parse(String type, Map<String, Object> mapping) throws MapperParsingException {
+ public DocumentMapper parse(String type, Map<String, Object> mapping) throws MapperParsingException {
if (type == null) {
throw new MapperParsingException("Failed to derive type");
}
diff --git a/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java b/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
index 8beef0bf46be0..42ac61bf98f73 100644
--- a/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
+++ b/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
@@ -29,7 +29,6 @@
import java.io.IOException;
import java.util.List;
import java.util.Objects;
-import java.util.function.Function;
/**
* DerivedFieldQuery used for querying derived fields. It contains the logic to execute an input lucene query against
@@ -39,7 +38,6 @@ public final class DerivedFieldQuery extends Query {
private final Query query;
private final DerivedFieldValueFetcher valueFetcher;
private final SearchLookup searchLookup;
- private final Function<Object, IndexableField> indexableFieldGenerator;
private final Analyzer indexAnalyzer;
/**
@@ -47,20 +45,11 @@ public final class DerivedFieldQuery extends Query {
* @param valueFetcher DerivedFieldValueFetcher ValueFetcher to fetch the value of a derived field from _source
* using LeafSearchLookup
* @param searchLookup SearchLookup to get the LeafSearchLookup look used by valueFetcher to fetch the _source
- * @param indexableFieldGenerator used to generate lucene IndexableField from a given object fetched by valueFetcher
- * to be used in lucene memory index.
*/
- public DerivedFieldQuery(
- Query query,
- DerivedFieldValueFetcher valueFetcher,
- SearchLookup searchLookup,
- Function<Object, IndexableField> indexableFieldGenerator,
- Analyzer indexAnalyzer
- ) {
+ public DerivedFieldQuery(Query query, DerivedFieldValueFetcher valueFetcher, SearchLookup searchLookup, Analyzer indexAnalyzer) {
this.query = query;
this.valueFetcher = valueFetcher;
this.searchLookup = searchLookup;
- this.indexableFieldGenerator = indexableFieldGenerator;
this.indexAnalyzer = indexAnalyzer;
}
@@ -75,7 +64,7 @@ public Query rewrite(IndexSearcher indexSearcher) throws IOException {
if (rewritten == query) {
return this;
}
- return new DerivedFieldQuery(rewritten, valueFetcher, searchLookup, indexableFieldGenerator, indexAnalyzer);
+ return new DerivedFieldQuery(rewritten, valueFetcher, searchLookup, indexAnalyzer);
}
@Override
@@ -91,12 +80,12 @@ public Scorer scorer(LeafReaderContext context) {
@Override
public boolean matches() {
leafSearchLookup.source().setSegmentAndDocument(context, approximation.docID());
- List<Object> values = valueFetcher.fetchValues(leafSearchLookup.source());
+ List<IndexableField> indexableFields = valueFetcher.getIndexableField(leafSearchLookup.source());
// TODO: in case of errors from script, should it be ignored and treated as missing field
// by using a configurable setting?
MemoryIndex memoryIndex = new MemoryIndex();
- for (Object value : values) {
- memoryIndex.addField(indexableFieldGenerator.apply(value), indexAnalyzer);
+ for (IndexableField indexableField : indexableFields) {
+ memoryIndex.addField(indexableField, indexAnalyzer);
}
float score = memoryIndex.search(query);
return score > 0.0f;
@@ -127,16 +116,12 @@ public boolean equals(Object o) {
return false;
}
DerivedFieldQuery other = (DerivedFieldQuery) o;
- return Objects.equals(this.query, other.query)
- && Objects.equals(this.valueFetcher, other.valueFetcher)
- && Objects.equals(this.searchLookup, other.searchLookup)
- && Objects.equals(this.indexableFieldGenerator, other.indexableFieldGenerator)
- && Objects.equals(this.indexAnalyzer, other.indexAnalyzer);
+ return Objects.equals(this.query, other.query) && Objects.equals(this.indexAnalyzer, other.indexAnalyzer);
}
@Override
public int hashCode() {
- return Objects.hash(classHash(), query, valueFetcher, searchLookup, indexableFieldGenerator, indexableFieldGenerator);
+ return Objects.hash(classHash(), query, indexAnalyzer);
}
@Override
diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
index f3b392559d33e..64643ad6d2c94 100644
--- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
+++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
@@ -45,6 +45,7 @@
import org.opensearch.common.TriFunction;
import org.opensearch.common.annotation.PublicApi;
import org.opensearch.common.lucene.search.Queries;
+import org.opensearch.common.regex.Regex;
import org.opensearch.common.util.BigArrays;
import org.opensearch.core.action.ActionListener;
import org.opensearch.core.common.ParsingException;
@@ -77,6 +78,7 @@
import java.io.IOException;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -119,6 +121,8 @@ public class QueryShardContext extends QueryRewriteContext {
private final ValuesSourceRegistry valuesSourceRegistry;
private BitSetProducer parentFilter;
+ private Map<String, MappedFieldType> derivedFieldTypeMap = new HashMap<>();
+
public QueryShardContext(
int shardId,
IndexSettings indexSettings,
@@ -329,7 +333,17 @@ public Map<String, Query> copyNamedQueries() {
* type then the fields will be returned with a type prefix.
*/
public Set<String> simpleMatchToIndexNames(String pattern) {
- return mapperService.simpleMatchToFullName(pattern);
+ Set<String> matchingFields = mapperService.simpleMatchToFullName(pattern);
+ if (derivedFieldTypeMap != null && !derivedFieldTypeMap.isEmpty()) {
+ Set<String> matchingDerivedFields = new HashSet<>(matchingFields);
+ for (String fieldName : derivedFieldTypeMap.keySet()) {
+ if (!matchingDerivedFields.contains(fieldName) && Regex.simpleMatch(pattern, fieldName)) {
+ matchingDerivedFields.add(fieldName);
+ }
+ }
+ return matchingDerivedFields;
+ }
+ return matchingFields;
}
/**
@@ -395,6 +409,14 @@ public ValuesSourceRegistry getValuesSourceRegistry() {
return valuesSourceRegistry;
}
+ public void setDerivedFieldTypes(Map<String, MappedFieldType> derivedFieldTypeMap) {
+ this.derivedFieldTypeMap = derivedFieldTypeMap;
+ }
+
+ public MappedFieldType getDerivedFieldType(String fieldName) {
+ return derivedFieldTypeMap == null ? null : derivedFieldTypeMap.get(fieldName);
+ }
+
public void setAllowUnmappedFields(boolean allowUnmappedFields) {
this.allowUnmappedFields = allowUnmappedFields;
}
@@ -404,7 +426,11 @@ public void setMapUnmappedFieldAsString(boolean mapUnmappedFieldAsString) {
}
MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMapping) {
- if (fieldMapping != null || allowUnmappedFields) {
+ if (fieldMapping != null) {
+ return fieldMapping;
+ } else if (getDerivedFieldType(name) != null) {
+ return getDerivedFieldType(name);
+ } else if (allowUnmappedFields) {
return fieldMapping;
} else if (mapUnmappedFieldAsString) {
TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, mapperService.getIndexAnalyzers());
diff --git a/server/src/main/java/org/opensearch/index/query/VectorGeoPointShapeQueryProcessor.java b/server/src/main/java/org/opensearch/index/query/VectorGeoPointShapeQueryProcessor.java
index eb7923a5b5212..c55d88439b11f 100644
--- a/server/src/main/java/org/opensearch/index/query/VectorGeoPointShapeQueryProcessor.java
+++ b/server/src/main/java/org/opensearch/index/query/VectorGeoPointShapeQueryProcessor.java
@@ -54,6 +54,7 @@
import org.opensearch.geometry.Polygon;
import org.opensearch.geometry.Rectangle;
import org.opensearch.geometry.ShapeType;
+import org.opensearch.index.mapper.DerivedFieldType;
import org.opensearch.index.mapper.GeoPointFieldMapper;
import org.opensearch.index.mapper.MappedFieldType;
@@ -78,7 +79,9 @@ public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relat
private void validateIsGeoPointFieldType(String fieldName, QueryShardContext context) {
MappedFieldType fieldType = context.fieldMapper(fieldName);
- if (fieldType instanceof GeoPointFieldMapper.GeoPointFieldType == false) {
+ if (fieldType instanceof GeoPointFieldMapper.GeoPointFieldType == false
+ && !(fieldType instanceof DerivedFieldType
+ && (((DerivedFieldType) fieldType).getTypeMappedFieldType() instanceof GeoPointFieldMapper.GeoPointFieldType))) {
throw new QueryShardException(
context,
"Expected "
diff --git a/server/src/main/java/org/opensearch/script/DerivedFieldScript.java b/server/src/main/java/org/opensearch/script/DerivedFieldScript.java
index e9988ec5aeef2..0a2b7cf691283 100644
--- a/server/src/main/java/org/opensearch/script/DerivedFieldScript.java
+++ b/server/src/main/java/org/opensearch/script/DerivedFieldScript.java
@@ -68,13 +68,6 @@ public DerivedFieldScript(Map<String, Object> params, SearchLookup lookup, LeafR
this.totalByteSize = 0;
}
- public DerivedFieldScript() {
- this.params = null;
- this.leafLookup = null;
- this.emittedValues = new ArrayList<>();
- this.totalByteSize = 0;
- }
-
/**
* Return the parameters for this script.
*/
diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java
index d48f6f6522ca5..13369d298e2d5 100644
--- a/server/src/main/java/org/opensearch/search/SearchService.java
+++ b/server/src/main/java/org/opensearch/search/SearchService.java
@@ -78,6 +78,9 @@
import org.opensearch.index.IndexService;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.engine.Engine;
+import org.opensearch.index.mapper.DerivedField;
+import org.opensearch.index.mapper.DerivedFieldMapper;
+import org.opensearch.index.mapper.MappedFieldType;
import org.opensearch.index.query.InnerHitContextBuilder;
import org.opensearch.index.query.MatchAllQueryBuilder;
import org.opensearch.index.query.MatchNoneQueryBuilder;
@@ -1067,6 +1070,28 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
// might end up with incorrect state since we are using now() or script services
// during rewrite and normalized / evaluate templates etc.
QueryShardContext context = new QueryShardContext(searchContext.getQueryShardContext());
+ if (request.source() != null
+ && request.source().size() != 0
+ && (request.source().getDerivedFieldsObject() != null || request.source().getDerivedFields() != null)) {
+ Map<String, MappedFieldType> derivedFieldTypeMap = new HashMap<>();
+ if (request.source().getDerivedFieldsObject() != null) {
+ Map<String, Object> derivedFieldObject = new HashMap<>();
+ derivedFieldObject.put(DerivedFieldMapper.CONTENT_TYPE, request.source().getDerivedFieldsObject());
+ derivedFieldTypeMap.putAll(
+ DerivedFieldMapper.getAllDerivedFieldTypeFromObject(derivedFieldObject, searchContext.mapperService())
+ );
+ }
+ if (request.source().getDerivedFields() != null) {
+ for (DerivedField derivedField : request.source().getDerivedFields()) {
+ derivedFieldTypeMap.put(
+ derivedField.getName(),
+ DerivedFieldMapper.getDerivedFieldType(derivedField, searchContext.mapperService())
+ );
+ }
+ }
+ context.setDerivedFieldTypes(derivedFieldTypeMap);
+ searchContext.getQueryShardContext().setDerivedFieldTypes(derivedFieldTypeMap);
+ }
Rewriteable.rewrite(request.getRewriteable(), context, true);
assert searchContext.getQueryShardContext().isCacheable();
success = true;
diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
index c930cd5752ecd..b40710146c672 100644
--- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
@@ -52,6 +52,8 @@
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentHelper;
import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.index.mapper.DerivedField;
+import org.opensearch.index.mapper.DerivedFieldMapper;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryRewriteContext;
import org.opensearch.index.query.Rewriteable;
@@ -114,6 +116,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R
public static final ParseField DOCVALUE_FIELDS_FIELD = new ParseField("docvalue_fields");
public static final ParseField FETCH_FIELDS_FIELD = new ParseField("fields");
public static final ParseField SCRIPT_FIELDS_FIELD = new ParseField("script_fields");
+ public static final ParseField DERIVED_FIELDS_FIELD = new ParseField(DerivedFieldMapper.CONTENT_TYPE);
public static final ParseField SCRIPT_FIELD = new ParseField("script");
public static final ParseField IGNORE_FAILURE_FIELD = new ParseField("ignore_failure");
public static final ParseField SORT_FIELD = new ParseField("sort");
@@ -193,6 +196,10 @@ public static HighlightBuilder highlight() {
private StoredFieldsContext storedFieldsContext;
private List<FieldAndFormat> docValueFields;
private List<ScriptField> scriptFields;
+ private Map<String, Object> derivedFieldsObject;
+
+ private List<DerivedField> derivedFields;
+
private FetchSourceContext fetchSourceContext;
private List<FieldAndFormat> fetchFields;
@@ -291,6 +298,14 @@ public SearchSourceBuilder(StreamInput in) throws IOException {
if (in.getVersion().onOrAfter(Version.V_2_13_0)) {
includeNamedQueriesScore = in.readOptionalBoolean();
}
+ if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+ if (in.readBoolean()) {
+ derivedFieldsObject = in.readMap();
+ }
+ if (in.readBoolean()) {
+ derivedFields = in.readList(DerivedField::new);
+ }
+ }
}
@Override
@@ -367,6 +382,18 @@ public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_2_13_0)) {
out.writeOptionalBoolean(includeNamedQueriesScore);
}
+ if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+ boolean hasDerivedFieldsObject = derivedFieldsObject != null;
+ out.writeBoolean(hasDerivedFieldsObject);
+ if (hasDerivedFieldsObject) {
+ out.writeMap(derivedFieldsObject);
+ }
+ boolean hasDerivedFields = derivedFields != null;
+ out.writeBoolean(hasDerivedFields);
+ if (hasDerivedFields) {
+ out.writeList(derivedFields);
+ }
+ }
}
/**
@@ -972,6 +999,28 @@ public List<ScriptField> scriptFields() {
return scriptFields;
}
+ public Map<String, Object> getDerivedFieldsObject() {
+ return derivedFieldsObject;
+ }
+
+ public List<DerivedField> getDerivedFields() {
+ return derivedFields;
+ }
+
+ /**
+ * Adds a derived field with the given name with provided type and script
+ * @param name name of the derived field
+ * @param type type of the derived field
+ * @param script script associated with derived field
+ */
+ public SearchSourceBuilder derivedField(String name, String type, Script script) {
+ if (derivedFields == null) {
+ derivedFields = new ArrayList<>();
+ }
+ derivedFields.add(new DerivedField(name, type, script));
+ return this;
+ }
+
/**
* Sets the boost a specific index or alias will receive when the query is executed
* against it.
@@ -1151,6 +1200,8 @@ private SearchSourceBuilder shallowCopy(
rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm;
rewrittenBuilder.collapse = collapse;
rewrittenBuilder.pointInTimeBuilder = pointInTimeBuilder;
+ rewrittenBuilder.derivedFieldsObject = derivedFieldsObject;
+ rewrittenBuilder.derivedFields = derivedFields;
return rewrittenBuilder;
}
@@ -1298,6 +1349,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th
pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser);
} else if (SEARCH_PIPELINE.match(currentFieldName, parser.getDeprecationHandler())) {
searchPipelineSource = parser.mapOrdered();
+ } else if (DERIVED_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+ derivedFieldsObject = parser.map();
} else {
throw new ParsingException(
parser.getTokenLocation(),
@@ -1530,6 +1583,21 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
if (searchPipelineSource != null) {
builder.field(SEARCH_PIPELINE.getPreferredName(), searchPipelineSource);
}
+
+ if (derivedFieldsObject != null || derivedFields != null) {
+ builder.startObject(DERIVED_FIELDS_FIELD.getPreferredName());
+ if (derivedFieldsObject != null) {
+ builder.mapContents(derivedFieldsObject);
+ }
+ if (derivedFields != null) {
+ for (DerivedField derivedField : derivedFields) {
+ derivedField.toXContent(builder, params);
+ }
+ }
+ builder.endObject();
+
+ }
+
return builder;
}
@@ -1805,7 +1873,9 @@ public int hashCode() {
extBuilders,
collapse,
trackTotalHitsUpTo,
- pointInTimeBuilder
+ pointInTimeBuilder,
+ derivedFieldsObject,
+ derivedFields
);
}
@@ -1848,7 +1918,9 @@ public boolean equals(Object obj) {
&& Objects.equals(extBuilders, other.extBuilders)
&& Objects.equals(collapse, other.collapse)
&& Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo)
- && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder);
+ && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder)
+ && Objects.equals(derivedFieldsObject, other.derivedFieldsObject)
+ && Objects.equals(derivedFields, other.derivedFields);
}
@Override
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightPhase.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightPhase.java
index 2fc9b214e3ebb..b16f06e7e3989 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightPhase.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightPhase.java
@@ -145,6 +145,9 @@ private Map<String, Function<HitContext, FieldHighlightContext>> contextBuilders
boolean fieldNameContainsWildcards = field.field().contains("*");
for (String fieldName : fieldNamesToHighlight) {
MappedFieldType fieldType = context.mapperService().fieldType(fieldName);
+ if (fieldType == null && context.getQueryShardContext().getDerivedFieldType(fieldName) != null) {
+ fieldType = context.getQueryShardContext().getDerivedFieldType(fieldName);
+ }
if (fieldType == null) {
continue;
}
@@ -170,12 +173,13 @@ private Map<String, Function<HitContext, FieldHighlightContext>> contextBuilders
Query highlightQuery = field.fieldOptions().highlightQuery();
boolean forceSource = highlightContext.forceSource(field);
+ MappedFieldType finalFieldType = fieldType;
builders.put(
fieldName,
hc -> new FieldHighlightContext(
- fieldType.name(),
+ finalFieldType.name(),
field,
- fieldType,
+ finalFieldType,
context,
hc,
highlightQuery == null ? query : highlightQuery,
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java
index 2238554a12149..55df80e7d7aa9 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java
@@ -35,6 +35,7 @@
import org.apache.lucene.search.highlight.Encoder;
import org.apache.lucene.search.highlight.SimpleHTMLEncoder;
import org.opensearch.index.fieldvisitor.CustomFieldsVisitor;
+import org.opensearch.index.mapper.DerivedFieldValueFetcher;
import org.opensearch.index.mapper.MappedFieldType;
import org.opensearch.index.mapper.ValueFetcher;
import org.opensearch.index.query.QueryShardContext;
@@ -77,6 +78,9 @@ public static List<Object> loadFieldValues(
return textsToHighlight != null ? textsToHighlight : Collections.emptyList();
}
ValueFetcher fetcher = fieldType.valueFetcher(context, null, null);
+ if (fetcher instanceof DerivedFieldValueFetcher) {
+ fetcher.setNextReader(hitContext.reader().getContext());
+ }
return fetcher.fetchValues(hitContext.sourceLookup());
}
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
index df85246a84d54..c791c8bc05054 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
@@ -48,6 +48,7 @@
import org.opensearch.common.CheckedSupplier;
import org.opensearch.core.common.Strings;
import org.opensearch.core.common.text.Text;
+import org.opensearch.index.mapper.DerivedFieldType;
import org.opensearch.index.mapper.DocumentMapper;
import org.opensearch.index.mapper.IdFieldMapper;
import org.opensearch.index.mapper.MappedFieldType;
@@ -159,6 +160,10 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) th
Integer fieldMaxAnalyzedOffset = fieldContext.field.fieldOptions().maxAnalyzerOffset();
int numberOfFragments = fieldContext.field.fieldOptions().numberOfFragments();
Analyzer analyzer = getAnalyzer(fieldContext.context.mapperService().documentMapper());
+ if (fieldContext.context.getQueryShardContext().getDerivedFieldType(fieldContext.fieldName) != null) {
+ analyzer = ((DerivedFieldType) fieldContext.context.getQueryShardContext().getDerivedFieldType(fieldContext.fieldName))
+ .getIndexAnalyzer();
+ }
if (fieldMaxAnalyzedOffset != null) {
analyzer = getLimitedOffsetAnalyzer(analyzer, fieldMaxAnalyzedOffset);
}
diff --git a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java
index bd6d7b88ade28..1307028dd27b0 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java
@@ -19,8 +19,10 @@
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
+import org.opensearch.common.collect.Tuple;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.core.index.Index;
+import org.opensearch.geometry.Rectangle;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.script.DerivedFieldScript;
@@ -32,6 +34,7 @@
import org.mockito.Mockito;
+import static org.opensearch.index.query.QueryBuilders.geoShapeQuery;
import static org.mockito.Mockito.when;
public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
@@ -40,13 +43,14 @@ public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
// Raw Message, Request Succeeded (boolean), Timestamp (long), Client IP, Method, Request Size (double), Duration (long)
private static final Object[][] raw_requests = new Object[][] {
{
- "40.135.0.0 GET /images/hm_bg.jpg?size=1.5KB HTTP/1.0 200 2024-03-20T08:30:45 1500",
+ "40.135.0.0 GET /images/hm_bg.jpg?size=1.5KB loc 10.0 20.0 HTTP/1.0 200 2024-03-20T08:30:45 1500",
true,
1710923445000L,
"40.135.0.0",
"GET",
1.5,
- 1500L },
+ 1500L,
+ new Tuple<>(10.0, 20.0) },
{
"232.0.0.0 GET /images/hm_bg.jpg?size=2.3KB HTTP/1.0 400 2024-03-20T09:15:20 2300",
false,
@@ -54,7 +58,8 @@ public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
"232.0.0.0",
"GET",
2.3,
- 2300L },
+ 2300L,
+ new Tuple<>(20.0, 30.0) },
{
"26.1.0.0 DELETE /images/hm_bg.jpg?size=3.7KB HTTP/1.0 200 2024-03-20T10:05:55 3700",
true,
@@ -62,7 +67,8 @@ public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
"26.1.0.0",
"DELETE",
3.7,
- 3700L },
+ 3700L,
+ new Tuple<>(30.0, 40.0) },
{
"247.37.0.0 GET /french/splash_inet.html?size=4.1KB HTTP/1.0 400 2024-03-20T11:20:10 4100",
false,
@@ -70,7 +76,8 @@ public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
"247.37.0.0",
"GET",
4.1,
- 4100L },
+ 4100L,
+ new Tuple<>(40.0, 50.0) },
{
"247.37.0.0 DELETE /french/splash_inet.html?size=5.8KB HTTP/1.0 400 2024-03-20T12:45:30 5800",
false,
@@ -78,7 +85,8 @@ public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
"247.37.0.0",
"DELETE",
5.8,
- 5800L },
+ 5800L,
+ new Tuple<>(50.0, 60.0) },
{
"10.20.30.40 GET /path/to/resource?size=6.3KB HTTP/1.0 200 2024-03-20T13:10:15 6300",
true,
@@ -86,7 +94,8 @@ public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
"10.20.30.40",
"GET",
6.3,
- 6300L },
+ 6300L,
+ new Tuple<>(60.0, 70.0) },
{
"50.60.70.80 GET /path/to/resource?size=7.2KB HTTP/1.0 404 2024-03-20T14:20:50 7200",
false,
@@ -94,7 +103,8 @@ public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
"50.60.70.80",
"GET",
7.2,
- 7200L },
+ 7200L,
+ new Tuple<>(70.0, 80.0) },
{
"127.0.0.1 PUT /path/to/resource?size=8.9KB HTTP/1.0 500 2024-03-20T15:30:25 8900",
false,
@@ -102,7 +112,8 @@ public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
"127.0.0.1",
"PUT",
8.9,
- 8900L },
+ 8900L,
+ new Tuple<>(80.0, 90.0) },
{
"127.0.0.1 GET /path/to/resource?size=9.4KB HTTP/1.0 200 2024-03-20T16:40:15 9400",
true,
@@ -110,7 +121,8 @@ public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
"127.0.0.1",
"GET",
9.4,
- 9400L },
+ 9400L,
+ new Tuple<>(85.0, 90.0) },
{
"192.168.1.1 GET /path/to/resource?size=10.7KB HTTP/1.0 400 2024-03-20T17:50:40 10700",
false,
@@ -118,7 +130,8 @@ public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
"192.168.1.1",
"GET",
10.7,
- 10700L } };
+ 10700L,
+ new Tuple<>(90.0, 90.0) } };
public void testAllPossibleQueriesOnDerivedFields() throws IOException {
MapperService mapperService = createMapperService(topMapping(b -> {
@@ -169,6 +182,12 @@ public void testAllPossibleQueriesOnDerivedFields() throws IOException {
b.field("script", "");
}
b.endObject();
+ b.startObject("geopoint");
+ {
+ b.field("type", "geo_point");
+ b.field("script", "");
+ }
+ b.endObject();
}
b.endObject();
}));
@@ -273,6 +292,13 @@ public void execute() {
query = QueryBuilders.regexpQuery("method", ".*LET.*").toQuery(queryShardContext);
topDocs = searcher.search(query, 10);
assertEquals(2, topDocs.totalHits.value);
+
+ // GeoPoint Query
+ scriptIndex[0] = 7;
+
+ query = geoShapeQuery("geopoint", new Rectangle(0.0, 55.0, 55.0, 0.0)).toQuery(queryShardContext);
+ topDocs = searcher.search(query, 10);
+ assertEquals(4, topDocs.totalHits.value);
}
}
}
diff --git a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldTypeTests.java
index 72fb7c88cc478..897848008fd5f 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldTypeTests.java
@@ -15,6 +15,7 @@
import org.apache.lucene.document.LatLonPoint;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.LongPoint;
+import org.opensearch.common.collect.Tuple;
import org.opensearch.script.Script;
import java.util.List;
@@ -28,9 +29,7 @@ private DerivedFieldType createDerivedFieldType(String type) {
Mapper.BuilderContext context = mock(Mapper.BuilderContext.class);
when(context.path()).thenReturn(new ContentPath());
return new DerivedFieldType(
- type + " _derived_field",
- type,
- new Script(""),
+ new DerivedField(type + " _derived_field", type, new Script("")),
DerivedFieldSupportedTypes.getFieldMapperFromType(type, type + "_derived_field", context),
DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(type, type + "_derived_field")
);
@@ -53,7 +52,7 @@ public void testDateType() {
public void testGeoPointType() {
DerivedFieldType dft = createDerivedFieldType("geo_point");
assertTrue(dft.typeFieldMapper instanceof GeoPointFieldMapper);
- assertTrue(dft.indexableFieldGenerator.apply(List.of(10.0, 20.0)) instanceof LatLonPoint);
+ assertTrue(dft.indexableFieldGenerator.apply(new Tuple<>(10.0, 20.0)) instanceof LatLonPoint);
expectThrows(ClassCastException.class, () -> dft.indexableFieldGenerator.apply(List.of(10.0)));
expectThrows(ClassCastException.class, () -> dft.indexableFieldGenerator.apply(List.of()));
expectThrows(ClassCastException.class, () -> dft.indexableFieldGenerator.apply(List.of("10")));
diff --git a/server/src/test/java/org/opensearch/index/query/DerivedFieldQueryTests.java b/server/src/test/java/org/opensearch/index/query/DerivedFieldQueryTests.java
index 1bb303a874b9a..5a11ebebb312e 100644
--- a/server/src/test/java/org/opensearch/index/query/DerivedFieldQueryTests.java
+++ b/server/src/test/java/org/opensearch/index/query/DerivedFieldQueryTests.java
@@ -22,6 +22,7 @@
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.opensearch.common.lucene.Lucene;
+import org.opensearch.index.mapper.DerivedFieldSupportedTypes;
import org.opensearch.index.mapper.DerivedFieldValueFetcher;
import org.opensearch.script.DerivedFieldScript;
import org.opensearch.script.Script;
@@ -75,14 +76,17 @@ public void execute() {
// Create ValueFetcher from mocked DerivedFieldScript.Factory
DerivedFieldScript.LeafFactory leafFactory = factory.newFactory((new Script("")).getParams(), searchLookup);
- DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(leafFactory);
+ DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(
+ leafFactory,
+ null,
+ DerivedFieldSupportedTypes.getIndexableFieldGeneratorType("keyword", "ip_from_raw_request")
+ );
// Create DerivedFieldQuery
DerivedFieldQuery derivedFieldQuery = new DerivedFieldQuery(
new TermQuery(new Term("ip_from_raw_request", "247.37.0.0")),
valueFetcher,
searchLookup,
- (o -> new KeywordField("ip_from_raw_request", (String) o, Field.Store.NO)),
Lucene.STANDARD_ANALYZER
);
diff --git a/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java
index 1a2ad49a3f334..6a7bf10835ddd 100644
--- a/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java
+++ b/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java
@@ -31,6 +31,7 @@
package org.opensearch.index.query;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
@@ -63,10 +64,17 @@
import org.opensearch.index.fielddata.LeafFieldData;
import org.opensearch.index.fielddata.ScriptDocValues;
import org.opensearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
+import org.opensearch.index.mapper.ContentPath;
+import org.opensearch.index.mapper.DerivedField;
+import org.opensearch.index.mapper.DerivedFieldMapper;
+import org.opensearch.index.mapper.DocumentMapper;
import org.opensearch.index.mapper.IndexFieldMapper;
import org.opensearch.index.mapper.MappedFieldType;
+import org.opensearch.index.mapper.Mapper;
import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.MappingLookup;
import org.opensearch.index.mapper.TextFieldMapper;
+import org.opensearch.script.Script;
import org.opensearch.search.lookup.LeafDocLookup;
import org.opensearch.search.lookup.LeafSearchLookup;
import org.opensearch.search.lookup.SearchLookup;
@@ -77,6 +85,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.function.BiFunction;
import java.util.function.Supplier;
@@ -118,6 +127,28 @@ public void testFailIfFieldMappingNotFound() {
assertThat(result.name(), equalTo("name"));
}
+ public void testDerivedFieldMapping() {
+ QueryShardContext context = createQueryShardContext(IndexMetadata.INDEX_UUID_NA_VALUE, null);
+ assertNull(context.failIfFieldMappingNotFound("test_derived", null));
+ context.setDerivedFieldTypes(null);
+ assertNull(context.failIfFieldMappingNotFound("test_derived", null));
+ DocumentMapper documentMapper = mock(DocumentMapper.class);
+ Mapper.BuilderContext builderContext = new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0));
+ DerivedFieldMapper derivedFieldMapper = new DerivedFieldMapper.Builder(new DerivedField("test_derived", "keyword", new Script("")))
+ .build(builderContext);
+ MappingLookup mappingLookup = new MappingLookup(
+ Collections.singletonList(derivedFieldMapper),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ 0,
+ new StandardAnalyzer()
+ );
+ when(documentMapper.mappers()).thenReturn(mappingLookup);
+ context.setDerivedFieldTypes(Map.of("test_derived", derivedFieldMapper.fieldType()));
+ context.setAllowUnmappedFields(false);
+ assertEquals(derivedFieldMapper.fieldType(), context.failIfFieldMappingNotFound("test_derived", null));
+ }
+
public void testToQueryFails() {
QueryShardContext context = createQueryShardContext(IndexMetadata.INDEX_UUID_NA_VALUE, null);
Exception exc = expectThrows(Exception.class, () -> context.toQuery(new AbstractQueryBuilder() {
diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
index bdbec62ab3e71..ba1047d3bd349 100644
--- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
@@ -70,6 +70,7 @@
import org.opensearch.index.IndexService;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.engine.Engine;
+import org.opensearch.index.mapper.DerivedFieldType;
import org.opensearch.index.query.AbstractQueryBuilder;
import org.opensearch.index.query.MatchAllQueryBuilder;
import org.opensearch.index.query.MatchNoneQueryBuilder;
@@ -546,6 +547,49 @@ public void testMaxDocvalueFieldsSearch() throws IOException {
}
}
+ public void testDerivedFieldsSearch() throws IOException {
+ createIndex("index");
+ final SearchService service = getInstanceFromNode(SearchService.class);
+ final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+ final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
+ final IndexShard indexShard = indexService.getShard(0);
+
+ SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
+ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+ searchRequest.source(searchSourceBuilder);
+
+ for (int i = 0; i < 5; i++) {
+ searchSourceBuilder.derivedField(
+ "field" + i,
+ "date",
+ new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap())
+ );
+ }
+ final ShardSearchRequest request = new ShardSearchRequest(
+ OriginalIndices.NONE,
+ searchRequest,
+ indexShard.shardId(),
+ 1,
+ new AliasFilter(null, Strings.EMPTY_ARRAY),
+ 1.0f,
+ -1,
+ null,
+ null
+ );
+
+ try (ReaderContext reader = createReaderContext(indexService, indexShard)) {
+ try (SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
+ assertNotNull(context);
+ for (int i = 0; i < 5; i++) {
+ DerivedFieldType derivedFieldType = (DerivedFieldType) context.getQueryShardContext().getDerivedFieldType("field" + i);
+ assertEquals("field" + i, derivedFieldType.name());
+ assertEquals("date", derivedFieldType.getType());
+ }
+ assertNull(context.getQueryShardContext().getDerivedFieldType("field" + 5));
+ }
+ }
+ }
+
/**
* test that getting more than the allowed number of script_fields throws an exception
*/
diff --git a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
index 3d0e5c3eaf1c0..eea7b1829e9b0 100644
--- a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
+++ b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
@@ -53,6 +53,7 @@
import org.opensearch.index.query.QueryRewriteContext;
import org.opensearch.index.query.RandomQueryBuilder;
import org.opensearch.index.query.Rewriteable;
+import org.opensearch.script.Script;
import org.opensearch.search.AbstractSearchTestCase;
import org.opensearch.search.rescore.QueryRescorerBuilder;
import org.opensearch.search.sort.FieldSortBuilder;
@@ -311,6 +312,51 @@ public void testParseSort() throws IOException {
}
}
+ public void testDerivedFieldsParsingAndSerialization() throws IOException {
+ {
+ String restContent = "{\n"
+ + " \"derived\": {\n"
+ + " \"duration\": {\n"
+ + " \"type\": \"long\",\n"
+ + " \"script\": \"emit(doc['test'])\"\n"
+ + " },\n"
+ + " \"ip_from_message\": {\n"
+ + " \"type\": \"keyword\",\n"
+ + " \"script\": \"emit(doc['message'])\"\n"
+ + " }\n"
+ + " },\n"
+ + " \"query\" : {\n"
+ + " \"match\": { \"content\": { \"query\": \"foo bar\" }}\n"
+ + " }\n"
+ + "}";
+
+ String expectedContent =
+ "{\"query\":{\"match\":{\"content\":{\"query\":\"foo bar\",\"operator\":\"OR\",\"prefix_length\":0,\"max_expansions\":50,\"fuzzy_transpositions\":true,\"lenient\":false,\"zero_terms_query\":\"NONE\",\"auto_generate_synonyms_phrase_query\":true,\"boost\":1.0}}},"
+ + "\"derived\":{"
+ + "\"duration\":{\"type\":\"long\",\"script\":\"emit(doc['test'])\"},\"ip_from_message\":{\"type\":\"keyword\",\"script\":\"emit(doc['message'])\"},\"derived_field\":{\"type\":\"keyword\",\"script\":{\"source\":\"emit(doc['message']\",\"lang\":\"painless\"}}}}";
+
+ try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+ SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser);
+ searchSourceBuilder.derivedField("derived_field", "keyword", new Script("emit(doc['message']"));
+ searchSourceBuilder = rewrite(searchSourceBuilder);
+ assertEquals(2, searchSourceBuilder.getDerivedFieldsObject().size());
+ assertEquals(1, searchSourceBuilder.getDerivedFields().size());
+
+ try (BytesStreamOutput output = new BytesStreamOutput()) {
+ searchSourceBuilder.writeTo(output);
+ try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+ SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in);
+ String actualContent = deserializedBuilder.toString();
+ assertEquals(expectedContent, actualContent);
+ assertEquals(searchSourceBuilder.hashCode(), deserializedBuilder.hashCode());
+ assertNotSame(searchSourceBuilder, deserializedBuilder);
+ }
+ }
+ }
+ }
+
+ }
+
public void testAggsParsing() throws IOException {
{
String restContent = "{\n"
diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java
new file mode 100644
index 0000000000000..28d97c74d9445
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java
@@ -0,0 +1,366 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.fetch.subphase.highlight;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.opensearch.Version;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.document.DocumentField;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.IndexService;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.mapper.ContentPath;
+import org.opensearch.index.mapper.DerivedField;
+import org.opensearch.index.mapper.DerivedFieldSupportedTypes;
+import org.opensearch.index.mapper.DerivedFieldType;
+import org.opensearch.index.mapper.Mapper;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.SourceToParse;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.query.Rewriteable;
+import org.opensearch.script.MockScriptEngine;
+import org.opensearch.script.Script;
+import org.opensearch.script.ScriptEngine;
+import org.opensearch.script.ScriptModule;
+import org.opensearch.script.ScriptService;
+import org.opensearch.script.ScriptType;
+import org.opensearch.search.SearchHit;
+import org.opensearch.search.fetch.FetchContext;
+import org.opensearch.search.fetch.FetchSubPhase;
+import org.opensearch.search.fetch.FetchSubPhaseProcessor;
+import org.opensearch.search.fetch.subphase.FieldAndFormat;
+import org.opensearch.search.fetch.subphase.FieldFetcher;
+import org.opensearch.search.internal.ContextIndexSearcher;
+import org.opensearch.test.OpenSearchSingleNodeTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DerivedFieldFetchAndHighlightTests extends OpenSearchSingleNodeTestCase {
+ private static String DERIVED_FIELD_SCRIPT_1 = "derived_field_script_1";
+ private static String DERIVED_FIELD_SCRIPT_2 = "derived_field_script_2";
+
+ private static String DERIVED_FIELD_1 = "derived_1";
+ private static String DERIVED_FIELD_2 = "derived_2";
+
+ public void testDerivedFieldFromIndexMapping() throws IOException {
+ // Create index and mapper service
+ // Define mapping for derived fields, create 2 derived fields derived_1 and derived_2
+ XContentBuilder mapping = XContentFactory.jsonBuilder()
+ .startObject()
+ .startObject("derived")
+ .startObject(DERIVED_FIELD_1)
+ .field("type", "keyword")
+ .startObject("script")
+ .field("source", DERIVED_FIELD_SCRIPT_1)
+ .field("lang", "mockscript")
+ .endObject()
+ .endObject()
+ .startObject(DERIVED_FIELD_2)
+ .field("type", "keyword")
+ .startObject("script")
+ .field("source", DERIVED_FIELD_SCRIPT_2)
+ .field("lang", "mockscript")
+ .endObject()
+ .endObject()
+ .endObject()
+ .endObject();
+
+ // Create source document with 2 fields field1 and field2.
+ // derived_1 will act on field1 and derived_2 will act on derived_2. DERIVED_FIELD_SCRIPT_1 substitutes whitespaces with _
+ XContentBuilder source = XContentFactory.jsonBuilder()
+ .startObject()
+ .field("field1", "some_text_1")
+ .field("field2", "some_text_2")
+ .endObject();
+
+ int docId = 0;
+ IndexService indexService = createIndex("test_index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping);
+ MapperService mapperService = indexService.mapperService();
+
+ try (
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, new IndexWriterConfig(mapperService.indexAnalyzer()));
+ ) {
+ iw.addDocument(
+ mapperService.documentMapper()
+ .parse(new SourceToParse("test_index", "0", BytesReference.bytes(source), MediaTypeRegistry.JSON))
+ .rootDoc()
+ );
+ try (IndexReader reader = iw.getReader()) {
+ IndexSearcher searcher = newSearcher(reader);
+ LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+ QueryShardContext mockShardContext = createQueryShardContext(mapperService, searcher);
+ mockShardContext.lookup().source().setSegmentAndDocument(context, docId);
+ // mockShardContext.setDerivedFieldTypes(Map.of("derived_2", createDerivedFieldType("derived_1", "keyword"), "derived_1",
+
+ // Assert the fetch phase works for both of the derived fields
+ Map<String, DocumentField> fields = fetchFields(mockShardContext, context, "*");
+
+ // Validate FetchPhase
+ {
+ assertEquals(fields.size(), 2);
+ assertEquals(1, fields.get(DERIVED_FIELD_1).getValues().size());
+ assertEquals(1, fields.get(DERIVED_FIELD_2).getValues().size());
+ assertEquals("some_text_1", fields.get(DERIVED_FIELD_1).getValue());
+ assertEquals("some_text_2", fields.get(DERIVED_FIELD_2).getValue());
+ }
+
+ // Create a HighlightBuilder of type unified, set its fields as derived_1 and derived_2
+ HighlightBuilder highlightBuilder = new HighlightBuilder();
+ highlightBuilder.highlighterType("unified");
+ highlightBuilder.field(DERIVED_FIELD_1);
+ highlightBuilder.field(DERIVED_FIELD_2);
+ highlightBuilder = Rewriteable.rewrite(highlightBuilder, mockShardContext);
+ SearchHighlightContext searchHighlightContext = highlightBuilder.build(mockShardContext);
+
+ // Create a HighlightPhase with highlighter defined above
+ HighlightPhase highlightPhase = new HighlightPhase(Collections.singletonMap("unified", new UnifiedHighlighter()));
+
+ // create a fetch context to be used by HighlightPhase processor
+ FetchContext fetchContext = mock(FetchContext.class);
+ when(fetchContext.mapperService()).thenReturn(mockShardContext.getMapperService());
+ when(fetchContext.getQueryShardContext()).thenReturn(mockShardContext);
+ when(fetchContext.getIndexSettings()).thenReturn(indexService.getIndexSettings());
+ when(fetchContext.searcher()).thenReturn(
+ new ContextIndexSearcher(
+ searcher.getIndexReader(),
+ searcher.getSimilarity(),
+ searcher.getQueryCache(),
+ searcher.getQueryCachingPolicy(),
+ true,
+ searcher.getExecutor(),
+ null
+ )
+ );
+
+ // The query used by FetchSubPhaseProcessor to highlight is a term query on DERIVED_FIELD_1
+ FetchSubPhaseProcessor subPhaseProcessor = highlightPhase.getProcessor(
+ fetchContext,
+ searchHighlightContext,
+ new TermQuery(new Term(DERIVED_FIELD_1, "some_text_1"))
+ );
+
+ // Create a search hit using the derived fields fetched above in fetch phase
+ SearchHit searchHit = new SearchHit(docId, "0", null, fields, null);
+
+ // Create a HitContext of search hit
+ FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(
+ searchHit,
+ context,
+ docId,
+ mockShardContext.lookup().source()
+ );
+ hitContext.sourceLookup().loadSourceIfNeeded();
+ // process the HitContext using the highlightPhase subPhaseProcessor
+ subPhaseProcessor.process(hitContext);
+
+ // Validate that 1 highlight field is present
+ assertEquals(hitContext.hit().getHighlightFields().size(), 1);
+ }
+ }
+ }
+
+ public void testDerivedFieldFromSearchMapping() throws IOException {
+ // Create source document with 2 fields field1 and field2.
+ // derived_1 will act on field1 and derived_2 will act on derived_2. DERIVED_FIELD_SCRIPT_1 substitutes whitespaces with _
+ XContentBuilder source = XContentFactory.jsonBuilder()
+ .startObject()
+ .field("field1", "some_text_1")
+ .field("field2", "some_text_2")
+ .endObject();
+
+ int docId = 0;
+
+ // Create index and mapper service
+ // We are not defining derived fields in index mapping here
+ XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().endObject();
+ IndexService indexService = createIndex("test_index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping);
+ MapperService mapperService = indexService.mapperService();
+
+ try (
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, new IndexWriterConfig(mapperService.indexAnalyzer()));
+ ) {
+ iw.addDocument(
+ mapperService.documentMapper()
+ .parse(new SourceToParse("test_index", "0", BytesReference.bytes(source), MediaTypeRegistry.JSON))
+ .rootDoc()
+ );
+ try (IndexReader reader = iw.getReader()) {
+ IndexSearcher searcher = newSearcher(reader);
+ LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+ QueryShardContext mockShardContext = createQueryShardContext(mapperService, searcher);
+ mockShardContext.lookup().source().setSegmentAndDocument(context, docId);
+
+ // This mock behavior is similar to adding derived fields in search request
+ mockShardContext.setDerivedFieldTypes(
+ Map.of(
+ DERIVED_FIELD_1,
+ createDerivedFieldType(DERIVED_FIELD_1, "keyword", DERIVED_FIELD_SCRIPT_1),
+ DERIVED_FIELD_2,
+ createDerivedFieldType(DERIVED_FIELD_2, "keyword", DERIVED_FIELD_SCRIPT_2)
+ )
+ );
+
+ // Assert the fetch phase works for both of the derived fields
+ Map<String, DocumentField> fields = fetchFields(mockShardContext, context, "derived_*");
+
+ // Validate FetchPhase
+ {
+ assertEquals(fields.size(), 2);
+ assertEquals(1, fields.get(DERIVED_FIELD_1).getValues().size());
+ assertEquals(1, fields.get(DERIVED_FIELD_2).getValues().size());
+ assertEquals("some_text_1", fields.get(DERIVED_FIELD_1).getValue());
+ assertEquals("some_text_2", fields.get(DERIVED_FIELD_2).getValue());
+ }
+
+ // Create a HighlightBuilder of type unified, set its fields as derived_1 and derived_2
+ HighlightBuilder highlightBuilder = new HighlightBuilder();
+ highlightBuilder.highlighterType("unified");
+ highlightBuilder.field(DERIVED_FIELD_1);
+ highlightBuilder.field(DERIVED_FIELD_2);
+ highlightBuilder = Rewriteable.rewrite(highlightBuilder, mockShardContext);
+ SearchHighlightContext searchHighlightContext = highlightBuilder.build(mockShardContext);
+
+ // Create a HighlightPhase with highlighter defined above
+ HighlightPhase highlightPhase = new HighlightPhase(Collections.singletonMap("unified", new UnifiedHighlighter()));
+
+ // create a fetch context to be used by HighlightPhase processor
+ FetchContext fetchContext = mock(FetchContext.class);
+ when(fetchContext.mapperService()).thenReturn(mockShardContext.getMapperService());
+ when(fetchContext.getQueryShardContext()).thenReturn(mockShardContext);
+ when(fetchContext.getIndexSettings()).thenReturn(indexService.getIndexSettings());
+ when(fetchContext.searcher()).thenReturn(
+ new ContextIndexSearcher(
+ searcher.getIndexReader(),
+ searcher.getSimilarity(),
+ searcher.getQueryCache(),
+ searcher.getQueryCachingPolicy(),
+ true,
+ searcher.getExecutor(),
+ null
+ )
+ );
+
+ // The query used by FetchSubPhaseProcessor to highlight is a term query on DERIVED_FIELD_1
+ FetchSubPhaseProcessor subPhaseProcessor = highlightPhase.getProcessor(
+ fetchContext,
+ searchHighlightContext,
+ new TermQuery(new Term(DERIVED_FIELD_1, "some_text_1"))
+ );
+
+ // Create a search hit using the derived fields fetched above in fetch phase
+ SearchHit searchHit = new SearchHit(docId, "0", null, fields, null);
+
+ // Create a HitContext of search hit
+ FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(
+ searchHit,
+ context,
+ docId,
+ mockShardContext.lookup().source()
+ );
+ hitContext.sourceLookup().loadSourceIfNeeded();
+ // process the HitContext using the highlightPhase subPhaseProcessor
+ subPhaseProcessor.process(hitContext);
+
+ // Validate that 1 highlight field is present
+ assertEquals(hitContext.hit().getHighlightFields().size(), 1);
+ }
+ }
+ }
+
+ public static Map<String, DocumentField> fetchFields(
+ QueryShardContext queryShardContext,
+ LeafReaderContext context,
+ String fieldPattern
+ ) throws IOException {
+ List<FieldAndFormat> fields = List.of(new FieldAndFormat(fieldPattern, null));
+ FieldFetcher fieldFetcher = FieldFetcher.create(queryShardContext, queryShardContext.lookup(), fields);
+ fieldFetcher.setNextReader(context);
+ return fieldFetcher.fetch(queryShardContext.lookup().source(), Set.of());
+ }
+
+ private static QueryShardContext createQueryShardContext(MapperService mapperService, IndexSearcher indexSearcher) {
+ Settings settings = Settings.builder()
+ .put("index.version.created", Version.CURRENT)
+ .put("index.number_of_shards", 1)
+ .put("index.number_of_replicas", 0)
+ .put(IndexMetadata.SETTING_INDEX_UUID, "uuid")
+ .build();
+ IndexMetadata indexMetadata = new IndexMetadata.Builder("index").settings(settings).build();
+ IndexSettings indexSettings = new IndexSettings(indexMetadata, settings);
+
+ ScriptService scriptService = getScriptService();
+ return new QueryShardContext(
+ 0,
+ indexSettings,
+ null,
+ null,
+ null,
+ mapperService,
+ null,
+ scriptService,
+ null,
+ null,
+ null,
+ indexSearcher,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+ }
+
+ private static ScriptService getScriptService() {
+ final MockScriptEngine engine = new MockScriptEngine(
+ MockScriptEngine.NAME,
+ Map.of(
+ DERIVED_FIELD_SCRIPT_1,
+ (script) -> ((String) ((Map<String, Object>) script.get("_source")).get("field1")).replace(" ", "_"),
+ DERIVED_FIELD_SCRIPT_2,
+ (script) -> ((String) ((Map<String, Object>) script.get("_source")).get("field2")).replace(" ", "_")
+ ),
+ Collections.emptyMap()
+ );
+ final Map<String, ScriptEngine> engines = singletonMap(engine.getType(), engine);
+ ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+ return scriptService;
+ }
+
+ private DerivedFieldType createDerivedFieldType(String name, String type, String script) {
+ Mapper.BuilderContext context = mock(Mapper.BuilderContext.class);
+ when(context.path()).thenReturn(new ContentPath());
+ return new DerivedFieldType(
+ new DerivedField(name, type, new Script(ScriptType.INLINE, "mockscript", script, emptyMap())),
+ DerivedFieldSupportedTypes.getFieldMapperFromType(type, name, context),
+ DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(type, name)
+ );
+ }
+}
diff --git a/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java
index 456b55883f91e..048f0acb60cde 100644
--- a/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java
@@ -35,6 +35,7 @@
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Scorable;
+import org.opensearch.common.collect.Tuple;
import org.opensearch.index.query.IntervalFilterScript;
import org.opensearch.index.similarity.ScriptedSimilarity.Doc;
import org.opensearch.index.similarity.ScriptedSimilarity.Field;
@@ -43,8 +44,10 @@
import org.opensearch.search.aggregations.pipeline.MovingFunctionScript;
import org.opensearch.search.lookup.LeafSearchLookup;
import org.opensearch.search.lookup.SearchLookup;
+import org.opensearch.search.lookup.SourceLookup;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -282,16 +285,39 @@ public double execute(Map<String, Object> params1, double[] values) {
IntervalFilterScript.Factory factory = mockCompiled::createIntervalFilterScript;
return context.factoryClazz.cast(factory);
} else if (context.instanceClazz.equals(DerivedFieldScript.class)) {
- DerivedFieldScript.Factory factory = (derivedFieldsParams, lookup) -> ctx -> new DerivedFieldScript(
- derivedFieldsParams,
- lookup,
- ctx
- ) {
+ DerivedFieldScript.Factory factory = new DerivedFieldScript.Factory() {
@Override
- public void execute() {
- Map<String, Object> vars = new HashMap<>(derivedFieldsParams);
- vars.put("params", derivedFieldsParams);
- script.apply(vars);
+ public boolean isResultDeterministic() {
+ return true;
+ }
+
+ @Override
+ public DerivedFieldScript.LeafFactory newFactory(Map<String, Object> derivedFieldParams, SearchLookup lookup) {
+ return ctx -> new DerivedFieldScript(derivedFieldParams, lookup, ctx) {
+ @Override
+ public void execute() {
+ Map<String, Object> vars = new HashMap<>(derivedFieldParams);
+ SourceLookup sourceLookup = lookup.source();
+ vars.put("params", derivedFieldParams);
+ vars.put("_source", sourceLookup.loadSourceIfNeeded());
+ Object result = script.apply(vars);
+ if (result instanceof ArrayList) {
+ for (Object v : ((ArrayList<?>) result)) {
+ if (v instanceof HashMap) {
+ addEmittedValue(new Tuple(((HashMap<?, ?>) v).get("lat"), ((HashMap<?, ?>) v).get("lon")));
+ } else {
+ addEmittedValue(v);
+ }
+ }
+ } else {
+ if (result instanceof HashMap) {
+ addEmittedValue(new Tuple(((HashMap<?, ?>) result).get("lat"), ((HashMap<?, ?>) result).get("lon")));
+ } else {
+ addEmittedValue(result);
+ }
+ }
+ }
+ };
}
};
return context.factoryClazz.cast(factory);