[DerivedField] object type support in mappings and new settings for derived field (opensearch-project#13717)

rishabhmaurya · web-flow · commit bb013dadc797 · 2024-06-03T13:47:08.000-07:00
---------

Signed-off-by: Rishabh Maurya &lt;rishabhmaurya05@gmail.com&gt;
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedField.java b/server/src/main/java/org/opensearch/index/mapper/DerivedField.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.index.mapper;
 
+import org.opensearch.Version;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -18,17 +19,21 @@
 import org.opensearch.script.Script;
 
 import java.io.IOException;
+import java.util.Map;
 import java.util.Objects;
 
 /**
  * DerivedField representation: expects a name, type and script.
  */
 @PublicApi(since = "2.14.0")
 public class DerivedField implements Writeable, ToXContentFragment {
-
     private final String name;
     private final String type;
     private final Script script;
+    private String sourceIndexedField;
+    private Map<String, Object> properties;
+    private Boolean ignoreMalformed;
+    private String format;
 
     public DerivedField(String name, String type, Script script) {
         this.name = name;
@@ -40,20 +45,51 @@ public DerivedField(StreamInput in) throws IOException {
         name = in.readString();
         type = in.readString();
         script = new Script(in);
+        if (in.getVersion().onOrAfter(Version.V_2_15_0)) {
+            if (in.readBoolean()) {
+                properties = in.readMap();
+            }
+            sourceIndexedField = in.readOptionalString();
+            format = in.readOptionalString();
+            ignoreMalformed = in.readOptionalBoolean();
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeString(type);
         script.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_2_15_0)) {
+            if (properties == null) {
+                out.writeBoolean(false);
+            } else {
+                out.writeBoolean(true);
+                out.writeMap(properties);
+            }
+            out.writeOptionalString(sourceIndexedField);
+            out.writeOptionalString(format);
+            out.writeOptionalBoolean(ignoreMalformed);
+        }
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject(name);
         builder.field("type", type);
         builder.field("script", script);
+        if (properties != null) {
+            builder.field("properties", properties);
+        }
+        if (sourceIndexedField != null) {
+            builder.field("source_indexed_field", sourceIndexedField);
+        }
+        if (format != null) {
+            builder.field("format", format);
+        }
+        if (ignoreMalformed != null) {
+            builder.field("ignore_malformed", ignoreMalformed);
+        }
         builder.endObject();
         return builder;
     }
@@ -70,9 +106,41 @@ public Script getScript() {
         return script;
     }
 
+    public Map<String, Object> getProperties() {
+        return properties;
+    }
+
+    public String getSourceIndexedField() {
+        return sourceIndexedField;
+    }
+
+    public String getFormat() {
+        return format;
+    }
+
+    public boolean getIgnoreMalformed() {
+        return Boolean.TRUE.equals(ignoreMalformed);
+    }
+
+    public void setProperties(Map<String, Object> properties) {
+        this.properties = properties;
+    }
+
+    public void setSourceIndexedField(String sourceIndexedField) {
+        this.sourceIndexedField = sourceIndexedField;
+    }
+
+    public void setFormat(String format) {
+        this.format = format;
+    }
+
+    public void setIgnoreMalformed(boolean ignoreMalformed) {
+        this.ignoreMalformed = ignoreMalformed;
+    }
+
     @Override
     public int hashCode() {
-        return Objects.hash(name, type, script);
+        return Objects.hash(name, type, script, sourceIndexedField, properties, ignoreMalformed, format);
     }
 
     @Override
@@ -84,7 +152,12 @@ public boolean equals(Object obj) {
             return false;
         }
         DerivedField other = (DerivedField) obj;
-        return Objects.equals(name, other.name) && Objects.equals(type, other.type) && Objects.equals(script, other.script);
+        return Objects.equals(name, other.name)
+            && Objects.equals(type, other.type)
+            && Objects.equals(script, other.script)
+            && Objects.equals(sourceIndexedField, other.sourceIndexedField)
+            && Objects.equals(properties, other.properties)
+            && Objects.equals(ignoreMalformed, other.ignoreMalformed)
+            && Objects.equals(format, other.format);
     }
-
 }
diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
@@ -1004,6 +1004,37 @@ public SearchSourceBuilder derivedField(String name, String type, Script script)
         return this;
     }
 
+    /**
+     * Adds a derived field with the given name with provided type, script and other parameters
+     * @param name name of the derived field
+     * @param type type of the derived field
+     * @param script script associated with derived field
+     * @param properties map of field name and type of field for nested fields within object derived field
+     * @param sourceIndexedField source text field which is indexed to filter documents for better performance
+     * @param format date format
+     * @param ignoreMalformed ignores malformed fields instead of failing search request
+     */
+    public SearchSourceBuilder derivedField(
+        String name,
+        String type,
+        Script script,
+        Map<String, Object> properties,
+        String sourceIndexedField,
+        String format,
+        Boolean ignoreMalformed
+    ) {
+        if (derivedFields == null) {
+            derivedFields = new ArrayList<>();
+        }
+        DerivedField derivedField = new DerivedField(name, type, script);
+        derivedField.setProperties(properties);
+        derivedField.setSourceIndexedField(sourceIndexedField);
+        derivedField.setFormat(format);
+        derivedField.setIgnoreMalformed(ignoreMalformed);
+        derivedFields.add(derivedField);
+        return this;
+    }
+
     /**
      * Sets the boost a specific index or alias will receive when the query is executed
      * against it.
diff --git a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
@@ -357,6 +357,70 @@ public void testDerivedFieldsParsingAndSerialization() throws IOException {
 
     }
 
+    public void testDerivedFieldsParsingAndSerializationObjectType() throws IOException {
+        {
+            String restContent = "{\n"
+                + "  \"derived\": {\n"
+                + "    \"duration\": {\n"
+                + "      \"type\": \"long\",\n"
+                + "      \"script\": \"emit(doc['test'])\"\n"
+                + "    },\n"
+                + "    \"ip_from_message\": {\n"
+                + "      \"type\": \"keyword\",\n"
+                + "      \"script\": \"emit(doc['message'])\"\n"
+                + "    },\n"
+                + "    \"object\": {\n"
+                + "      \"type\": \"object\",\n"
+                + "      \"script\": \"emit(doc['test'])\",\n"
+                + "      \"format\": \"dd-MM-yyyy\",\n"
+                + "      \"source_indexed_field\": \"test\",\n"
+                + "      \"ignore_malformed\": true,\n"
+                + "      \"properties\": {\n"
+                + "         \"sub_field\": \"text\"\n"
+                + "       }\n"
+                + "    }\n"
+                + "  },\n"
+                + "    \"query\" : {\n"
+                + "        \"match\": { \"content\": { \"query\": \"foo bar\" }}\n"
+                + "     }\n"
+                + "}";
+
+            String expectedContent =
+                "{\"query\":{\"match\":{\"content\":{\"query\":\"foo bar\",\"operator\":\"OR\",\"prefix_length\":0,\"max_expansions\":50,\"fuzzy_transpositions\":true,\"lenient\":false,\"zero_terms_query\":\"NONE\",\"auto_generate_synonyms_phrase_query\":true,\"boost\":1.0}}},\"derived\":{\"duration\":{\"type\":\"long\",\"script\":\"emit(doc['test'])\"},\"ip_from_message\":{\"type\":\"keyword\",\"script\":\"emit(doc['message'])\"},\"object\":{\"format\":\"dd-MM-yyyy\",\"source_indexed_field\":\"test\",\"ignore_malformed\":true,\"type\":\"object\",\"script\":\"emit(doc['test'])\",\"properties\":{\"sub_field\":\"text\"}},\"derived_field\":{\"type\":\"object\",\"script\":{\"source\":\"emit(doc['message']\",\"lang\":\"painless\"},\"properties\":{\"sub_field_2\":\"keyword\"},\"source_indexed_field\":\"message\",\"format\":\"dd-MM-yyyy\",\"ignore_malformed\":true}}}";
+
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser);
+                searchSourceBuilder.derivedField(
+                    "derived_field",
+                    "object",
+                    new Script("emit(doc['message']"),
+                    Map.of("sub_field_2", "keyword"),
+                    "message",
+                    "dd-MM-yyyy",
+                    true
+                );
+                searchSourceBuilder = rewrite(searchSourceBuilder);
+                assertEquals(3, searchSourceBuilder.getDerivedFieldsObject().size());
+                assertEquals(1, searchSourceBuilder.getDerivedFields().size());
+                assertEquals(1, searchSourceBuilder.getDerivedFields().get(0).getProperties().size());
+                assertEquals("message", searchSourceBuilder.getDerivedFields().get(0).getSourceIndexedField());
+                assertEquals("dd-MM-yyyy", searchSourceBuilder.getDerivedFields().get(0).getFormat());
+                assertTrue(searchSourceBuilder.getDerivedFields().get(0).getIgnoreMalformed());
+
+                try (BytesStreamOutput output = new BytesStreamOutput()) {
+                    searchSourceBuilder.writeTo(output);
+                    try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                        SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in);
+                        String actualContent = deserializedBuilder.toString();
+                        assertEquals(expectedContent, actualContent);
+                        assertEquals(searchSourceBuilder.hashCode(), deserializedBuilder.hashCode());
+                        assertNotSame(searchSourceBuilder, deserializedBuilder);
+                    }
+                }
+            }
+        }
+    }
+
     public void testAggsParsing() throws IOException {
         {
             String restContent = "{\n"
