Add test for fetch and highlight phase for derived fields

Signed-off-by: Rishabh Maurya <rishabhmaurya05@gmail.com>

Author: rishabhmaurya

server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java

@@ -36,7 +36,7 @@
  * it is used to create an IndexableField for the provided type and object. It is useful when indexing into
  * lucene MemoryIndex in {@link org.opensearch.index.query.DerivedFieldQuery}.
  */
-enum DerivedFieldSupportedTypes {
+public enum DerivedFieldSupportedTypes {
 
     BOOLEAN("boolean", (name, context) -> {
         BooleanFieldMapper.Builder builder = new BooleanFieldMapper.Builder(name);

server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java

@@ -83,7 +83,7 @@ public DerivedFieldValueFetcher valueFetcher(QueryShardContext context, SearchLo
         if (format != null) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
         }
-        return new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context, searchLookup));
+        return new DerivedFieldValueFetcher(getDerivedFieldLeafFactory(context, searchLookup == null ? context.lookup() : searchLookup));
     }
 
     @Override

server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java

@@ -127,16 +127,12 @@ public boolean equals(Object o) {
             return false;
         }
         DerivedFieldQuery other = (DerivedFieldQuery) o;
-        return Objects.equals(this.query, other.query)
-            && Objects.equals(this.valueFetcher, other.valueFetcher)
-            && Objects.equals(this.searchLookup, other.searchLookup)
-            && Objects.equals(this.indexableFieldGenerator, other.indexableFieldGenerator)
-            && Objects.equals(this.indexAnalyzer, other.indexAnalyzer);
+        return Objects.equals(this.query, other.query) && Objects.equals(this.indexAnalyzer, other.indexAnalyzer);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(classHash(), query, valueFetcher, searchLookup, indexableFieldGenerator, indexableFieldGenerator);
+        return Objects.hash(classHash(), query, indexAnalyzer);
     }
 
     @Override

server/src/main/java/org/opensearch/index/query/QueryShardContext.java

@@ -45,6 +45,7 @@
 import org.opensearch.common.TriFunction;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.search.Queries;
+import org.opensearch.common.regex.Regex;
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.ParsingException;
@@ -332,7 +333,15 @@ public Map<String, Query> copyNamedQueries() {
      * type then the fields will be returned with a type prefix.
      */
     public Set<String> simpleMatchToIndexNames(String pattern) {
-        return mapperService.simpleMatchToFullName(pattern);
+        Set<String> matchingFields = mapperService.simpleMatchToFullName(pattern);
+        if (derivedFieldTypeMap != null && !derivedFieldTypeMap.isEmpty()) {
+            for (String fieldName : derivedFieldTypeMap.keySet()) {
+                if (Regex.simpleMatch(pattern, fieldName)) {
+                    matchingFields.add(fieldName);
+                }
+            }
+        }
+        return matchingFields;
     }
 
     /**

server/src/test/java/org/opensearch/search/fetch/subphase/FieldFetcherTests.java

@@ -43,34 +43,20 @@
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.query.QueryShardContext;
-import org.opensearch.script.MockScriptEngine;
-import org.opensearch.script.ScriptEngine;
-import org.opensearch.script.ScriptModule;
-import org.opensearch.script.ScriptService;
-import org.opensearch.search.lookup.LeafSearchLookup;
-import org.opensearch.search.lookup.SearchLookup;
 import org.opensearch.search.lookup.SourceLookup;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItems;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class FieldFetcherTests extends OpenSearchSingleNodeTestCase {
 
-    private static String DERIVED_FIELD_SCRIPT_1 = "derived_field_script_1";
-    private static String DERIVED_FIELD_SCRIPT_2 = "derived_field_script_2";
-
     public void testLeafValues() throws IOException {
         MapperService mapperService = createMapperService();
         XContentBuilder source = XContentFactory.jsonBuilder()
@@ -449,45 +435,6 @@ public void testTextSubFields() throws IOException {
         }
     }
 
-    public void testDerivedFields() throws IOException {
-        XContentBuilder mapping = XContentFactory.jsonBuilder()
-            .startObject()
-            .startObject("derived")
-            .startObject("derived_1")
-            .field("type", "keyword")
-            .startObject("script")
-            .field("source", DERIVED_FIELD_SCRIPT_1)
-            .field("lang", "mockscript")
-            .endObject()
-            .endObject()
-            .startObject("derived_2")
-            .field("type", "keyword")
-            .startObject("script")
-            .field("source", DERIVED_FIELD_SCRIPT_2)
-            .field("lang", "mockscript")
-            .endObject()
-            .endObject()
-            .endObject()
-            .endObject();
-
-        IndexService indexService = createIndex("index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping);
-        MapperService mapperService = indexService.mapperService();
-
-        XContentBuilder source = XContentFactory.jsonBuilder()
-            .startObject()
-            .field("field1", "some text 1")
-            .field("field2", "some text 2")
-            .endObject();
-
-        Map<String, DocumentField> fields = fetchFields(mapperService, source, "*");
-        assertThat(fields.size(), equalTo(2));
-        assertThat(fields.keySet(), containsInAnyOrder("derived_1", "derived_2"));
-        assertThat(fields.get("derived_1").getValues().size(), equalTo(1));
-        assertThat(fields.get("derived_2").getValues().size(), equalTo(1));
-        assertThat(fields.get("derived_1").getValue(), equalTo("some text 1"));
-        assertThat(fields.get("derived_2").getValue(), equalTo("some text 2"));
-    }
-
     private static Map<String, DocumentField> fetchFields(MapperService mapperService, XContentBuilder source, String fieldPattern)
         throws IOException {
 
@@ -501,13 +448,7 @@ private static Map<String, DocumentField> fetchFields(MapperService mapperServic
         SourceLookup sourceLookup = new SourceLookup();
         sourceLookup.setSource(BytesReference.bytes(source));
 
-        SearchLookup searchLookup = mock(SearchLookup.class);
-        LeafSearchLookup leafSearchLookup = mock(LeafSearchLookup.class);
-        when(searchLookup.source()).thenReturn(sourceLookup);
-        when(searchLookup.getLeafSearchLookup(any())).thenReturn(leafSearchLookup);
-        when(leafSearchLookup.source()).thenReturn(sourceLookup);
-        FieldFetcher fieldFetcher = FieldFetcher.create(createQueryShardContext(mapperService), searchLookup, fields);
-        fieldFetcher.setNextReader(null);
+        FieldFetcher fieldFetcher = FieldFetcher.create(createQueryShardContext(mapperService), null, fields);
         return fieldFetcher.fetch(sourceLookup, Set.of());
     }
 
@@ -556,19 +497,6 @@ private static QueryShardContext createQueryShardContext(MapperService mapperSer
             .build();
         IndexMetadata indexMetadata = new IndexMetadata.Builder("index").settings(settings).build();
         IndexSettings indexSettings = new IndexSettings(indexMetadata, settings);
-
-        final MockScriptEngine engine = new MockScriptEngine(
-            MockScriptEngine.NAME,
-            Map.of(
-                DERIVED_FIELD_SCRIPT_1,
-                (script) -> ((Map<String, Object>) script.get("_source")).get("field1"),
-                DERIVED_FIELD_SCRIPT_2,
-                (script) -> ((Map<String, Object>) script.get("_source")).get("field2")
-            ),
-            Collections.emptyMap()
-        );
-        final Map<String, ScriptEngine> engines = singletonMap(engine.getType(), engine);
-        ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
         return new QueryShardContext(
             0,
             indexSettings,
@@ -577,7 +505,7 @@ private static QueryShardContext createQueryShardContext(MapperService mapperSer
             null,
             mapperService,
             null,
-            scriptService,
+            null,
             null,
             null,
             null,