Skip to content

Commit 645b1f1

Browse files
rishabhmauryarishabhmaurya
authored
[Derived Fields] PR4: Capability to define derived fields in search request (opensearch-project#12850)
* Support derived fields definition in search request * adds support for fetch phase on derived fields * adds support for highlighting on derived fields --------- Signed-off-by: Rishabh Maurya <rishabhmaurya05@gmail.com>
1 parent 4fc3a02 commit 645b1f1

26 files changed

+1373
-215
lines changed

‎client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java

+229
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,19 @@
5454
import org.opensearch.action.search.SearchScrollRequest;
5555
import org.opensearch.client.core.CountRequest;
5656
import org.opensearch.client.core.CountResponse;
57+
import org.opensearch.common.geo.ShapeRelation;
5758
import org.opensearch.common.unit.TimeValue;
5859
import org.opensearch.common.xcontent.XContentFactory;
5960
import org.opensearch.core.common.bytes.BytesReference;
6061
import org.opensearch.core.rest.RestStatus;
6162
import org.opensearch.core.xcontent.MediaTypeRegistry;
6263
import org.opensearch.core.xcontent.XContentBuilder;
64+
import org.opensearch.geometry.Rectangle;
65+
import org.opensearch.index.query.GeoShapeQueryBuilder;
6366
import org.opensearch.index.query.MatchQueryBuilder;
6467
import org.opensearch.index.query.QueryBuilder;
6568
import org.opensearch.index.query.QueryBuilders;
69+
import org.opensearch.index.query.RangeQueryBuilder;
6670
import org.opensearch.index.query.ScriptQueryBuilder;
6771
import org.opensearch.index.query.TermsQueryBuilder;
6872
import org.opensearch.join.aggregations.Children;
@@ -102,6 +106,8 @@
102106
import org.opensearch.search.suggest.Suggest;
103107
import org.opensearch.search.suggest.SuggestBuilder;
104108
import org.opensearch.search.suggest.phrase.PhraseSuggestionBuilder;
109+
import org.joda.time.DateTime;
110+
import org.joda.time.DateTimeZone;
105111
import org.hamcrest.Matchers;
106112
import org.junit.Before;
107113

@@ -116,6 +122,7 @@
116122
import java.util.concurrent.TimeUnit;
117123

118124
import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
125+
import static org.opensearch.index.query.QueryBuilders.geoShapeQuery;
119126
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent;
120127
import static org.hamcrest.Matchers.arrayContaining;
121128
import static org.hamcrest.Matchers.both;
@@ -764,6 +771,228 @@ public void testSearchWithWeirdScriptFields() throws Exception {
764771
}
765772
}
766773

774+
public void testSearchWithDerivedFields() throws Exception {
775+
// Just testing DerivedField definition from SearchSourceBuilder derivedField()
776+
// We are not testing the full functionality here
777+
Request doc = new Request("PUT", "test/_doc/1");
778+
doc.setJsonEntity("{\"field\":\"value\"}");
779+
client().performRequest(doc);
780+
client().performRequest(new Request("POST", "/test/_refresh"));
781+
// Keyword field
782+
{
783+
SearchRequest searchRequest = new SearchRequest("test").source(
784+
SearchSourceBuilder.searchSource()
785+
.derivedField("result", "keyword", new Script("emit(params._source[\"field\"])"))
786+
.fetchField("result")
787+
.query(new TermsQueryBuilder("result", "value"))
788+
);
789+
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
790+
SearchHit searchHit = searchResponse.getHits().getAt(0);
791+
List<Object> values = searchHit.getFields().get("result").getValues();
792+
assertNotNull(values);
793+
assertEquals(1, values.size());
794+
assertEquals("value", values.get(0));
795+
796+
// multi valued
797+
searchRequest = new SearchRequest("test").source(
798+
SearchSourceBuilder.searchSource()
799+
.derivedField(
800+
"result",
801+
"keyword",
802+
new Script("emit(params._source[\"field\"]);emit(params._source[\"field\"] + \"_2\")")
803+
)
804+
.query(new TermsQueryBuilder("result", "value_2"))
805+
.fetchField("result")
806+
);
807+
searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
808+
searchHit = searchResponse.getHits().getAt(0);
809+
values = searchHit.getFields().get("result").getValues();
810+
assertNotNull(values);
811+
assertEquals(2, values.size());
812+
assertEquals("value", values.get(0));
813+
assertEquals("value_2", values.get(1));
814+
}
815+
// Boolean field
816+
{
817+
SearchRequest searchRequest = new SearchRequest("test").source(
818+
SearchSourceBuilder.searchSource()
819+
.derivedField("result", "boolean", new Script("emit(((String)params._source[\"field\"]).equals(\"value\"))"))
820+
.query(new TermsQueryBuilder("result", "true"))
821+
.fetchField("result")
822+
);
823+
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
824+
SearchHit searchHit = searchResponse.getHits().getAt(0);
825+
List<Object> values = searchHit.getFields().get("result").getValues();
826+
assertNotNull(values);
827+
assertEquals(1, values.size());
828+
assertEquals(true, values.get(0));
829+
}
830+
// Long field
831+
{
832+
SearchRequest searchRequest = new SearchRequest("test").source(
833+
SearchSourceBuilder.searchSource()
834+
.derivedField("result", "long", new Script("emit(Long.MAX_VALUE)"))
835+
.query(new RangeQueryBuilder("result").from(Long.MAX_VALUE - 1).to(Long.MAX_VALUE))
836+
.fetchField("result")
837+
);
838+
839+
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
840+
SearchHit searchHit = searchResponse.getHits().getAt(0);
841+
List<Object> values = searchHit.getFields().get("result").getValues();
842+
assertNotNull(values);
843+
assertEquals(1, values.size());
844+
assertEquals(Long.MAX_VALUE, values.get(0));
845+
846+
// multi-valued
847+
searchRequest = new SearchRequest("test").source(
848+
SearchSourceBuilder.searchSource()
849+
.derivedField("result", "long", new Script("emit(Long.MAX_VALUE); emit(Long.MIN_VALUE);"))
850+
.query(new RangeQueryBuilder("result").from(Long.MIN_VALUE).to(Long.MIN_VALUE + 1))
851+
.fetchField("result")
852+
);
853+
854+
searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
855+
searchHit = searchResponse.getHits().getAt(0);
856+
values = searchHit.getFields().get("result").getValues();
857+
assertNotNull(values);
858+
assertEquals(2, values.size());
859+
assertEquals(Long.MAX_VALUE, values.get(0));
860+
assertEquals(Long.MIN_VALUE, values.get(1));
861+
}
862+
// Double field
863+
{
864+
SearchRequest searchRequest = new SearchRequest("test").source(
865+
SearchSourceBuilder.searchSource()
866+
.derivedField("result", "double", new Script("emit(Double.MAX_VALUE)"))
867+
.query(new RangeQueryBuilder("result").from(Double.MAX_VALUE - 1).to(Double.MAX_VALUE))
868+
.fetchField("result")
869+
);
870+
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
871+
SearchHit searchHit = searchResponse.getHits().getAt(0);
872+
List<Object> values = searchHit.getFields().get("result").getValues();
873+
assertNotNull(values);
874+
assertEquals(1, values.size());
875+
assertEquals(Double.MAX_VALUE, values.get(0));
876+
877+
// multi-valued
878+
searchRequest = new SearchRequest("test").source(
879+
SearchSourceBuilder.searchSource()
880+
.derivedField("result", "double", new Script("emit(Double.MAX_VALUE); emit(Double.MIN_VALUE);"))
881+
.query(new RangeQueryBuilder("result").from(Double.MIN_VALUE).to(Double.MIN_VALUE + 1))
882+
.fetchField("result")
883+
);
884+
885+
searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
886+
searchHit = searchResponse.getHits().getAt(0);
887+
values = searchHit.getFields().get("result").getValues();
888+
assertNotNull(values);
889+
assertEquals(2, values.size());
890+
assertEquals(Double.MAX_VALUE, values.get(0));
891+
assertEquals(Double.MIN_VALUE, values.get(1));
892+
}
893+
// Date field
894+
{
895+
DateTime date1 = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC);
896+
DateTime date2 = new DateTime(1990, 12, 30, 0, 0, DateTimeZone.UTC);
897+
SearchRequest searchRequest = new SearchRequest("test").source(
898+
SearchSourceBuilder.searchSource()
899+
.derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L)"))
900+
.query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
901+
.fetchField("result")
902+
);
903+
904+
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
905+
SearchHit searchHit = searchResponse.getHits().getAt(0);
906+
List<Object> values = searchHit.getFields().get("result").getValues();
907+
assertNotNull(values);
908+
assertEquals(1, values.size());
909+
assertEquals(date1.toString(), values.get(0));
910+
911+
// multi-valued
912+
searchRequest = new SearchRequest("test").source(
913+
SearchSourceBuilder.searchSource()
914+
.derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L); " + "emit(" + date2.getMillis() + "L)"))
915+
.query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
916+
.fetchField("result")
917+
);
918+
919+
searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
920+
searchHit = searchResponse.getHits().getAt(0);
921+
values = searchHit.getFields().get("result").getValues();
922+
assertNotNull(values);
923+
assertEquals(2, values.size());
924+
assertEquals(date1.toString(), values.get(0));
925+
assertEquals(date2.toString(), values.get(1));
926+
}
927+
// Geo field
928+
{
929+
GeoShapeQueryBuilder qb = geoShapeQuery("result", new Rectangle(-35, 35, 35, -35));
930+
qb.relation(ShapeRelation.INTERSECTS);
931+
SearchRequest searchRequest = new SearchRequest("test").source(
932+
SearchSourceBuilder.searchSource()
933+
.derivedField("result", "geo_point", new Script("emit(10.0, 20.0)"))
934+
.query(qb)
935+
.fetchField("result")
936+
);
937+
938+
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
939+
SearchHit searchHit = searchResponse.getHits().getAt(0);
940+
List<Object> values = searchHit.getFields().get("result").getValues();
941+
assertNotNull(values);
942+
assertEquals(1, values.size());
943+
assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
944+
assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));
945+
946+
// multi-valued
947+
searchRequest = new SearchRequest("test").source(
948+
SearchSourceBuilder.searchSource()
949+
.derivedField("result", "geo_point", new Script("emit(10.0, 20.0); emit(20.0, 30.0);"))
950+
.query(qb)
951+
.fetchField("result")
952+
);
953+
954+
searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
955+
searchHit = searchResponse.getHits().getAt(0);
956+
values = searchHit.getFields().get("result").getValues();
957+
assertNotNull(values);
958+
assertEquals(2, values.size());
959+
assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
960+
assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));
961+
assertEquals(20.0, ((HashMap) values.get(1)).get("lat"));
962+
assertEquals(30.0, ((HashMap) values.get(1)).get("lon"));
963+
}
964+
// IP field
965+
{
966+
SearchRequest searchRequest = new SearchRequest("test").source(
967+
SearchSourceBuilder.searchSource().derivedField("result", "ip", new Script("emit(\"10.0.0.1\")")).fetchField("result")
968+
);
969+
970+
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
971+
SearchHit searchHit = searchResponse.getHits().getAt(0);
972+
List<Object> values = searchHit.getFields().get("result").getValues();
973+
assertNotNull(values);
974+
assertEquals(1, values.size());
975+
assertEquals("10.0.0.1", values.get(0));
976+
977+
// multi-valued
978+
searchRequest = new SearchRequest("test").source(
979+
SearchSourceBuilder.searchSource()
980+
.derivedField("result", "ip", new Script("emit(\"10.0.0.1\"); emit(\"10.0.0.2\");"))
981+
.fetchField("result")
982+
);
983+
984+
searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
985+
searchHit = searchResponse.getHits().getAt(0);
986+
values = searchHit.getFields().get("result").getValues();
987+
assertNotNull(values);
988+
assertEquals(2, values.size());
989+
assertEquals("10.0.0.1", values.get(0));
990+
assertEquals("10.0.0.2", values.get(1));
991+
992+
}
993+
994+
}
995+
767996
public void testSearchScroll() throws Exception {
768997
for (int i = 0; i < 100; i++) {
769998
XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();

0 commit comments

Comments
 (0)