Skip to content

Commit 3e3b6b4

Browse files
rishabhmauryaqreshi
rishabhmaurya
authored and
qreshi
committed
Add tests for all supported query types for derived fields
Signed-off-by: Rishabh Maurya <rishabhmaurya05@gmail.com>
1 parent 4c6aaa3 commit 3e3b6b4

File tree

6 files changed

+326
-44
lines changed

6 files changed

+326
-44
lines changed

‎server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java

+12-9
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,7 @@ private static DerivedFieldMapper toType(FieldMapper in) {
5656
*/
5757
public static class Builder extends ParametrizedFieldMapper.Builder {
5858
// TODO: The type of parameter may change here if the actual underlying FieldType object is needed
59-
private final Parameter<String> type = Parameter.stringParam(
60-
"type",
61-
false,
62-
m -> toType(m).type,
63-
"text"
64-
);
59+
private final Parameter<String> type = Parameter.stringParam("type", false, m -> toType(m).type, "text");
6560

6661
private final Parameter<Script> script = new Parameter<>(
6762
"script",
@@ -83,9 +78,17 @@ protected List<Parameter<?>> getParameters() {
8378
@Override
8479
public DerivedFieldMapper build(BuilderContext context) {
8580
FieldMapper fieldMapper = DerivedFieldSupportedTypes.getFieldMapperFromType(type.getValue(), name, context);
86-
Function<Object, IndexableField> fieldFunction =
87-
DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(type.getValue(), name);
88-
DerivedFieldType ft = new DerivedFieldType(buildFullName(context), type.getValue(), script.getValue(), fieldMapper, fieldFunction);
81+
Function<Object, IndexableField> fieldFunction = DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(
82+
type.getValue(),
83+
name
84+
);
85+
DerivedFieldType ft = new DerivedFieldType(
86+
buildFullName(context),
87+
type.getValue(),
88+
script.getValue(),
89+
fieldMapper,
90+
fieldFunction
91+
);
8992
return new DerivedFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this);
9093
}
9194
}

‎server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -721,7 +721,8 @@ public void toXContent(XContentBuilder builder, Params params, ToXContent custom
721721
doXContent(builder, params);
722722

723723
// sort the mappers so we get consistent serialization format
724-
Mapper[] derivedSortedMappers = mappers.values().stream()
724+
Mapper[] derivedSortedMappers = mappers.values()
725+
.stream()
725726
.filter(m -> m instanceof DerivedFieldMapper)
726727
.toArray(size -> new Mapper[size]);
727728
Arrays.sort(derivedSortedMappers, new Comparator<Mapper>() {
@@ -731,7 +732,8 @@ public int compare(Mapper o1, Mapper o2) {
731732
}
732733
});
733734

734-
Mapper[] sortedMappers = mappers.values().stream()
735+
Mapper[] sortedMappers = mappers.values()
736+
.stream()
735737
.filter(m -> !(m instanceof DerivedFieldMapper))
736738
.toArray(size -> new Mapper[size]);
737739
Arrays.sort(sortedMappers, new Comparator<Mapper>() {

‎server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -671,7 +671,9 @@ public final void parse(String name, ParserContext parserContext, Map<String, Ob
671671
}
672672
}
673673
String type = (String) fieldNode.get("type");
674-
if (paramsMap.get("type") == null) { fieldNode.remove("type"); }
674+
if (paramsMap.get("type") == null) {
675+
fieldNode.remove("type");
676+
}
675677

676678
for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
677679
Map.Entry<String, Object> entry = iterator.next();

‎server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java

+279
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.index.mapper;
10+
11+
import org.apache.lucene.document.Document;
12+
import org.apache.lucene.document.Field;
13+
import org.apache.lucene.document.TextField;
14+
import org.apache.lucene.index.DirectoryReader;
15+
import org.apache.lucene.index.IndexReader;
16+
import org.apache.lucene.index.IndexWriter;
17+
import org.apache.lucene.index.IndexWriterConfig;
18+
import org.apache.lucene.search.IndexSearcher;
19+
import org.apache.lucene.search.Query;
20+
import org.apache.lucene.search.TopDocs;
21+
import org.apache.lucene.store.Directory;
22+
import org.opensearch.common.lucene.Lucene;
23+
import org.opensearch.core.index.Index;
24+
import org.opensearch.index.query.QueryBuilders;
25+
import org.opensearch.index.query.QueryShardContext;
26+
import org.opensearch.script.DerivedFieldScript;
27+
28+
import java.io.IOException;
29+
import java.util.ArrayList;
30+
import java.util.List;
31+
import java.util.Set;
32+
33+
import org.mockito.Mockito;
34+
35+
import static org.mockito.Mockito.when;
36+
37+
public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
38+
39+
// First element is the document ingested, other elements corresponds to value returned against a given derived field script
40+
// Raw Message, Request Succeeded (boolean), Timestamp (long), Client IP, Method, Request Size (double), Duration (long)
41+
private static final Object[][] raw_requests = new Object[][] {
42+
{
43+
"40.135.0.0 GET /images/hm_bg.jpg?size=1.5KB HTTP/1.0 200 2024-03-20T08:30:45 1500",
44+
true,
45+
1710923445000L,
46+
"40.135.0.0",
47+
"GET",
48+
1.5,
49+
1500L },
50+
{
51+
"232.0.0.0 GET /images/hm_bg.jpg?size=2.3KB HTTP/1.0 400 2024-03-20T09:15:20 2300",
52+
false,
53+
1710926120000L,
54+
"232.0.0.0",
55+
"GET",
56+
2.3,
57+
2300L },
58+
{
59+
"26.1.0.0 DELETE /images/hm_bg.jpg?size=3.7KB HTTP/1.0 200 2024-03-20T10:05:55 3700",
60+
true,
61+
1710929155000L,
62+
"26.1.0.0",
63+
"DELETE",
64+
3.7,
65+
3700L },
66+
{
67+
"247.37.0.0 GET /french/splash_inet.html?size=4.1KB HTTP/1.0 400 2024-03-20T11:20:10 4100",
68+
false,
69+
1710933610000L,
70+
"247.37.0.0",
71+
"GET",
72+
4.1,
73+
4100L },
74+
{
75+
"247.37.0.0 DELETE /french/splash_inet.html?size=5.8KB HTTP/1.0 400 2024-03-20T12:45:30 5800",
76+
false,
77+
1710938730000L,
78+
"247.37.0.0",
79+
"DELETE",
80+
5.8,
81+
5800L },
82+
{
83+
"10.20.30.40 GET /path/to/resource?size=6.3KB HTTP/1.0 200 2024-03-20T13:10:15 6300",
84+
true,
85+
1710940215000L,
86+
"10.20.30.40",
87+
"GET",
88+
6.3,
89+
6300L },
90+
{
91+
"50.60.70.80 GET /path/to/resource?size=7.2KB HTTP/1.0 404 2024-03-20T14:20:50 7200",
92+
false,
93+
1710944450000L,
94+
"50.60.70.80",
95+
"GET",
96+
7.2,
97+
7200L },
98+
{
99+
"127.0.0.1 PUT /path/to/resource?size=8.9KB HTTP/1.0 500 2024-03-20T15:30:25 8900",
100+
false,
101+
1710948625000L,
102+
"127.0.0.1",
103+
"PUT",
104+
8.9,
105+
8900L },
106+
{
107+
"127.0.0.1 GET /path/to/resource?size=9.4KB HTTP/1.0 200 2024-03-20T16:40:15 9400",
108+
true,
109+
1710952815000L,
110+
"127.0.0.1",
111+
"GET",
112+
9.4,
113+
9400L },
114+
{
115+
"192.168.1.1 GET /path/to/resource?size=10.7KB HTTP/1.0 400 2024-03-20T17:50:40 10700",
116+
false,
117+
1710957040000L,
118+
"192.168.1.1",
119+
"GET",
120+
10.7,
121+
10700L } };
122+
123+
public void testAllPossibleQueriesOnDerivedFields() throws IOException {
124+
MapperService mapperService = createMapperService(topMapping(b -> {
125+
b.startObject("properties");
126+
{
127+
b.startObject("raw_message");
128+
{
129+
b.field("type", "text");
130+
}
131+
b.endObject();
132+
}
133+
b.endObject();
134+
b.startObject("derived");
135+
{
136+
b.startObject("request_succeeded");
137+
{
138+
b.field("type", "boolean");
139+
b.field("script", "");
140+
}
141+
b.endObject();
142+
b.startObject("@timestamp");
143+
{
144+
b.field("type", "date");
145+
b.field("script", "");
146+
}
147+
b.endObject();
148+
b.startObject("client_ip");
149+
{
150+
b.field("type", "ip");
151+
b.field("script", "");
152+
}
153+
b.endObject();
154+
b.startObject("method");
155+
{
156+
b.field("type", "keyword");
157+
b.field("script", "");
158+
}
159+
b.endObject();
160+
b.startObject("request_size");
161+
{
162+
b.field("type", "double");
163+
b.field("script", "");
164+
}
165+
b.endObject();
166+
b.startObject("duration");
167+
{
168+
b.field("type", "long");
169+
b.field("script", "");
170+
}
171+
b.endObject();
172+
}
173+
b.endObject();
174+
}));
175+
176+
List<Document> docs = new ArrayList<>();
177+
for (Object[] request : raw_requests) {
178+
Document document = new Document();
179+
document.add(new TextField("raw_message", (String) request[0], Field.Store.YES));
180+
docs.add(document);
181+
}
182+
183+
int[] scriptIndex = { 1 };
184+
185+
// Mock DerivedFieldScript.Factory
186+
DerivedFieldScript.Factory factory = (params, lookup) -> (DerivedFieldScript.LeafFactory) ctx -> new DerivedFieldScript(
187+
params,
188+
lookup,
189+
ctx
190+
) {
191+
int docId = 0;
192+
193+
@Override
194+
public void setDocument(int docId) {
195+
super.setDocument(docId);
196+
this.docId = docId;
197+
}
198+
199+
@Override
200+
public Object execute() {
201+
return raw_requests[docId][scriptIndex[0]];
202+
}
203+
};
204+
205+
QueryShardContext queryShardContext = createQueryShardContext(mapperService);
206+
when(queryShardContext.compile(Mockito.any(), Mockito.any())).thenReturn(factory);
207+
when(queryShardContext.sourcePath("raw_message")).thenReturn(Set.of("raw_message"));
208+
when(queryShardContext.index()).thenReturn(new Index("test_index", "uuid"));
209+
210+
// Index and Search
211+
try (Directory dir = newDirectory()) {
212+
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
213+
for (Document d : docs) {
214+
iw.addDocument(d);
215+
}
216+
try (IndexReader reader = DirectoryReader.open(iw)) {
217+
iw.close();
218+
219+
IndexSearcher searcher = new IndexSearcher(reader);
220+
Query query = QueryBuilders.termQuery("request_succeeded", "true").toQuery(queryShardContext);
221+
TopDocs topDocs = searcher.search(query, 10);
222+
assertEquals(4, topDocs.totalHits.value);
223+
224+
// IP Field Term Query
225+
scriptIndex[0] = 3;
226+
query = QueryBuilders.termQuery("client_ip", "192.168.0.0/16").toQuery(queryShardContext);
227+
topDocs = searcher.search(query, 10);
228+
assertEquals(1, topDocs.totalHits.value);
229+
230+
scriptIndex[0] = 4;
231+
query = QueryBuilders.termsQuery("method", "DELETE", "PUT").toQuery(queryShardContext);
232+
topDocs = searcher.search(query, 10);
233+
assertEquals(3, topDocs.totalHits.value);
234+
235+
query = QueryBuilders.termsQuery("method", "delete").toQuery(queryShardContext);
236+
topDocs = searcher.search(query, 10);
237+
assertEquals(0, topDocs.totalHits.value);
238+
239+
query = QueryBuilders.termQuery("method", "delete").caseInsensitive(true).toQuery(queryShardContext);
240+
topDocs = searcher.search(query, 10);
241+
assertEquals(2, topDocs.totalHits.value);
242+
243+
// Range queries of types - date, long and double
244+
scriptIndex[0] = 2;
245+
query = QueryBuilders.rangeQuery("@timestamp").from("2024-03-20T14:20:50").toQuery(queryShardContext);
246+
topDocs = searcher.search(query, 10);
247+
assertEquals(4, topDocs.totalHits.value);
248+
249+
scriptIndex[0] = 5;
250+
query = QueryBuilders.rangeQuery("request_size").from("4.1").toQuery(queryShardContext);
251+
topDocs = searcher.search(query, 10);
252+
assertEquals(7, topDocs.totalHits.value);
253+
254+
scriptIndex[0] = 6;
255+
query = QueryBuilders.rangeQuery("duration").from("5800").toQuery(queryShardContext);
256+
topDocs = searcher.search(query, 10);
257+
assertEquals(6, topDocs.totalHits.value);
258+
259+
scriptIndex[0] = 4;
260+
261+
// Prefix Query
262+
query = QueryBuilders.prefixQuery("method", "DE").toQuery(queryShardContext);
263+
topDocs = searcher.search(query, 10);
264+
assertEquals(2, topDocs.totalHits.value);
265+
266+
scriptIndex[0] = 4;
267+
query = QueryBuilders.wildcardQuery("method", "G*").toQuery(queryShardContext);
268+
topDocs = searcher.search(query, 10);
269+
assertEquals(7, topDocs.totalHits.value);
270+
271+
// Regexp Query
272+
scriptIndex[0] = 4;
273+
query = QueryBuilders.regexpQuery("method", ".*LET.*").toQuery(queryShardContext);
274+
topDocs = searcher.search(query, 10);
275+
assertEquals(2, topDocs.totalHits.value);
276+
}
277+
}
278+
}
279+
}

0 commit comments

Comments
 (0)