Skip to content

Commit 41a3055

Browse files
gaobinlonggaobinlong
authored
Rename ingest processor supports overriding target field if exists (opensearch-project#12990)
* Rename ingest processor supports overriding target field if exists Signed-off-by: Gao Binlong <gbinlong@amazon.com> * Modify change log Signed-off-by: Gao Binlong <gbinlong@amazon.com> --------- Signed-off-by: Gao Binlong <gbinlong@amazon.com>
1 parent 2aad499 commit 41a3055

File tree

6 files changed

+96
-26
lines changed

6 files changed

+96
-26
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
1111
- Allow setting KEYSTORE_PASSWORD through env variable ([#12865](https://github.com/opensearch-project/OpenSearch/pull/12865))
1212
- [Concurrent Segment Search] Perform buildAggregation concurrently and support Composite Aggregations ([#12697](https://github.com/opensearch-project/OpenSearch/pull/12697))
1313
- [Concurrent Segment Search] Disable concurrent segment search for system indices and throttled requests ([#12954](https://github.com/opensearch-project/OpenSearch/pull/12954))
14+
- Rename ingest processor supports overriding target field if exists ([#12990](https://github.com/opensearch-project/OpenSearch/pull/12990))
1415
- [Tiered Caching] Make took time caching policy setting dynamic ([#13063](https://github.com/opensearch-project/OpenSearch/pull/13063))
1516
- Derived fields support to derive field values at query time without indexing ([#12569](https://github.com/opensearch-project/OpenSearch/pull/12569))
1617
- Detect breaking changes on pull requests ([#9044](https://github.com/opensearch-project/OpenSearch/pull/9044))

‎modules/ingest-common/src/main/java/org/opensearch/ingest/common/RenameProcessor.java

+13-4
Original file line numberDiff line numberDiff line change
@@ -52,18 +52,21 @@ public final class RenameProcessor extends AbstractProcessor {
5252
private final TemplateScript.Factory field;
5353
private final TemplateScript.Factory targetField;
5454
private final boolean ignoreMissing;
55+
private final boolean overrideTarget;
5556

5657
RenameProcessor(
5758
String tag,
5859
String description,
5960
TemplateScript.Factory field,
6061
TemplateScript.Factory targetField,
61-
boolean ignoreMissing
62+
boolean ignoreMissing,
63+
boolean overrideTarget
6264
) {
6365
super(tag, description);
6466
this.field = field;
6567
this.targetField = targetField;
6668
this.ignoreMissing = ignoreMissing;
69+
this.overrideTarget = overrideTarget;
6770
}
6871

6972
TemplateScript.Factory getField() {
@@ -78,6 +81,10 @@ boolean isIgnoreMissing() {
7881
return ignoreMissing;
7982
}
8083

84+
boolean isOverrideTarget() {
85+
return overrideTarget;
86+
}
87+
8188
@Override
8289
public IngestDocument execute(IngestDocument document) {
8390
String path = document.renderTemplate(field);
@@ -94,9 +101,10 @@ public IngestDocument execute(IngestDocument document) {
94101
// We fail here if the target field point to an array slot that is out of range.
95102
// If we didn't do this then we would fail if we set the value in the target_field
96103
// and then on failure processors would not see that value we tried to rename as we already
97-
// removed it.
104+
// removed it. If the target field is out of range, we throw the exception no matter
105+
// what the parameter overrideTarget is.
98106
String target = document.renderTemplate(targetField);
99-
if (document.hasField(target, true)) {
107+
if (document.hasField(target, true) && !overrideTarget) {
100108
throw new IllegalArgumentException("field [" + target + "] already exists");
101109
}
102110

@@ -143,7 +151,8 @@ public RenameProcessor create(
143151
scriptService
144152
);
145153
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
146-
return new RenameProcessor(processorTag, description, fieldTemplate, targetFieldTemplate, ignoreMissing);
154+
boolean overrideTarget = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "override_target", false);
155+
return new RenameProcessor(processorTag, description, fieldTemplate, targetFieldTemplate, ignoreMissing, overrideTarget);
147156
}
148157
}
149158
}

‎modules/ingest-common/src/test/java/org/opensearch/ingest/common/DotExpanderProcessorTests.java

+1
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ public void testEscapeFields_valueField() throws Exception {
105105
null,
106106
new TestTemplateService.MockTemplateScript.Factory("foo"),
107107
new TestTemplateService.MockTemplateScript.Factory("foo.bar"),
108+
false,
108109
false
109110
);
110111
processor.execute(document);

‎modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorFactoryTests.java

+13
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,19 @@ public void testCreateWithIgnoreMissing() throws Exception {
7777
assertThat(renameProcessor.isIgnoreMissing(), equalTo(true));
7878
}
7979

80+
public void testCreateWithOverrideTarget() throws Exception {
81+
Map<String, Object> config = new HashMap<>();
82+
config.put("field", "old_field");
83+
config.put("target_field", "new_field");
84+
config.put("override_target", true);
85+
String processorTag = randomAlphaOfLength(10);
86+
RenameProcessor renameProcessor = factory.create(null, processorTag, null, config);
87+
assertThat(renameProcessor.getTag(), equalTo(processorTag));
88+
assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
89+
assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
90+
assertThat(renameProcessor.isOverrideTarget(), equalTo(true));
91+
}
92+
8093
public void testCreateNoFieldPresent() throws Exception {
8194
Map<String, Object> config = new HashMap<>();
8295
config.put("target_field", "new_field");

‎modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorTests.java

+29-22
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public void testRename() throws Exception {
5959
do {
6060
newFieldName = RandomDocumentPicks.randomFieldName(random());
6161
} while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName));
62-
Processor processor = createRenameProcessor(fieldName, newFieldName, false);
62+
Processor processor = createRenameProcessor(fieldName, newFieldName, false, false);
6363
processor.execute(ingestDocument);
6464
assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue));
6565
}
@@ -77,7 +77,7 @@ public void testRenameArrayElement() throws Exception {
7777
document.put("one", one);
7878
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
7979

80-
Processor processor = createRenameProcessor("list.0", "item", false);
80+
Processor processor = createRenameProcessor("list.0", "item", false, false);
8181
processor.execute(ingestDocument);
8282
Object actualObject = ingestDocument.getSourceAndMetadata().get("list");
8383
assertThat(actualObject, instanceOf(List.class));
@@ -90,7 +90,7 @@ public void testRenameArrayElement() throws Exception {
9090
assertThat(actualObject, instanceOf(String.class));
9191
assertThat(actualObject, equalTo("item1"));
9292

93-
processor = createRenameProcessor("list.0", "list.3", false);
93+
processor = createRenameProcessor("list.0", "list.3", false, randomBoolean());
9494
try {
9595
processor.execute(ingestDocument);
9696
fail("processor execute should have failed");
@@ -105,7 +105,7 @@ public void testRenameArrayElement() throws Exception {
105105
public void testRenameNonExistingField() throws Exception {
106106
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
107107
String fieldName = RandomDocumentPicks.randomFieldName(random());
108-
Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), false);
108+
Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), false, false);
109109
try {
110110
processor.execute(ingestDocument);
111111
fail("processor execute should have failed");
@@ -114,7 +114,7 @@ public void testRenameNonExistingField() throws Exception {
114114
}
115115

116116
// when using template snippet, the resolved field path maybe empty
117-
processor = createRenameProcessor("", RandomDocumentPicks.randomFieldName(random()), false);
117+
processor = createRenameProcessor("", RandomDocumentPicks.randomFieldName(random()), false, false);
118118
try {
119119
processor.execute(ingestDocument);
120120
fail("processor execute should have failed");
@@ -127,38 +127,44 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
127127
IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
128128
IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
129129
String fieldName = RandomDocumentPicks.randomFieldName(random());
130-
Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), true);
130+
Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), true, false);
131131
processor.execute(ingestDocument);
132132
assertIngestDocument(originalIngestDocument, ingestDocument);
133133

134134
// when using template snippet, the resolved field path maybe empty
135-
processor = createRenameProcessor("", RandomDocumentPicks.randomFieldName(random()), true);
135+
processor = createRenameProcessor("", RandomDocumentPicks.randomFieldName(random()), true, false);
136136
processor.execute(ingestDocument);
137137
assertIngestDocument(originalIngestDocument, ingestDocument);
138138
}
139139

140140
public void testRenameNewFieldAlreadyExists() throws Exception {
141141
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
142-
String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
143-
Processor processor = createRenameProcessor(
144-
RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument),
145-
fieldName,
146-
false
147-
);
142+
String field = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
143+
Object fieldValue = ingestDocument.getFieldValue(field, Object.class);
144+
String targetField = RandomDocumentPicks.addRandomField(random(), ingestDocument, RandomDocumentPicks.randomFieldValue(random()));
145+
146+
Processor processor = createRenameProcessor(field, targetField, false, false);
148147
try {
149148
processor.execute(ingestDocument);
150149
fail("processor execute should have failed");
151150
} catch (IllegalArgumentException e) {
152-
assertThat(e.getMessage(), equalTo("field [" + fieldName + "] already exists"));
151+
assertThat(e.getMessage(), equalTo("field [" + targetField + "] already exists"));
153152
}
153+
154+
Processor processorWithOverrideTarget = createRenameProcessor(field, targetField, false, true);
155+
156+
processorWithOverrideTarget.execute(ingestDocument);
157+
assertThat(ingestDocument.hasField(field), equalTo(false));
158+
assertThat(ingestDocument.hasField(targetField), equalTo(true));
159+
assertThat(ingestDocument.getFieldValue(targetField, Object.class), equalTo(fieldValue));
154160
}
155161

156162
public void testRenameExistingFieldNullValue() throws Exception {
157163
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
158164
String fieldName = RandomDocumentPicks.randomFieldName(random());
159165
ingestDocument.setFieldValue(fieldName, null);
160166
String newFieldName = randomValueOtherThanMany(ingestDocument::hasField, () -> RandomDocumentPicks.randomFieldName(random()));
161-
Processor processor = createRenameProcessor(fieldName, newFieldName, false);
167+
Processor processor = createRenameProcessor(fieldName, newFieldName, false, false);
162168
processor.execute(ingestDocument);
163169
if (newFieldName.startsWith(fieldName + '.')) {
164170
assertThat(ingestDocument.getFieldValue(fieldName, Object.class), instanceOf(Map.class));
@@ -182,7 +188,7 @@ public Object put(String key, Object value) {
182188
source.put("list", Collections.singletonList("item"));
183189

184190
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
185-
Processor processor = createRenameProcessor("list", "new_field", false);
191+
Processor processor = createRenameProcessor("list", "new_field", false, false);
186192
try {
187193
processor.execute(ingestDocument);
188194
fail("processor execute should have failed");
@@ -206,7 +212,7 @@ public Object remove(Object key) {
206212
source.put("list", Collections.singletonList("item"));
207213

208214
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
209-
Processor processor = createRenameProcessor("list", "new_field", false);
215+
Processor processor = createRenameProcessor("list", "new_field", false, false);
210216
try {
211217
processor.execute(ingestDocument);
212218
fail("processor execute should have failed");
@@ -221,12 +227,12 @@ public void testRenameLeafIntoBranch() throws Exception {
221227
Map<String, Object> source = new HashMap<>();
222228
source.put("foo", "bar");
223229
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
224-
Processor processor1 = createRenameProcessor("foo", "foo.bar", false);
230+
Processor processor1 = createRenameProcessor("foo", "foo.bar", false, false);
225231
processor1.execute(ingestDocument);
226232
assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));
227233
assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar"));
228234

229-
Processor processor2 = createRenameProcessor("foo.bar", "foo.bar.baz", false);
235+
Processor processor2 = createRenameProcessor("foo.bar", "foo.bar.baz", false, false);
230236
processor2.execute(ingestDocument);
231237
assertThat(
232238
ingestDocument.getFieldValue("foo", Map.class),
@@ -236,18 +242,19 @@ public void testRenameLeafIntoBranch() throws Exception {
236242
assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar"));
237243

238244
// for fun lets try to restore it (which don't allow today)
239-
Processor processor3 = createRenameProcessor("foo.bar.baz", "foo", false);
245+
Processor processor3 = createRenameProcessor("foo.bar.baz", "foo", false, false);
240246
Exception e = expectThrows(IllegalArgumentException.class, () -> processor3.execute(ingestDocument));
241247
assertThat(e.getMessage(), equalTo("field [foo] already exists"));
242248
}
243249

244-
private RenameProcessor createRenameProcessor(String field, String targetField, boolean ignoreMissing) {
250+
private RenameProcessor createRenameProcessor(String field, String targetField, boolean ignoreMissing, boolean overrideTarget) {
245251
return new RenameProcessor(
246252
randomAlphaOfLength(10),
247253
null,
248254
new TestTemplateService.MockTemplateScript.Factory(field),
249255
new TestTemplateService.MockTemplateScript.Factory(targetField),
250-
ignoreMissing
256+
ignoreMissing,
257+
overrideTarget
251258
);
252259
}
253260
}

‎modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename_processor.yml

+39
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,42 @@ teardown:
6464
index: test
6565
id: 1
6666
- match: { _source.message: "foo bar baz" }
67+
68+
---
69+
"Test rename processor with override_target":
70+
- skip:
71+
version: " - 2.13.99"
72+
reason: "introduced in 2.14.0"
73+
- do:
74+
ingest.put_pipeline:
75+
id: "my_pipeline"
76+
body: >
77+
{
78+
"description": "_description",
79+
"processors": [
80+
{
81+
"rename" : {
82+
"field" : "foo",
83+
"target_field" : "bar",
84+
"override_target" : true
85+
}
86+
}
87+
]
88+
}
89+
- match: { acknowledged: true }
90+
91+
- do:
92+
index:
93+
index: test
94+
id: 1
95+
pipeline: "my_pipeline"
96+
body: {
97+
foo: "foo",
98+
bar: "bar"
99+
}
100+
101+
- do:
102+
get:
103+
index: test
104+
id: 1
105+
- match: { _source: { "bar": "foo" } }

0 commit comments

Comments
 (0)