diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 6ca1543d85c7d..727684b9542b4 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add task completion count in search backpressure stats API ([#10028](https://github.com/opensearch-project/OpenSearch/pull/10028/)) - Deprecate CamelCase `PathHierarchy` tokenizer name in favor to lowercase `path_hierarchy` ([#10894](https://github.com/opensearch-project/OpenSearch/pull/10894)) - Breaking change: Do not request "search_pipelines" metrics by default in NodesInfoRequest ([#12497](https://github.com/opensearch-project/OpenSearch/pull/12497)) +- Refactor `:libs` module `bootstrap` package to eliminate top level split packages [#17117](https://github.com/opensearch-project/OpenSearch/pull/17117)) ### Deprecated @@ -46,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021)) - Remove custom Map, List and Set collection classes ([#6871](https://github.com/opensearch-project/OpenSearch/pull/6871)) - Remove `index.store.hybrid.mmap.extensions` setting in favor of `index.store.hybrid.nio.extensions` setting ([#9392](https://github.com/opensearch-project/OpenSearch/pull/9392)) +- Move o.o.action.support.master classes ([#17104](https://github.com/opensearch-project/OpenSearch/pull/17104)) ### Fixed - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 499405ac508e4..ec7742b8563bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Added a new `time` field to replace the deprecated `getTime` field in `GetStats`. ([#17009](https://github.com/opensearch-project/OpenSearch/pull/17009)) - Improve flat_object field parsing performance by reducing two passes to a single pass ([#16297](https://github.com/opensearch-project/OpenSearch/pull/16297)) - Improve performance of the bitmap filtering([#16936](https://github.com/opensearch-project/OpenSearch/pull/16936/)) +- Added new Setting property UnmodifiableOnRestore to prevent updating settings on restore snapshot ([#16957](https://github.com/opensearch-project/OpenSearch/pull/16957)) +- Introduce Template query ([#16818](https://github.com/opensearch-project/OpenSearch/pull/16818)) +- Propagate the sourceIncludes and excludes fields from fetchSourceContext to FieldsVisitor. ([#17080](https://github.com/opensearch-project/OpenSearch/pull/17080)) ### Dependencies - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504)) @@ -108,7 +111,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix Shallow copy snapshot failures on closed index ([#16868](https://github.com/opensearch-project/OpenSearch/pull/16868)) - Fix multi-value sort for unsigned long ([#16732](https://github.com/opensearch-project/OpenSearch/pull/16732)) - The `phone-search` analyzer no longer emits the tel/sip prefix, international calling code, extension numbers and unformatted input as a token ([#16993](https://github.com/opensearch-project/OpenSearch/pull/16993)) +- Stop processing search requests when _msearch request is cancelled ([#17005](https://github.com/opensearch-project/OpenSearch/pull/17005)) - Fix GRPC AUX_TRANSPORT_PORT and SETTING_GRPC_PORT settings and remove lingering HTTP terminology ([#17037](https://github.com/opensearch-project/OpenSearch/pull/17037)) +- Fix exists queries on nested flat_object fields throws exception ([#16803](https://github.com/opensearch-project/OpenSearch/pull/16803)) ### Security diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 93821a3da4c71..68d8543ee2725 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -16,7 +16,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Craig Perkins | [cwperks](https://github.com/cwperks) | Amazon | | Dan Widdis | [dbwiddis](https://github.com/dbwiddis) | Amazon | | Daniel "dB." Doubrovkine | [dblock](https://github.com/dblock) | Amazon | -| Gao Binlong | [gaobinlong](https://github.com/gaobinlong) | Amazon | +| Binlong Gao | [gaobinlong](https://github.com/gaobinlong) | Amazon | | Gaurav Bafna | [gbbafna](https://github.com/gbbafna) | Amazon | | Jay Deng | [jed326](https://github.com/jed326) | Amazon | | Kunal Kotwani | [kotwanikunal](https://github.com/kotwanikunal) | Amazon | diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/JarHellTask.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/JarHellTask.java index ebe0b25a3a685..47ad8cc524a3b 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/JarHellTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/JarHellTask.java @@ -63,7 +63,7 @@ public JarHellTask(Project project) { public void runJarHellCheck() { LoggedExec.javaexec(project, spec -> { spec.environment("CLASSPATH", getClasspath().getAsPath()); - spec.getMainClass().set("org.opensearch.bootstrap.JarHell"); + spec.getMainClass().set("org.opensearch.common.bootstrap.JarHell"); }); } diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java index 2ed801b7fb9c6..70a1ed478ff63 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java @@ -94,7 +94,7 @@ public class ThirdPartyAuditTask extends DefaultTask { CliMain.EXIT_VIOLATION, CliMain.EXIT_UNSUPPORTED_JDK ); - private static final String JDK_JAR_HELL_MAIN_CLASS = "org.opensearch.bootstrap.JdkJarHellCheck"; + private static final String JDK_JAR_HELL_MAIN_CLASS = "org.opensearch.common.bootstrap.JdkJarHellCheck"; private Set missingClassExcludes = new TreeSet<>(); diff --git a/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/TestClasspathUtils.java b/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/TestClasspathUtils.java index ec9a5fb157ccc..84362966d7300 100644 --- a/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/TestClasspathUtils.java +++ b/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/TestClasspathUtils.java @@ -48,7 +48,7 @@ public class TestClasspathUtils { public static void setupJarJdkClasspath(File projectRoot) { try { URL originLocation = TestClasspathUtils.class.getClassLoader() - .loadClass("org.opensearch.bootstrap.JdkJarHellCheck") + .loadClass("org.opensearch.common.bootstrap.JdkJarHellCheck") .getProtectionDomain() .getCodeSource() .getLocation(); diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java index eb0a8b0e8f40a..0344695ad3848 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java @@ -38,7 +38,7 @@ import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.cluster.RemoteInfoRequest; import org.opensearch.client.cluster.RemoteInfoResponse; import org.opensearch.client.indices.ComponentTemplatesExistRequest; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/IndicesClient.java index 281f020533d51..efb65bf6865d4 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/IndicesClient.java @@ -51,7 +51,7 @@ import org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequest; import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.indices.AnalyzeRequest; import org.opensearch.client.indices.AnalyzeResponse; import org.opensearch.client.indices.CloseIndexRequest; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/IngestClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/IngestClient.java index 29e5c5369f184..55d481ebd07f3 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/IngestClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/IngestClient.java @@ -38,7 +38,7 @@ import org.opensearch.action.ingest.PutPipelineRequest; import org.opensearch.action.ingest.SimulatePipelineRequest; import org.opensearch.action.ingest.SimulatePipelineResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.core.action.ActionListener; import java.io.IOException; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java index 83c3ba8164c4b..f0764989da78e 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java @@ -68,7 +68,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.SearchScrollRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.update.UpdateRequest; import org.opensearch.action.update.UpdateResponse; import org.opensearch.client.core.CountRequest; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/SearchPipelineClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/SearchPipelineClient.java index 0014bdb8c8182..17a5e5701ad24 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/SearchPipelineClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/SearchPipelineClient.java @@ -12,7 +12,7 @@ import org.opensearch.action.search.GetSearchPipelineRequest; import org.opensearch.action.search.GetSearchPipelineResponse; import org.opensearch.action.search.PutSearchPipelineRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.core.action.ActionListener; import java.io.IOException; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/SnapshotClient.java index 87a0e45eafe49..54d09ed2baf11 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/SnapshotClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/SnapshotClient.java @@ -50,7 +50,7 @@ import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.core.action.ActionListener; import java.io.IOException; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CloseIndexResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CloseIndexResponse.java index e32c33484140d..38ce53bcc0dfa 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CloseIndexResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CloseIndexResponse.java @@ -32,7 +32,7 @@ package org.opensearch.client.indices; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.master.ShardsAcknowledgedResponse; +import org.opensearch.action.support.clustermanager.ShardsAcknowledgedResponse; import org.opensearch.common.Nullable; import org.opensearch.core.ParseField; import org.opensearch.core.action.support.DefaultShardOperationFailedException; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexResponse.java index 21e2ba4b342a6..24f752c2bbf83 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/CreateIndexResponse.java @@ -32,7 +32,7 @@ package org.opensearch.client.indices; -import org.opensearch.action.support.master.ShardsAcknowledgedResponse; +import org.opensearch.action.support.clustermanager.ShardsAcknowledgedResponse; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.ObjectParser; diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverResponse.java b/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverResponse.java index b146d4f94e131..5d56cba9e8db5 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverResponse.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/indices/rollover/RolloverResponse.java @@ -32,7 +32,7 @@ package org.opensearch.client.indices.rollover; -import org.opensearch.action.support.master.ShardsAcknowledgedResponse; +import org.opensearch.action.support.clustermanager.ShardsAcknowledgedResponse; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.ConstructingObjectParser; import org.opensearch.core.xcontent.XContentParser; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java index 79481fd03b2a1..137c053786349 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/ClusterClientIT.java @@ -42,7 +42,7 @@ import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsResponse; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.cluster.RemoteConnectionInfo; import org.opensearch.client.cluster.RemoteInfoRequest; import org.opensearch.client.cluster.RemoteInfoResponse; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/ClusterRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/ClusterRequestConvertersTests.java index 3415868c9f8c6..24d27dcb7f6ad 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/ClusterRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/ClusterRequestConvertersTests.java @@ -38,7 +38,7 @@ import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.support.ActiveShardCount; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.client.cluster.RemoteInfoRequest; import org.opensearch.cluster.health.ClusterHealthStatus; import org.opensearch.common.Priority; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java index ccdcc21f0fc8b..0399e4667d85d 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesClientIT.java @@ -65,7 +65,7 @@ import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; import org.opensearch.action.support.broadcast.BroadcastResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.indices.AnalyzeRequest; import org.opensearch.client.indices.AnalyzeResponse; import org.opensearch.client.indices.CloseIndexRequest; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java index c3a0f049f375e..a42c00ab2c2c5 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java @@ -53,7 +53,7 @@ import org.opensearch.action.admin.indices.shrink.ResizeType; import org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; import org.opensearch.action.admin.indices.validate.query.ValidateQueryRequest; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.client.indices.AnalyzeRequest; import org.opensearch.client.indices.CloseIndexRequest; import org.opensearch.client.indices.CreateDataStreamRequest; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java index 33bff06a83065..5e1510aab0784 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IngestClientIT.java @@ -41,7 +41,7 @@ import org.opensearch.action.ingest.SimulateDocumentVerboseResult; import org.opensearch.action.ingest.SimulatePipelineRequest; import org.opensearch.action.ingest.SimulatePipelineResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/IngestRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/IngestRequestConvertersTests.java index 38dde4be3dd8e..6cadb2d8b72d5 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/IngestRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/IngestRequestConvertersTests.java @@ -40,7 +40,7 @@ import org.opensearch.action.ingest.GetPipelineRequest; import org.opensearch.action.ingest.PutPipelineRequest; import org.opensearch.action.ingest.SimulatePipelineRequest; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.test.OpenSearchTestCase; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java index 38f5d9302440e..a35213c1c3c0c 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java @@ -62,8 +62,8 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.action.support.replication.ReplicationRequest; import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.RequestConverters.EndpointBuilder; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SearchPipelineClientIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/SearchPipelineClientIT.java index 9304be7f21899..7b94b19549633 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/SearchPipelineClientIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/SearchPipelineClientIT.java @@ -12,7 +12,7 @@ import org.opensearch.action.search.GetSearchPipelineRequest; import org.opensearch.action.search.GetSearchPipelineResponse; import org.opensearch.action.search.PutSearchPipelineRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotIT.java index 362a8f10d6a77..af32e0d76178d 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotIT.java @@ -51,7 +51,7 @@ import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.rest.RestStatus; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java index 8b6910ffebe4a..18b1c9382583e 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java @@ -45,7 +45,7 @@ import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.unit.ByteSizeUnit; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/core/AcknowledgedResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/core/AcknowledgedResponseTests.java index e184df7ad013c..758685b24cf24 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/core/AcknowledgedResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/core/AcknowledgedResponseTests.java @@ -40,12 +40,12 @@ import static org.hamcrest.Matchers.is; public class AcknowledgedResponseTests extends AbstractResponseTestCase< - org.opensearch.action.support.master.AcknowledgedResponse, + org.opensearch.action.support.clustermanager.AcknowledgedResponse, AcknowledgedResponse> { @Override - protected org.opensearch.action.support.master.AcknowledgedResponse createServerTestInstance(XContentType xContentType) { - return new org.opensearch.action.support.master.AcknowledgedResponse(randomBoolean()); + protected org.opensearch.action.support.clustermanager.AcknowledgedResponse createServerTestInstance(XContentType xContentType) { + return new org.opensearch.action.support.clustermanager.AcknowledgedResponse(randomBoolean()); } @Override @@ -55,7 +55,7 @@ protected AcknowledgedResponse doParseToClientInstance(XContentParser parser) th @Override protected void assertInstances( - org.opensearch.action.support.master.AcknowledgedResponse serverTestInstance, + org.opensearch.action.support.clustermanager.AcknowledgedResponse serverTestInstance, AcknowledgedResponse clientInstance ) { assertThat(clientInstance.isAcknowledged(), is(serverTestInstance.isAcknowledged())); diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java index 17ea5b273d2a2..2199824cb910b 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/ClusterClientDocumentationIT.java @@ -40,7 +40,7 @@ import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.opensearch.action.support.ActiveShardCount; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchRestHighLevelClientTestCase; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java index ce080b45273b4..84995caffa0fe 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IndicesClientDocumentationIT.java @@ -61,7 +61,7 @@ import org.opensearch.action.admin.indices.validate.query.ValidateQueryResponse; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.GetAliasesResponse; import org.opensearch.client.OpenSearchRestHighLevelClientTestCase; import org.opensearch.client.RequestOptions; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java index 28909cf58541a..ddea2069f46e7 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/IngestClientDocumentationIT.java @@ -43,7 +43,7 @@ import org.opensearch.action.ingest.SimulatePipelineRequest; import org.opensearch.action.ingest.SimulatePipelineResponse; import org.opensearch.action.ingest.SimulateProcessorResult; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchRestHighLevelClientTestCase; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java index 6949bc382bfe8..b7ee638788f58 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/SnapshotClientDocumentationIT.java @@ -51,7 +51,7 @@ import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchRestHighLevelClientTestCase; import org.opensearch.client.Request; import org.opensearch.client.RequestOptions; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java index 2e2d15df5392a..11becf8e55942 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/documentation/StoredScriptsDocumentationIT.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.opensearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; import org.opensearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchRestHighLevelClientTestCase; import org.opensearch.client.RequestOptions; import org.opensearch.client.RestHighLevelClient; diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java index 6aafee142bd22..36e71bf9f0830 100644 --- a/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java +++ b/client/rest-high-level/src/test/java/org/opensearch/client/indices/CloseIndexResponseTests.java @@ -32,8 +32,8 @@ package org.opensearch.client.indices; import org.opensearch.OpenSearchStatusException; -import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.action.support.master.ShardsAcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.ShardsAcknowledgedResponse; import org.opensearch.client.AbstractResponseTestCase; import org.opensearch.common.xcontent.LoggingDeprecationHandler; import org.opensearch.common.xcontent.XContentType; diff --git a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java index 511d6974085aa..d5a0102ba86af 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/InstallPluginCommand.java @@ -52,12 +52,12 @@ import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider; import org.opensearch.Build; import org.opensearch.Version; -import org.opensearch.bootstrap.JarHell; import org.opensearch.cli.EnvironmentAwareCommand; import org.opensearch.cli.ExitCodes; import org.opensearch.cli.Terminal; import org.opensearch.cli.UserException; import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.bootstrap.JarHell; import org.opensearch.common.collect.Tuple; import org.opensearch.common.hash.MessageDigests; import org.opensearch.common.util.io.IOUtils; diff --git a/libs/common/src/main/java/org/opensearch/bootstrap/JarHell.java b/libs/common/src/main/java/org/opensearch/common/bootstrap/JarHell.java similarity index 99% rename from libs/common/src/main/java/org/opensearch/bootstrap/JarHell.java rename to libs/common/src/main/java/org/opensearch/common/bootstrap/JarHell.java index fc5e364241d12..470b92aaa2fab 100644 --- a/libs/common/src/main/java/org/opensearch/bootstrap/JarHell.java +++ b/libs/common/src/main/java/org/opensearch/common/bootstrap/JarHell.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.bootstrap; +package org.opensearch.common.bootstrap; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.io.PathUtils; diff --git a/libs/common/src/main/java/org/opensearch/bootstrap/JdkJarHellCheck.java b/libs/common/src/main/java/org/opensearch/common/bootstrap/JdkJarHellCheck.java similarity index 98% rename from libs/common/src/main/java/org/opensearch/bootstrap/JdkJarHellCheck.java rename to libs/common/src/main/java/org/opensearch/common/bootstrap/JdkJarHellCheck.java index 97b323975db0a..2a25f32b363c6 100644 --- a/libs/common/src/main/java/org/opensearch/bootstrap/JdkJarHellCheck.java +++ b/libs/common/src/main/java/org/opensearch/common/bootstrap/JdkJarHellCheck.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.bootstrap; +package org.opensearch.common.bootstrap; import org.opensearch.common.SuppressForbidden; diff --git a/libs/common/src/main/java/org/opensearch/bootstrap/package-info.java b/libs/common/src/main/java/org/opensearch/common/bootstrap/package-info.java similarity index 85% rename from libs/common/src/main/java/org/opensearch/bootstrap/package-info.java rename to libs/common/src/main/java/org/opensearch/common/bootstrap/package-info.java index f522b1bb91444..8d05b614b7f38 100644 --- a/libs/common/src/main/java/org/opensearch/bootstrap/package-info.java +++ b/libs/common/src/main/java/org/opensearch/common/bootstrap/package-info.java @@ -7,4 +7,4 @@ */ /** Contains JarHell Classes */ -package org.opensearch.bootstrap; +package org.opensearch.common.bootstrap; diff --git a/libs/common/src/test/java/org/opensearch/bootstrap/JarHellTests.java b/libs/common/src/test/java/org/opensearch/common/bootstrap/JarHellTests.java similarity index 99% rename from libs/common/src/test/java/org/opensearch/bootstrap/JarHellTests.java rename to libs/common/src/test/java/org/opensearch/common/bootstrap/JarHellTests.java index d1851850e78e1..549c4bd652e2f 100644 --- a/libs/common/src/test/java/org/opensearch/bootstrap/JarHellTests.java +++ b/libs/common/src/test/java/org/opensearch/common/bootstrap/JarHellTests.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.bootstrap; +package org.opensearch.common.bootstrap; import org.opensearch.common.io.PathUtils; import org.opensearch.core.common.Strings; diff --git a/libs/common/src/test/resources/org/opensearch/bootstrap/duplicate-classes.jar b/libs/common/src/test/resources/org/opensearch/common/bootstrap/duplicate-classes.jar similarity index 100% rename from libs/common/src/test/resources/org/opensearch/bootstrap/duplicate-classes.jar rename to libs/common/src/test/resources/org/opensearch/common/bootstrap/duplicate-classes.jar diff --git a/libs/common/src/test/resources/org/opensearch/bootstrap/duplicate-xmlbeans-classes.jar b/libs/common/src/test/resources/org/opensearch/common/bootstrap/duplicate-xmlbeans-classes.jar similarity index 100% rename from libs/common/src/test/resources/org/opensearch/bootstrap/duplicate-xmlbeans-classes.jar rename to libs/common/src/test/resources/org/opensearch/common/bootstrap/duplicate-xmlbeans-classes.jar diff --git a/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java b/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java index 4c71f80a30926..3027f1406f089 100644 --- a/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java +++ b/modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java @@ -35,7 +35,7 @@ import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.unit.ByteSizeUnit; diff --git a/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java b/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java index 7e938623975fc..baa0b77981195 100644 --- a/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java +++ b/modules/search-pipeline-common/src/internalClusterTest/java/org/opensearch/search/pipeline/common/SearchPipelineCommonIT.java @@ -19,7 +19,7 @@ import org.opensearch.action.search.PutSearchPipelineRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; diff --git a/plugins/ingest-attachment/src/main/java/org/opensearch/ingest/attachment/TikaImpl.java b/plugins/ingest-attachment/src/main/java/org/opensearch/ingest/attachment/TikaImpl.java index fe783e5ddb675..d999d20537485 100644 --- a/plugins/ingest-attachment/src/main/java/org/opensearch/ingest/attachment/TikaImpl.java +++ b/plugins/ingest-attachment/src/main/java/org/opensearch/ingest/attachment/TikaImpl.java @@ -41,8 +41,8 @@ import org.apache.tika.parser.ParserDecorator; import org.opensearch.SpecialPermission; import org.opensearch.bootstrap.FilePermissionUtils; -import org.opensearch.bootstrap.JarHell; import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.bootstrap.JarHell; import org.opensearch.common.io.PathUtils; import java.io.ByteArrayInputStream; diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingIT.java index 51e0979324623..7a90974ff1673 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingIT.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/opensearch/index/mapper/size/SizeMappingIT.java @@ -33,7 +33,7 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.get.GetResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.plugin.mapper.MapperSizePlugin; diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/DeleteQueryGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/DeleteQueryGroupAction.java index c78952a2f89ad..b638dbd61ca1a 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/DeleteQueryGroupAction.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/DeleteQueryGroupAction.java @@ -9,7 +9,7 @@ package org.opensearch.plugin.wlm.action; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action for delete QueryGroup diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/DeleteQueryGroupRequest.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/DeleteQueryGroupRequest.java index e514943c2c7e9..e798c8e137062 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/DeleteQueryGroupRequest.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/DeleteQueryGroupRequest.java @@ -9,7 +9,7 @@ package org.opensearch.plugin.wlm.action; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportDeleteQueryGroupAction.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportDeleteQueryGroupAction.java index e4d3908d4a208..dd37f9df399ce 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportDeleteQueryGroupAction.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/TransportDeleteQueryGroupAction.java @@ -9,8 +9,8 @@ package org.opensearch.plugin.wlm.action; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java index f9332ff3022dc..73dff306d0e69 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java @@ -11,7 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.ResourceNotFoundException; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.AckedClusterStateUpdateTask; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/TransportDeleteQueryGroupActionTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/TransportDeleteQueryGroupActionTests.java index 253d65f8da80f..39d263bfdb150 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/TransportDeleteQueryGroupActionTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/action/TransportDeleteQueryGroupActionTests.java @@ -9,7 +9,7 @@ package org.opensearch.plugin.wlm.action; import org.opensearch.action.support.ActionFilters; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteQueryGroupActionTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteQueryGroupActionTests.java index 72191e076bb87..28ed813ec4130 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteQueryGroupActionTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rest/RestDeleteQueryGroupActionTests.java @@ -8,7 +8,7 @@ package org.opensearch.plugin.wlm.rest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.unit.TimeValue; diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java index 08b51fd46cfcf..67e47be1a55ce 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java @@ -9,7 +9,7 @@ package org.opensearch.plugin.wlm.service; import org.opensearch.ResourceNotFoundException; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.AckedClusterStateUpdateTask; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml index 6fc2654bcfc8f..0c00ff6e4a6a8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml @@ -506,20 +506,6 @@ teardown: - length: { hits.hits: 2 } - # Exists Query with nested dot path, use the flat_object_field_name.last_key - - do: - search: - body: { - _source: true, - query: { - "exists": { - "field": issue.labels.type - } - } - } - - - length: { hits.hits: 3 } - # Exists Query without dot path for the flat_object_field_name - do: search: @@ -613,3 +599,22 @@ teardown: - length: { hits.hits: 1 } - match: { hits.hits.0._source.issue.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + +--- +"Exists query for sub field": + - skip: + version: " - 2.99.99" + reason: "exists query for sub field of flat_object field has bug before 3.0.0" + + - do: + search: + body: { + _source: true, + query: { + "exists": { + "field": issue.labels.category.type + } + } + } + + - length: { hits.hits: 3 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/105_partial_flat_object_nested.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/105_partial_flat_object_nested.yml index 549ddbdde7bab..ccb9c3e3fc23d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/105_partial_flat_object_nested.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/105_partial_flat_object_nested.yml @@ -506,23 +506,6 @@ teardown: - length: { hits.hits: 2 } - # Exists Query with nested dot path, use the flat_object_field_name.last_key - - do: - search: - body: { - _source: true, - query: { - nested: { - path: "issue", - query: { - "exists": { - "field": issue.labels.type - } } } - } - } - - - length: { hits.hits: 2 } - # Exists Query without dot path for the flat_object_field_name - do: search: @@ -634,3 +617,27 @@ teardown: - length: { hits.hits: 1 } - match: { hits.hits.0._source.issue.0.labels.comment: [ [ "Doe","Shipped" ],[ "John","Approved" ] ] } + +--- +"Exists query for sub field": + - skip: + version: " - 2.99.99" + reason: "exists query for sub field of flat_object field has bug before 3.0.0" + + - do: + search: + body: { + _source: true, + query: { + nested: { + path: "issue", + query: { + "exists": { + "field": issue.labels.category.type + } + } + } + } + } + + - length: { hits.hits: 2 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml index 2a469aa5ff04d..3966b59c4a045 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/90_flat_object.yml @@ -16,7 +16,10 @@ setup: type : "flat_object" required_matches: type : "long" - + infos: + properties: + info: + type: "flat_object" - do: index: index: test @@ -60,7 +63,12 @@ setup: "review": [["bad",30.41],["ok",80.0]], "publishDate": "2016-01-01" }, - "required_matches": 1 + "required_matches": 1, + "infos": { + "info":{ + "name": "name1" + } + } } # Do index refresh - do: @@ -73,6 +81,40 @@ teardown: - do: indices.delete: index: test + +--- +"Exist query in root field": + - skip: + version: "- 2.99.99" + reason: "the query would throw exception prior to 2.99.99" + + - do: + search: + body: { + _source: true, + size: 10, + query: { + exists: { + field: "catalog" + } + } + } + - length: { hits.hits: 2 } + + - do: + search: + body: { + _source: true, + size: 10, + query: { + exists: { + field: "infos" + } + } + } + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.infos.info.name: "name1" } + --- "Invalid docs": - skip: @@ -135,7 +177,7 @@ teardown: - match: { test.mappings.properties.catalog.type: flat_object } - match: { test.mappings.properties.required_matches.type: long } # https://github.com/opensearch-project/OpenSearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/test#length - - length: { test.mappings.properties: 3 } + - length: { test.mappings.properties: 4 } - length: { test.mappings.properties.catalog: 1 } --- diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/91_flat_object_null_value.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/91_flat_object_null_value.yml index 716b6fb51cb43..455eb4fb91a43 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/91_flat_object_null_value.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/91_flat_object_null_value.yml @@ -373,7 +373,7 @@ teardown: body: { _source: true, query: { - exists: { "field": "record.d" } + exists: { "field": "record.name.d.name" } }, sort: [{ order: asc}] } diff --git a/server/build.gradle b/server/build.gradle index 6559c7247200a..82eafb07a7ad3 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -70,7 +70,6 @@ dependencies { api project(":libs:opensearch-telemetry") api project(":libs:opensearch-task-commons") - compileOnly project(':libs:opensearch-plugin-classloader') testRuntimeOnly project(':libs:opensearch-plugin-classloader') diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/CreateIndexIT.java index c44215fc2f327..d713c9cc86841 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/CreateIndexIT.java @@ -39,7 +39,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader; import org.opensearch.cluster.applicationtemplates.SystemTemplate; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java index 82ab5b0118c0e..c36f8e38d2cdd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/datastream/DataStreamTestCase.java @@ -12,7 +12,7 @@ import org.opensearch.action.admin.indices.rollover.RolloverResponse; import org.opensearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction; import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.cluster.metadata.DataStream; import org.opensearch.cluster.metadata.Template; diff --git a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java index cf83f20244a4b..52c4deb11fe4a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java @@ -40,7 +40,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.ingest.PutPipelineRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.replication.ReplicationRequest; import org.opensearch.action.update.UpdateRequest; import org.opensearch.cluster.metadata.IndexMetadata; diff --git a/server/src/internalClusterTest/java/org/opensearch/aliases/IndexAliasesIT.java b/server/src/internalClusterTest/java/org/opensearch/aliases/IndexAliasesIT.java index 292a13bdf7f5c..0503442728dee 100644 --- a/server/src/internalClusterTest/java/org/opensearch/aliases/IndexAliasesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/aliases/IndexAliasesIT.java @@ -41,7 +41,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest.RefreshPolicy; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexAbstraction; diff --git a/server/src/internalClusterTest/java/org/opensearch/blocks/SimpleBlocksIT.java b/server/src/internalClusterTest/java/org/opensearch/blocks/SimpleBlocksIT.java index 6275571cc2371..e1370390e74be 100644 --- a/server/src/internalClusterTest/java/org/opensearch/blocks/SimpleBlocksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/blocks/SimpleBlocksIT.java @@ -41,7 +41,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.ActiveShardCount; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java index cc0264f375103..2906aa0e086fb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/RareClusterStateIT.java @@ -37,7 +37,7 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestBuilder; import org.opensearch.action.index.IndexResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.action.shard.ShardStateAction; diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java index 3718dce538053..5d9b08f3aa004 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/shards/ClusterShardLimitIT.java @@ -37,7 +37,7 @@ import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java index 8d0a3b5788a70..29d60a375fabc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java @@ -48,7 +48,7 @@ import org.opensearch.action.admin.indices.stats.IndicesStatsResponse; import org.opensearch.action.admin.indices.stats.ShardStats; import org.opensearch.action.support.ActionTestUtils; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Requests; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.ElectionSchedulerFactory; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 007fefa1499ab..87577cf2e24cc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -11,7 +11,7 @@ import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.Rounding; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; diff --git a/server/src/internalClusterTest/java/org/opensearch/index/seqno/RetentionLeaseIT.java b/server/src/internalClusterTest/java/org/opensearch/index/seqno/RetentionLeaseIT.java index 6163edada9f6e..51a16acb68089 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/seqno/RetentionLeaseIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/seqno/RetentionLeaseIT.java @@ -33,7 +33,7 @@ package org.opensearch.index.seqno; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.replication.ReplicationResponse; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/mapping/UpdateMappingIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/mapping/UpdateMappingIntegrationIT.java index 2c5d408b3b8f6..a3b1b388eb482 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -36,7 +36,7 @@ import org.opensearch.action.admin.indices.refresh.RefreshResponse; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.action.index.MappingUpdatedAction; import org.opensearch.cluster.metadata.MappingMetadata; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseWhileRelocatingShardsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseWhileRelocatingShardsIT.java index d4df608547a9e..bab1c01321ba9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseWhileRelocatingShardsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/state/CloseWhileRelocatingShardsIT.java @@ -33,7 +33,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.IndexRoutingTable; diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java index 0bf561c606a2d..e697634355eca 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java @@ -41,7 +41,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; diff --git a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java index 0eb37a7b25618..45f976493edc8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestClientIT.java @@ -50,7 +50,7 @@ import org.opensearch.action.ingest.SimulateDocumentBaseResult; import org.opensearch.action.ingest.SimulatePipelineRequest; import org.opensearch.action.ingest.SimulatePipelineResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.update.UpdateRequest; import org.opensearch.client.Requests; import org.opensearch.common.settings.Settings; diff --git a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java index 4c949e11459ab..7a789441c2b55 100644 --- a/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java @@ -35,7 +35,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.opensearch.OpenSearchParseException; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 70e283791fc3e..3b96636cfe771 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -72,7 +72,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA; @@ -494,6 +496,51 @@ public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, Exe assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); } + public void testSuccessfulIndexRestoredFromSnapshotWithUpdatedSetting() throws IOException, ExecutionException, InterruptedException { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + String indexName1 = "testindex1"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true)); + + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + + final int numDocsInIndex1 = randomIntBetween(20, 30); + indexDocuments(client(), indexName1, numDocsInIndex1); + flushAndRefresh(indexName1); + ensureGreen(indexName1); + + logger.info("--> snapshot"); + SnapshotInfo snapshotInfo1 = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1))); + assertThat(snapshotInfo1.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards())); + assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); + + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName1)).get()); + assertFalse(indexExists(indexName1)); + + // try index restore with index.number_of_replicas setting modified. index.number_of_replicas can be modified on restore + Settings numberOfReplicasSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(numberOfReplicasSettings) + .setIndices(indexName1) + .get(); + + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + ensureGreen(indexName1); + assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); + } + protected IndexShard getIndexShard(String node, String indexName) { final Index index = resolveIndex(indexName); IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); @@ -706,7 +753,7 @@ public void testInvalidRestoreRequestScenarios() throws Exception { indexDocuments(client, index, numDocsInIndex, numDocsInIndex + randomIntBetween(2, 5)); ensureGreen(index); - // try index restore with remote store disabled + // try index restore with index.remote_store.enabled ignored SnapshotRestoreException exception = expectThrows( SnapshotRestoreException.class, () -> client().admin() @@ -721,26 +768,37 @@ public void testInvalidRestoreRequestScenarios() throws Exception { ); assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.enabled] on restore")); - // try index restore with remote store repository modified - Settings remoteStoreIndexSettings = Settings.builder() - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo) - .build(); + // try index restore with index.remote_store.segment.repository ignored + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository] on restore")); + // try index restore with index.remote_store.translog.repository ignored exception = expectThrows( SnapshotRestoreException.class, () -> client().admin() .cluster() .prepareRestoreSnapshot(snapshotRepo, snapshotName1) .setWaitForCompletion(false) - .setIndexSettings(remoteStoreIndexSettings) + .setIgnoreIndexSettings(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY) .setIndices(index) .setRenamePattern(index) .setRenameReplacement(restoredIndex) .get() ); - assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore")); + assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.translog.repository] on restore")); - // try index restore with remote store repository and translog store repository disabled + // try index restore with index.remote_store.segment.repository and index.remote_store.translog.repository ignored exception = expectThrows( SnapshotRestoreException.class, () -> client().admin() @@ -757,6 +815,81 @@ public void testInvalidRestoreRequestScenarios() throws Exception { .get() ); assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore")); + + // try index restore with index.remote_store.enabled modified + Settings remoteStoreIndexSettings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false).build(); + + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(remoteStoreIndexSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.enabled]" + " on restore")); + + // try index restore with index.remote_store.segment.repository modified + Settings remoteStoreSegmentIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo) + .build(); + + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(remoteStoreSegmentIndexSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore")); + + // try index restore with index.remote_store.translog.repository modified + Settings remoteStoreTranslogIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, newRemoteStoreRepo) + .build(); + + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(remoteStoreTranslogIndexSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.translog.repository]" + " on restore")); + + // try index restore with index.remote_store.translog.repository and index.remote_store.segment.repository modified + Settings multipleRemoteStoreIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, newRemoteStoreRepo) + .build(); + + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(multipleRemoteStoreIndexSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore")); } public void testCreateSnapshotV2_Orphan_Timestamp_Cleanup() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index 6a5adf5ea4fb7..e2fada2e3496c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -18,7 +18,7 @@ import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.metadata.ComponentTemplate; diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java index 1493a1b259e13..19953b147bdf9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RestoreShallowSnapshotV2IT.java @@ -67,7 +67,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS; import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG; @@ -561,6 +563,51 @@ public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, Exe assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); } + public void testSuccessfulIndexRestoredFromSnapshotWithUpdatedSetting() throws IOException, ExecutionException, InterruptedException { + internalCluster().startClusterManagerOnlyNode(); + internalCluster().startDataOnlyNodes(2); + + String indexName1 = "testindex1"; + String snapshotRepoName = "test-restore-snapshot-repo"; + String snapshotName1 = "test-restore-snapshot1"; + Path absolutePath1 = randomRepoPath().toAbsolutePath(); + logger.info("Snapshot Path [{}]", absolutePath1); + + createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true)); + + Settings indexSettings = getIndexSettings(1, 0).build(); + createIndex(indexName1, indexSettings); + + final int numDocsInIndex1 = randomIntBetween(20, 30); + indexDocuments(client(), indexName1, numDocsInIndex1); + flushAndRefresh(indexName1); + ensureGreen(indexName1); + + logger.info("--> snapshot"); + SnapshotInfo snapshotInfo1 = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1))); + assertThat(snapshotInfo1.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards())); + assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS)); + + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(indexName1)).get()); + assertFalse(indexExists(indexName1)); + + // try index restore with index.number_of_replicas setting modified. index.number_of_replicas can be modified on restore + Settings numberOfReplicasSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); + + RestoreSnapshotResponse restoreSnapshotResponse1 = client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepoName, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(numberOfReplicasSettings) + .setIndices(indexName1) + .get(); + + assertEquals(restoreSnapshotResponse1.status(), RestStatus.ACCEPTED); + ensureGreen(indexName1); + assertDocsPresentInIndex(client(), indexName1, numDocsInIndex1); + } + private IndexShard getIndexShard(String node, String indexName) { final Index index = resolveIndex(indexName); IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); @@ -773,7 +820,7 @@ public void testInvalidRestoreRequestScenarios() throws Exception { indexDocuments(client, index, numDocsInIndex, numDocsInIndex + randomIntBetween(2, 5)); ensureGreen(index); - // try index restore with remote store disabled + // try index restore with index.remote_store.enabled ignored SnapshotRestoreException exception = expectThrows( SnapshotRestoreException.class, () -> client().admin() @@ -788,26 +835,37 @@ public void testInvalidRestoreRequestScenarios() throws Exception { ); assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.enabled] on restore")); - // try index restore with remote store repository modified - Settings remoteStoreIndexSettings = Settings.builder() - .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo) - .build(); + // try index restore with index.remote_store.segment.repository ignored + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository] on restore")); + // try index restore with index.remote_store.translog.repository ignored exception = expectThrows( SnapshotRestoreException.class, () -> client().admin() .cluster() .prepareRestoreSnapshot(snapshotRepo, snapshotName1) .setWaitForCompletion(false) - .setIndexSettings(remoteStoreIndexSettings) + .setIgnoreIndexSettings(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY) .setIndices(index) .setRenamePattern(index) .setRenameReplacement(restoredIndex) .get() ); - assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore")); + assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.translog.repository] on restore")); - // try index restore with remote store repository and translog store repository disabled + // try index restore with index.remote_store.segment.repository and index.remote_store.translog.repository ignored exception = expectThrows( SnapshotRestoreException.class, () -> client().admin() @@ -824,6 +882,81 @@ public void testInvalidRestoreRequestScenarios() throws Exception { .get() ); assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore")); + + // try index restore with index.remote_store.enabled modified + Settings remoteStoreIndexSettings = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false).build(); + + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(remoteStoreIndexSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.enabled]" + " on restore")); + + // try index restore with index.remote_store.segment.repository modified + Settings remoteStoreSegmentIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo) + .build(); + + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(remoteStoreSegmentIndexSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore")); + + // try index restore with index.remote_store.translog.repository modified + Settings remoteStoreTranslogIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, newRemoteStoreRepo) + .build(); + + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(remoteStoreTranslogIndexSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.translog.repository]" + " on restore")); + + // try index restore with index.remote_store.translog.repository and index.remote_store.segment.repository modified + Settings multipleRemoteStoreIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo) + .put(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, newRemoteStoreRepo) + .build(); + + exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(multipleRemoteStoreIndexSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore")); } public void testRestoreOperationsUsingDifferentRepos() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java index a54bc8b6e5fff..0ab86abea963b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/suggest/CompletionSuggestSearchIT.java @@ -44,7 +44,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchPhaseExecutionException; import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.FieldMemoryStats; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.Fuzziness; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java index ddbe0520f0c1f..bfaefa1e91a8f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java @@ -36,7 +36,7 @@ import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexStatus; import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.cluster.metadata.RepositoryMetadata; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java index b00a8c8d64ed9..e0e18f7aaad6f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java @@ -40,7 +40,7 @@ import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.delete.DeleteResponse; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java index cab750a421f0d..252efcdc979bb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsIT.java @@ -39,7 +39,7 @@ import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse; import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.SnapshotDeletionsInProgress; import org.opensearch.cluster.SnapshotsInProgress; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java index ab5b22c69b517..375ba16d91bc1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java @@ -11,7 +11,7 @@ import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 686853c42aa03..18d23532c84e0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -42,7 +42,7 @@ import org.opensearch.action.admin.indices.stats.ShardStats; import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.support.ActiveShardCount; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterState; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index 85d81761ea4a0..d09d087417d6a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -9,7 +9,7 @@ package org.opensearch.snapshots; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.UUIDs; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java index 7b2ad2bccd2b1..5e70299486825 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotV2IT.java @@ -9,7 +9,7 @@ package org.opensearch.snapshots; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java index 271fcf166139f..35fd716c89e2b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java @@ -38,7 +38,7 @@ import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.index.IndexRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.metadata.RepositoriesMetadata; diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java index e76587653e99a..36ab97d0b730f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java @@ -32,6 +32,7 @@ package org.opensearch.snapshots; +import org.opensearch.Version; import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -49,10 +50,12 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.unit.ByteSizeUnit; import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexSettings; import org.opensearch.indices.InvalidIndexNameException; import org.opensearch.repositories.RepositoriesService; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Locale; @@ -1112,4 +1115,389 @@ public void testRestoreBalancedReplica() { } } + private String index; + private String snapshotRepo; + private String snapshotName1; + private String snapshotName2; + private Path absolutePath1; + private String restoredIndex; + private Settings indexSettings; + private SnapshotInfo snapshotInfo; + private SnapshotInfo snapshotInfo2; + + public void setupSnapshotRestore() { + index = "test-index"; + snapshotRepo = "test-restore-snapshot-repo"; + snapshotName1 = "test-restore-snapshot1"; + snapshotName2 = "test-restore-snapshot2"; + absolutePath1 = randomRepoPath().toAbsolutePath(); + + logger.info("Snapshot Path [{}]", absolutePath1); + restoredIndex = index + "-restored"; + + createRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, true)); + + indexSettings = getIndexSettings(1, 0).build(); + createIndex(index, indexSettings); + ensureGreen(index); + + logger.info("--> snapshot"); + + snapshotInfo = createSnapshot(snapshotRepo, snapshotName1, new ArrayList<>(List.of(index))); + assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards())); + + updateRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, false)); + snapshotInfo2 = createSnapshot(snapshotRepo, snapshotName2, new ArrayList<>(List.of(index))); + assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS)); + assertThat(snapshotInfo2.successfulShards(), greaterThan(0)); + assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards())); + ensureGreen(index); + } + + public void testInvalidRestoreRequest_UserUnRemovableSettingsIgnored() throws Exception { + setupSnapshotRestore(); + + // try index restore with USER_UNREMOVABLE_SETTINGS setting ignored + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.enabled] on restore")); + + } + + public void testInvalidRestoreRequest_UnmodifiableOnRestoreIgnored() throws Exception { + setupSnapshotRestore(); + + // try index restore with UnmodifiableOnRestore setting ignored + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(IndexMetadata.SETTING_NUMBER_OF_SHARDS) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove UnmodifiableOnRestore setting [index.number_of_shards] on restore")); + + } + + public void testInvalidRestoreRequest_MixRemovableAndUnmodifiableOnRestoreIgnored() throws Exception { + setupSnapshotRestore(); + + // try index restore with mix of removable and UnmodifiableOnRestore settings ignored + // index.version.created is UnmodifiableOnRestore, index.number_of_search_only_replicas is removable + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(IndexMetadata.SETTING_VERSION_CREATED, IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove UnmodifiableOnRestore setting [index.version.created] on restore")); + } + + public void testInvalidRestoreRequest_MixRemovableAndUserUnRemovableSettingsIgnored() throws Exception { + setupSnapshotRestore(); + + // try index restore with mix of removable and USER_UNREMOVABLE_SETTINGS settings ignored + // index.number_of_replicas is USER_UNREMOVABLE_SETTINGS, index.number_of_search_only_replicas is removable + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove setting [index.number_of_replicas] on restore")); + + } + + public void testInvalidRestoreRequest_MixUnmodifiableOnRestoreAndUserUnRemovableSettingsIgnored() throws Exception { + setupSnapshotRestore(); + + // try index restore with mix of UnmodifiableOnRestore and USER_UNREMOVABLE_SETTINGS settings ignored + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, IndexMetadata.SETTING_NUMBER_OF_SHARDS) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove setting [index.number_of_replicas]" + " on restore")); + + } + + public void testInvalidRestoreRequest_MultipleUnmodifiableOnRestoreIgnored() throws Exception { + setupSnapshotRestore(); + + // try index restore with multiple UnmodifiableOnRestore settings ignored + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(IndexMetadata.SETTING_NUMBER_OF_SHARDS, IndexMetadata.SETTING_VERSION_CREATED) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove UnmodifiableOnRestore setting [index.number_of_shards]" + " on restore")); + + } + + public void testInvalidRestoreRequest_MultipleUserUnRemovableSettingsIgnored() throws Exception { + setupSnapshotRestore(); + + // try index restore with multiple USER_UNREMOVABLE_SETTINGS settings ignored + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIgnoreIndexSettings(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot remove setting [index.number_of_replicas]" + " on restore")); + + } + + public void testInvalidRestoreRequest_UnmodifiableOnRestoreModified() throws Exception { + setupSnapshotRestore(); + + // try index restore with UnmodifiableOnRestore setting modified + Settings numberOfShardsSettingsDiff = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3).build(); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(numberOfShardsSettingsDiff) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.number_of_shards]" + " on restore")); + + } + + public void testInvalidRestoreRequest_UnmodifiableOnRestoreUnchanged() throws Exception { + setupSnapshotRestore(); + + // try index restore with UnmodifiableOnRestore setting unchanged + Settings numberOfShardsSettingsSame = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build(); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(numberOfShardsSettingsSame) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.number_of_shards]" + " on restore")); + + } + + public void testInvalidRestoreRequest_UserUnmodifiableSettingsModified() throws Exception { + setupSnapshotRestore(); + + // try index restore with USER_UNMODIFIABLE_SETTINGS setting modified + Settings remoteStoreEnabledSetting = Settings.builder().put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false).build(); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(remoteStoreEnabledSetting) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.enabled]" + " on restore")); + + } + + public void testInvalidRestoreRequest_MixModifiableAndUnmodifiableOnRestoreModified() throws Exception { + setupSnapshotRestore(); + + // try index restore with mix of modifiable and UnmodifiableOnRestore settings modified + // index.version.created is UnmodifiableOnRestore, index.number_of_search_only_replicas is modifiable + Settings mixedSettingsUnmodifiableOnRestore = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_EMPTY) + .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .build(); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(mixedSettingsUnmodifiableOnRestore) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.version.created]" + " on restore")); + + } + + public void testInvalidRestoreRequest_MixModifiableAndUserUnmodifiableSettingsModified() throws Exception { + setupSnapshotRestore(); + + // try index restore with mix of modifiable and USER_UNMODIFIABLE_SETTINGS settings modified + // index.remote_store.enabled is USER_UNMODIFIABLE_SETTINGS, index.number_of_search_only_replicas is modifiable + Settings mixedSettingsUserUnmodifiableSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) + .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .build(); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(mixedSettingsUserUnmodifiableSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.enabled]" + " on restore")); + + } + + public void testInvalidRestoreRequest_MixUnmodifiableOnRestoreAndUserUnmodifiableSettingsModified() throws Exception { + setupSnapshotRestore(); + + // try index restore with mix of UnmodifiableOnRestore and USER_UNMODIFIABLE_SETTINGS settings modified + // index.remote_store.enabled is USER_UNMODIFIABLE_SETTINGS, index.version.created is UnmodifiableOnRestore + Settings mixedSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_EMPTY) + .build(); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(mixedSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.enabled]" + " on restore")); + + } + + public void testInvalidRestoreRequest_MultipleUnmodifiableOnRestoreModified() throws Exception { + setupSnapshotRestore(); + + // try index restore with multiple UnmodifiableOnRestore settings modified + Settings unmodifiableOnRestoreSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_EMPTY) + .build(); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(unmodifiableOnRestoreSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify UnmodifiableOnRestore setting [index.number_of_shards]" + " on restore")); + + } + + public void testInvalidRestoreRequest_MultipleUserUnmodifiableSettingsModified() throws Exception { + setupSnapshotRestore(); + + // try index restore with multiple USER_UNMODIFIABLE_SETTINGS settings modified + Settings userUnmodifiableSettings = Settings.builder() + .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false) + .put(IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS, 1) + .build(); + + SnapshotRestoreException exception = expectThrows( + SnapshotRestoreException.class, + () -> client().admin() + .cluster() + .prepareRestoreSnapshot(snapshotRepo, snapshotName1) + .setWaitForCompletion(false) + .setIndexSettings(userUnmodifiableSettings) + .setIndices(index) + .setRenamePattern(index) + .setRenameReplacement(restoredIndex) + .get() + ); + assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.enabled]" + " on restore")); + + } + + protected Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) { + Settings.Builder settingsBuilder = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s"); + return settingsBuilder; + } + } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java index a19bbe49ad340..a952aee7704dd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SearchableSnapshotIT.java @@ -22,7 +22,7 @@ import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; import org.opensearch.action.index.IndexRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java index 13b056a019200..32de8080a53bc 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/delete/DeleteDecommissionStateResponse.java @@ -8,7 +8,7 @@ package org.opensearch.action.admin.cluster.decommission.awareness.delete; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java index a2401cdf91b07..6a9ed609c0c95 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/decommission/awareness/put/DecommissionResponse.java @@ -8,7 +8,7 @@ package org.opensearch.action.admin.cluster.decommission.awareness.put; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java index 3e408c6114690..27da4349af2df 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/CleanupRepositoryRequest.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.repositories.cleanup; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryAction.java index 2031e4f7a716f..5f17afe2abf76 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.repositories.delete; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Unregister repository action diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java index 04fdf22bee593..4dc686d5943aa 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.repositories.delete; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java index 6f5d0495e1c9f..1f0ca5593c585 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/DeleteRepositoryRequestBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.cluster.repositories.delete; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java index 3d779befe474e..edf4a39b6d0cf 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.repositories.delete; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryAction.java index c2f90d869d873..9e56d1dfb3560 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.repositories.put; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Register repository action diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java index 3fbd5743b5c8c..70d0cba0d823b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequest.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.admin.cluster.crypto.CryptoSettings; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java index 22aa6d7dc7c00..29512261dbddf 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/PutRepositoryRequestBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.repositories.put; import org.opensearch.action.admin.cluster.crypto.CryptoSettings; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java index 1eadab6b1352e..eaaa041247b8a 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.repositories.put; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java index ae6c92d8625ca..0443360a3cfe2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/VerifyRepositoryRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.repositories.verify; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java index 1cefaa4866110..d4bb83cdd5f29 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.reroute; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.cluster.routing.allocation.command.AllocationCommand; import org.opensearch.cluster.routing.allocation.command.AllocationCommands; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java index fc8310bdf7852..5ea56c8975b8a 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.reroute; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.cluster.routing.allocation.command.AllocationCommand; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index ff01888040e82..ff00aa21f5c62 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.reroute; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.routing.allocation.RoutingExplanations; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java index 77aee99c2e902..911c191b51c96 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.settings; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java index 53f1f17bbeb50..11854a8ad931d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsRequestBuilder.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.settings; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java index 2dfdb49736773..dd21159329dea 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/ClusterUpdateSettingsResponse.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.cluster.settings; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java index f6a18ae5055ae..ddcfc2bedb97b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/delete/ClusterDeleteWeightedRoutingResponse.java @@ -8,7 +8,7 @@ package org.opensearch.action.admin.cluster.shards.routing.weighted.delete; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java index 4fee2f05a8715..2fc1c1b2dfd61 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/routing/weighted/put/ClusterPutWeightedRoutingResponse.java @@ -8,7 +8,7 @@ package org.opensearch.action.admin.cluster.shards.routing.weighted.put; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotAction.java index c6fe102544a7e..189b6aa7b7544 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.snapshots.clone; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action for cloning a snapshot diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java index 839a1b935ad1f..0585626020c1d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/CloneSnapshotRequestBuilder.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionType; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java index 6da26d9a2da45..19071447a9556 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.snapshots.clone; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotAction.java index 0b98a4b31fd53..60d9cadc0aede 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.snapshots.delete; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Delete snapshot action diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java index f4da1ec0f7785..c2386a0039e88 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequestBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.cluster.snapshots.delete; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java index e8462e4d822f2..b755d27639f80 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.snapshots.delete; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptAction.java index 3645ef21d2e12..483004a3365c5 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.storedscripts; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action for deleting stored scripts diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java index 0bb4f3625ddad..2400cfcc00592 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.storedscripts; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java index cbadde386d5a9..a0c0bd62a3f45 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/DeleteStoredScriptRequestBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.cluster.storedscripts; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptAction.java index 2845d895a69e8..cc571c2f26136 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.cluster.storedscripts; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action for putting stored script diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index 8731b18fff338..60c4b32ada73e 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.XContentType; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java index 46773177e9a74..f91bb98ed2f22 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/PutStoredScriptRequestBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.cluster.storedscripts; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.bytes.BytesReference; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java index b0863939fd04c..9688b6659a810 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.storedscripts; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java index 61ee641b4764d..eb2838cbef5c8 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.cluster.storedscripts; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesAction.java index 4d735e984c34e..9ce10c2853ff6 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.alias; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action for listing index aliases diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java index 6ce62dda32d0a..dd915f8857162 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequest.java @@ -36,7 +36,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.AliasesRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.cluster.metadata.AliasAction; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.xcontent.XContentFactory; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java index d262c9cd42ce9..a9ce37455ee36 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/IndicesAliasesRequestBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.indices.alias; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; import org.opensearch.index.query.QueryBuilder; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index 42e02e9e1aff8..50f799e9f263f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -36,8 +36,8 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.RequestValidators; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ack.ClusterStateUpdateResponse; import org.opensearch.cluster.block.ClusterBlockException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java index e785c31c4a0b9..fd3f3799463c9 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequest.java @@ -36,7 +36,7 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequestBuilder.java index 92c32c9ace490..b44c05b0a2889 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexRequestBuilder.java @@ -34,7 +34,7 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java index 2e0c5cb5842b4..af21512081b30 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/close/CloseIndexResponse.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.close; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.master.ShardsAcknowledgedResponse; +import org.opensearch.action.support.clustermanager.ShardsAcknowledgedResponse; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.action.support.DefaultShardOperationFailedException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java index 6bb1bf0a3b97b..0c2e1954ad37e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequest.java @@ -41,7 +41,7 @@ import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.cluster.metadata.Context; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java index 27a580434333a..cf65c295b5d81 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -34,7 +34,7 @@ import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.support.ActiveShardCount; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.cluster.metadata.Context; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java index 3258ffd8672a1..7d7c5206dacc6 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/CreateIndexResponse.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.create; -import org.opensearch.action.support.master.ShardsAcknowledgedResponse; +import org.opensearch.action.support.clustermanager.ShardsAcknowledgedResponse; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexAction.java index 6559ef4cd89bd..2ccc422f2edd6 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.dangling.delete; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * This action causes a dangling index to be considered as deleted by the cluster. diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java index c5c03c93785d2..4dcfb1f6b0fb4 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/DeleteDanglingIndexRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.dangling.delete; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java index 751a872ee7dcd..1e52eb95c71b0 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java @@ -42,8 +42,8 @@ import org.opensearch.action.admin.indices.dangling.list.ListDanglingIndicesResponse; import org.opensearch.action.admin.indices.dangling.list.NodeListDanglingIndicesResponse; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.AckedClusterStateUpdateTask; import org.opensearch.cluster.ClusterState; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexAction.java index 5f7a096b1d749..308720aa6139f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.dangling.import_index; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Represents a request to import a particular dangling index. diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java index 2702b6a05c4bb..42a92c5450c7d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/ImportDanglingIndexRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.dangling.import_index; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java index 3f47d1bf083f4..86acf78b8b2a0 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/import_index/TransportImportDanglingIndexAction.java @@ -42,7 +42,7 @@ import org.opensearch.action.admin.indices.dangling.find.NodeFindDanglingIndexResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.HandledTransportAction; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.inject.Inject; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java index 4c1690d25fbd9..6b5091e0b2ab5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java @@ -37,9 +37,9 @@ import org.opensearch.action.ValidateActions; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java index 6b0aec6a31839..2212974985e3f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java @@ -38,9 +38,9 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.block.ClusterBlockException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexAction.java index 696c1244c7504..a3aa9e751a8ec 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.delete; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action for deleting an index diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java index 5fbefbc6e1591..4bfee40b2d49f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequest.java @@ -35,7 +35,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java index 6cf0920f8570f..e02d46e5fd671 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/DeleteIndexRequestBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.delete; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java index 410a58afc95f1..57f7dcfdf611d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -37,8 +37,8 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.DestructiveOperations; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ack.ClusterStateUpdateResponse; import org.opensearch.cluster.block.ClusterBlockException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/AutoPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/AutoPutMappingAction.java index f2430eb54db9b..6f0cad2fe178d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/AutoPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/AutoPutMappingAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.mapping.put; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action to automatically put field mappings. diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingAction.java index 8bca1b59ee2e2..9088d1241ad2a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.mapping.put; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action to put field mappings. diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java index 8122db3278795..e33385c859ab7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequest.java @@ -37,8 +37,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java index d44b243bb0edb..a32875151fc3a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/PutMappingRequestBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.mapping.put; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.index.Index; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java index 4722c1048014f..f50acb6e2d56e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.indices.mapping.put; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index ac797914aafd8..9727c6b62093e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -37,8 +37,8 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.RequestValidators; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ack.ClusterStateUpdateResponse; import org.opensearch.cluster.block.ClusterBlockException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java index f48ec1ae6fb71..2975c8d0ab145 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequest.java @@ -36,7 +36,7 @@ import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequestBuilder.java index 19770255b0ee1..e31cba2f02a66 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexRequestBuilder.java @@ -34,7 +34,7 @@ import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java index 78af1abc3ce31..217e4762b26a5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/open/OpenIndexResponse.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.open; -import org.opensearch.action.support.master.ShardsAcknowledgedResponse; +import org.opensearch.action.support.clustermanager.ShardsAcknowledgedResponse; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java index 1fb8514cbf48c..7d283de6d36a6 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequest.java @@ -35,7 +35,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.cluster.metadata.IndexMetadata.APIBlock; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java index ebcdf700d3b6e..14bfd7a74400f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockRequestBuilder.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.readonly; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.cluster.metadata.IndexMetadata.APIBlock; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java index 3ab64fa55af8b..c6677268109c2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/AddIndexBlockResponse.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.readonly; import org.opensearch.OpenSearchException; -import org.opensearch.action.support.master.ShardsAcknowledgedResponse; +import org.opensearch.action.support.clustermanager.ShardsAcknowledgedResponse; import org.opensearch.common.Nullable; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.action.support.DefaultShardOperationFailedException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java index 68c0076bbd302..54f124f39bd3a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverRequest.java @@ -36,7 +36,7 @@ import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java index b7df35cd480bb..dc5292c89fb9c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/RolloverResponse.java @@ -32,7 +32,7 @@ package org.opensearch.action.admin.indices.rollover; -import org.opensearch.action.support.master.ShardsAcknowledgedResponse; +import org.opensearch.action.support.clustermanager.ShardsAcknowledgedResponse; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index 779b136abef5c..c7457ca7cb137 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -36,8 +36,8 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ack.ClusterStateUpdateResponse; import org.opensearch.cluster.block.ClusterBlockException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsAction.java index 2333a2aad6bc6..aa26acb7e3fc5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.settings.put; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Action for updating index settings diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java index 45172e313dfcc..a2282ef453189 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequest.java @@ -35,7 +35,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java index 08d7a240aa007..9c8c5b3025aa2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/UpdateSettingsRequestBuilder.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.settings.put; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java index a5225f2243876..1459cc124c3bc 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java @@ -38,7 +38,7 @@ import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java index f9d90d46b0904..d645784ab82b1 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequestBuilder.java @@ -34,7 +34,7 @@ import org.opensearch.action.ActionType; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.support.ActiveShardCount; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java index 1f427a349c2ea..a4478686dc87c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComponentTemplateAction.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ActionType; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateAction.java index 496358cdfd2b1..43d46f3f35526 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateAction.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ActionType; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java index 789d03f8e8d8c..5773fcf93c49e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.template.delete; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action for deleting an index template diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java index 60771cfa453ae..809624f66e417 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/DeleteIndexTemplateRequestBuilder.java @@ -31,8 +31,8 @@ package org.opensearch.action.admin.indices.template.delete; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java index 30cb0cb3e5d00..d1fe08cb5926c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java @@ -35,8 +35,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java index 27ea64809e3a7..53098447112ac 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java @@ -35,8 +35,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java index c9542c7a58810..a607166942d4c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java @@ -35,8 +35,8 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComponentTemplateAction.java index d12f99ec345d3..c759638e5dc4e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComponentTemplateAction.java @@ -34,8 +34,8 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.ActionType; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.metadata.ComponentTemplate; import org.opensearch.common.Nullable; import org.opensearch.core.common.Strings; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java index ed209e18b64ef..abd0b1ad44561 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutComposableIndexTemplateAction.java @@ -36,8 +36,8 @@ import org.opensearch.action.ActionType; import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.metadata.ComposableIndexTemplate; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Nullable; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateAction.java index 06a9f6fbba409..eb21b81350fda 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.template.put; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * An action for putting an index template into the cluster state diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java index 931d12de574ae..3b423e42e25e5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.indices.template.put; import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeOperationRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index 6eb87bee9ffa7..66e2fe5c535db 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.template.put; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java index 8a31c36d723b4..a5c3590a0a6d7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java @@ -33,8 +33,8 @@ package org.opensearch.action.admin.indices.template.put; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index 4431949c2e42b..b9f27c00d0d98 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -35,8 +35,8 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/tiering/HotToWarmTieringResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/tiering/HotToWarmTieringResponse.java index 275decf7a8ea5..3a74987cfff42 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/tiering/HotToWarmTieringResponse.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/tiering/HotToWarmTieringResponse.java @@ -8,7 +8,7 @@ package org.opensearch.action.admin.indices.tiering; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringIndexRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringIndexRequest.java index ed458a47ddb7d..df08b14d85953 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringIndexRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/tiering/TieringIndexRequest.java @@ -11,7 +11,7 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.IndicesRequest; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java index 286724d78bb63..7e5744bfc5674 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java @@ -36,8 +36,8 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ack.ClusterStateUpdateResponse; import org.opensearch.cluster.block.ClusterBlockException; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsAction.java index 05944e781d109..4c42b4abbf678 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.admin.indices.upgrade.post; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action for upgrading index settings diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsRequest.java index 306d29dd84f13..40cdfdaa811d9 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsRequest.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/UpgradeSettingsRequest.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.collect.Tuple; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java index abb3c3f4db5f6..413e6700a758c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/DeleteViewAction.java @@ -12,9 +12,9 @@ import org.opensearch.action.ActionType; import org.opensearch.action.ValidateActions; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java index 294f88decba1f..a7b43b1f51df9 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/view/ViewService.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.action.search.SearchAction; import org.opensearch.action.search.SearchResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; diff --git a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineAction.java b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineAction.java index 6017be9747912..82bb78a9b89d6 100644 --- a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.ingest; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action to delete a pipeline diff --git a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java index b9d916e152c3d..8c851dbc86292 100644 --- a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequest.java @@ -33,7 +33,7 @@ package org.opensearch.action.ingest; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequestBuilder.java b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequestBuilder.java index bc253db85bb0f..12ae44c6c2956 100644 --- a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineRequestBuilder.java @@ -33,7 +33,7 @@ package org.opensearch.action.ingest; import org.opensearch.action.ActionRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; diff --git a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java index fe68f06d0d32e..05db9927c906f 100644 --- a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java @@ -33,8 +33,8 @@ package org.opensearch.action.ingest; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineAction.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineAction.java index 1fcbd783d246b..be47bff8f4e92 100644 --- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineAction.java @@ -33,7 +33,7 @@ package org.opensearch.action.ingest; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Transport action to put a new pipeline diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java index 06e89b5f2908b..40585b26b5d92 100644 --- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequest.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java index e8d6a4d332319..42ac84186eaa2 100644 --- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineRequestBuilder.java @@ -33,7 +33,7 @@ package org.opensearch.action.ingest; import org.opensearch.action.ActionRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.bytes.BytesReference; diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java index e2d206e8c4f6d..d8e82ece056b5 100644 --- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java @@ -35,8 +35,8 @@ import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.OriginSettingClient; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterState; diff --git a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineAction.java b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineAction.java index 65f8cf3de9506..372cee6323d70 100644 --- a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineAction.java +++ b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineAction.java @@ -9,7 +9,7 @@ package org.opensearch.action.search; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Action type to delete a search pipeline diff --git a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java index 0cde81203063c..083bd9307ffb8 100644 --- a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineRequest.java @@ -9,7 +9,7 @@ package org.opensearch.action.search; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineTransportAction.java b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineTransportAction.java index ac83a6bb6b765..652791d3105f6 100644 --- a/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/search/DeleteSearchPipelineTransportAction.java @@ -9,8 +9,8 @@ package org.opensearch.action.search; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.block.ClusterBlockLevel; diff --git a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineAction.java b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineAction.java index 798c8211ee505..5fff11f1298c4 100644 --- a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineAction.java +++ b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineAction.java @@ -9,7 +9,7 @@ package org.opensearch.action.search; import org.opensearch.action.ActionType; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; /** * Action type to put a new search pipeline diff --git a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java index 15b4ea648af29..a5ef4e36df832 100644 --- a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java +++ b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineRequest.java @@ -10,7 +10,7 @@ import org.opensearch.Version; import org.opensearch.action.ActionRequestValidationException; -import org.opensearch.action.support.master.AcknowledgedRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesReference; diff --git a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineTransportAction.java b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineTransportAction.java index 903b7dfce09c0..312c73698e27f 100644 --- a/server/src/main/java/org/opensearch/action/search/PutSearchPipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/search/PutSearchPipelineTransportAction.java @@ -11,8 +11,8 @@ import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest; import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.OriginSettingClient; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.ClusterState; diff --git a/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java b/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java index 5eb305d91ee04..f93508b0ba55b 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java +++ b/server/src/main/java/org/opensearch/action/search/SearchResponseSections.java @@ -121,6 +121,29 @@ public SearchResponseSections( this.searchExtBuilders.addAll(Objects.requireNonNull(searchExtBuilders, "searchExtBuilders must not be null")); } + public SearchResponseSections( + SearchHits hits, + Aggregations aggregations, + Suggest suggest, + boolean timedOut, + Boolean terminatedEarly, + SearchProfileShardResults profileResults, + int numReducePhases, + List searchExtBuilders + ) { + this( + hits, + aggregations, + suggest, + timedOut, + terminatedEarly, + profileResults, + numReducePhases, + searchExtBuilders, + Collections.emptyList() + ); + } + public final boolean timedOut() { return this.timedOut; } diff --git a/server/src/main/java/org/opensearch/action/search/TransportMultiSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportMultiSearchAction.java index 146b4010af4b3..dcb2ce6eb88da 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportMultiSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportMultiSearchAction.java @@ -44,6 +44,9 @@ import org.opensearch.common.util.concurrent.OpenSearchExecutors; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.tasks.TaskCancelledException; +import org.opensearch.core.tasks.TaskId; +import org.opensearch.tasks.CancellableTask; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -193,6 +196,19 @@ private void handleResponse(final int responseSlot, final MultiSearchResponse.It if (responseCounter.decrementAndGet() == 0) { assert requests.isEmpty(); finish(); + } else if (isCancelled(request.request.getParentTask())) { + // Drain the rest of the queue + SearchRequestSlot request; + while ((request = requests.poll()) != null) { + responses.set( + request.responseSlot, + new MultiSearchResponse.Item(null, new TaskCancelledException("Parent task was cancelled")) + ); + if (responseCounter.decrementAndGet() == 0) { + assert requests.isEmpty(); + finish(); + } + } } else { if (thread == Thread.currentThread()) { // we are on the same thread, we need to fork to another thread to avoid recursive stack overflow on a single thread @@ -220,6 +236,14 @@ private long buildTookInMillis() { }); } + private boolean isCancelled(TaskId taskId) { + if (taskId.isSet()) { + CancellableTask task = taskManager.getCancellableTask(taskId.getId()); + return task != null && task.isCancelled(); + } + return false; + } + /** * Slots a search request * diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index dfec2e1fda738..898174d60de76 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -476,7 +476,7 @@ private void executeRequest( } else { Rewriteable.rewriteAndFetch( sr.source(), - searchService.getRewriteContext(timeProvider::getAbsoluteStartMillis), + searchService.getRewriteContext(timeProvider::getAbsoluteStartMillis, searchRequest), rewriteListener ); } diff --git a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedRequest.java b/server/src/main/java/org/opensearch/action/support/clustermanager/AcknowledgedRequest.java similarity index 96% rename from server/src/main/java/org/opensearch/action/support/master/AcknowledgedRequest.java rename to server/src/main/java/org/opensearch/action/support/clustermanager/AcknowledgedRequest.java index 59f238a202788..4543c9392e62b 100644 --- a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedRequest.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/AcknowledgedRequest.java @@ -29,8 +29,9 @@ * GitHub history for details. */ -package org.opensearch.action.support.master; +package org.opensearch.action.support.clustermanager; +import org.opensearch.action.support.master.MasterNodeRequest; import org.opensearch.cluster.ack.AckedRequest; import org.opensearch.common.unit.TimeValue; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedRequestBuilder.java b/server/src/main/java/org/opensearch/action/support/clustermanager/AcknowledgedRequestBuilder.java similarity index 94% rename from server/src/main/java/org/opensearch/action/support/master/AcknowledgedRequestBuilder.java rename to server/src/main/java/org/opensearch/action/support/clustermanager/AcknowledgedRequestBuilder.java index 7a0824c6d30ca..ea50774d17d19 100644 --- a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedRequestBuilder.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/AcknowledgedRequestBuilder.java @@ -29,9 +29,10 @@ * GitHub history for details. */ -package org.opensearch.action.support.master; +package org.opensearch.action.support.clustermanager; import org.opensearch.action.ActionType; +import org.opensearch.action.support.master.MasterNodeOperationRequestBuilder; import org.opensearch.client.OpenSearchClient; import org.opensearch.common.unit.TimeValue; diff --git a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java b/server/src/main/java/org/opensearch/action/support/clustermanager/AcknowledgedResponse.java similarity index 98% rename from server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java rename to server/src/main/java/org/opensearch/action/support/clustermanager/AcknowledgedResponse.java index 279ad401f7e56..d93a1133789fa 100644 --- a/server/src/main/java/org/opensearch/action/support/master/AcknowledgedResponse.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/AcknowledgedResponse.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.action.support.master; +package org.opensearch.action.support.clustermanager; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.ParseField; diff --git a/server/src/main/java/org/opensearch/action/support/master/ShardsAcknowledgedResponse.java b/server/src/main/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponse.java similarity index 98% rename from server/src/main/java/org/opensearch/action/support/master/ShardsAcknowledgedResponse.java rename to server/src/main/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponse.java index fd54e810528d3..7173d8462bbd5 100644 --- a/server/src/main/java/org/opensearch/action/support/master/ShardsAcknowledgedResponse.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponse.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.action.support.master; +package org.opensearch.action.support.clustermanager; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java index 4e167d10b99fa..95498f2bcbcd1 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/opensearch/bootstrap/Bootstrap.java @@ -47,6 +47,7 @@ import org.opensearch.cli.UserException; import org.opensearch.common.PidFile; import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.bootstrap.JarHell; import org.opensearch.common.inject.CreationException; import org.opensearch.common.logging.LogConfigurator; import org.opensearch.common.logging.Loggers; diff --git a/server/src/main/java/org/opensearch/bootstrap/Security.java b/server/src/main/java/org/opensearch/bootstrap/Security.java index 563a026109059..acf2d7ec6a5ac 100644 --- a/server/src/main/java/org/opensearch/bootstrap/Security.java +++ b/server/src/main/java/org/opensearch/bootstrap/Security.java @@ -34,6 +34,7 @@ import org.opensearch.cli.Command; import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.bootstrap.JarHell; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; diff --git a/server/src/main/java/org/opensearch/client/ClusterAdminClient.java b/server/src/main/java/org/opensearch/client/ClusterAdminClient.java index 5ce4d442fbe0b..23cb267446d7d 100644 --- a/server/src/main/java/org/opensearch/client/ClusterAdminClient.java +++ b/server/src/main/java/org/opensearch/client/ClusterAdminClient.java @@ -157,7 +157,7 @@ import org.opensearch.action.search.GetSearchPipelineRequest; import org.opensearch.action.search.GetSearchPipelineResponse; import org.opensearch.action.search.PutSearchPipelineRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.action.ActionListener; diff --git a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java index 588584cd8a280..e68e1faddb1a8 100644 --- a/server/src/main/java/org/opensearch/client/IndicesAdminClient.java +++ b/server/src/main/java/org/opensearch/client/IndicesAdminClient.java @@ -128,7 +128,7 @@ import org.opensearch.action.admin.indices.view.CreateViewAction; import org.opensearch.action.admin.indices.view.DeleteViewAction; import org.opensearch.action.admin.indices.view.GetViewAction; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.metadata.IndexMetadata.APIBlock; import org.opensearch.common.Nullable; import org.opensearch.common.action.ActionFuture; diff --git a/server/src/main/java/org/opensearch/client/support/AbstractClient.java b/server/src/main/java/org/opensearch/client/support/AbstractClient.java index f4683ab516cef..07747647535f8 100644 --- a/server/src/main/java/org/opensearch/client/support/AbstractClient.java +++ b/server/src/main/java/org/opensearch/client/support/AbstractClient.java @@ -395,7 +395,7 @@ import org.opensearch.action.search.SearchScrollRequest; import org.opensearch.action.search.SearchScrollRequestBuilder; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.termvectors.MultiTermVectorsAction; import org.opensearch.action.termvectors.MultiTermVectorsRequest; import org.opensearch.action.termvectors.MultiTermVectorsRequestBuilder; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index f70282986ad4e..cee331788e4b7 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -230,7 +230,15 @@ static Setting buildNumberOfShardsSetting() { + "]" ); } - return Setting.intSetting(SETTING_NUMBER_OF_SHARDS, defaultNumShards, 1, maxNumShards, Property.IndexScope, Property.Final); + return Setting.intSetting( + SETTING_NUMBER_OF_SHARDS, + defaultNumShards, + 1, + maxNumShards, + Property.IndexScope, + Property.Final, + Property.UnmodifiableOnRestore + ); } public static final String INDEX_SETTING_PREFIX = "index."; @@ -559,13 +567,15 @@ public static APIBlock readFrom(StreamInput input) throws IOException { SETTING_VERSION_CREATED, Version.V_EMPTY, Property.IndexScope, - Property.PrivateIndex + Property.PrivateIndex, + Property.UnmodifiableOnRestore ); public static final String SETTING_VERSION_CREATED_STRING = "index.version.created_string"; public static final String SETTING_VERSION_UPGRADED = "index.version.upgraded"; public static final String SETTING_VERSION_UPGRADED_STRING = "index.version.upgraded_string"; public static final String SETTING_CREATION_DATE = "index.creation_date"; + /** * The user provided name for an index. This is the plain string provided by the user when the index was created. * It might still contain date math expressions etc. (added in 5.0) @@ -589,6 +599,7 @@ public static APIBlock readFrom(StreamInput input) throws IOException { Function.identity(), Property.IndexScope ); + public static final String INDEX_UUID_NA_VALUE = Strings.UNKNOWN_UUID_VALUE; public static final String INDEX_ROUTING_REQUIRE_GROUP_PREFIX = "index.routing.allocation.require"; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateDataStreamService.java index 22ad21f54e556..1cf68d8b29a6f 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -38,7 +38,7 @@ import org.opensearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.opensearch.action.support.ActiveShardCount; import org.opensearch.action.support.ActiveShardsObserver; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.AckedClusterStateUpdateTask; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ack.ClusterStateUpdateRequest; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index b5b2b71f977fa..f052e9940bb9a 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -231,6 +231,10 @@ public MetadataCreateIndexService( : null; } + public IndexScopedSettings getIndexScopedSettings() { + return indexScopedSettings; + } + /** * Add a provider to be invoked to get additional index settings prior to an index being created */ diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java index e4afc798cc64d..3af18470df787 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java @@ -38,8 +38,8 @@ import org.apache.lucene.util.automaton.Operations; import org.opensearch.Version; import org.opensearch.action.admin.indices.alias.Alias; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateUpdateTask; import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/TemplateUpgradeService.java b/server/src/main/java/org/opensearch/cluster/metadata/TemplateUpgradeService.java index 10f458561bffe..001c30e14ebbe 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/TemplateUpgradeService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/TemplateUpgradeService.java @@ -38,7 +38,7 @@ import org.opensearch.Version; import org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterState; diff --git a/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java index 7655135b06d6c..8c10623e48fe4 100644 --- a/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java @@ -759,6 +759,14 @@ public boolean isFinalSetting(String key) { return setting != null && setting.isFinal(); } + /** + * Returns true if the setting for the given key is unmodifiableOnRestore. Otherwise false. + */ + public boolean isUnmodifiableOnRestoreSetting(String key) { + final Setting setting = get(key); + return setting != null && setting.isUnmodifiableOnRestore(); + } + /** * Returns a settings object that contains all settings that are not * already set in the given source. The diff contains either the default value for each diff --git a/server/src/main/java/org/opensearch/common/settings/Setting.java b/server/src/main/java/org/opensearch/common/settings/Setting.java index 081029c1c106c..eb63522270e87 100644 --- a/server/src/main/java/org/opensearch/common/settings/Setting.java +++ b/server/src/main/java/org/opensearch/common/settings/Setting.java @@ -171,7 +171,13 @@ public enum Property { /** * Extension scope */ - ExtensionScope + ExtensionScope, + + /** + * Mark this setting as immutable on snapshot restore + * i.e. the setting will not be allowed to be removed or modified during restore + */ + UnmodifiableOnRestore } private final Key key; @@ -208,10 +214,13 @@ private Setting( final EnumSet propertiesAsSet = EnumSet.copyOf(Arrays.asList(properties)); if (propertiesAsSet.contains(Property.Dynamic) && propertiesAsSet.contains(Property.Final)) { throw new IllegalArgumentException("final setting [" + key + "] cannot be dynamic"); + } else if (propertiesAsSet.contains(Property.UnmodifiableOnRestore) && propertiesAsSet.contains(Property.Dynamic)) { + throw new IllegalArgumentException("UnmodifiableOnRestore setting [" + key + "] cannot be dynamic"); } checkPropertyRequiresIndexScope(propertiesAsSet, Property.NotCopyableOnResize); checkPropertyRequiresIndexScope(propertiesAsSet, Property.InternalIndex); checkPropertyRequiresIndexScope(propertiesAsSet, Property.PrivateIndex); + checkPropertyRequiresIndexScope(propertiesAsSet, Property.UnmodifiableOnRestore); checkPropertyRequiresNodeScope(propertiesAsSet, Property.Consistent); this.properties = propertiesAsSet; } @@ -348,6 +357,10 @@ public final boolean isFinal() { return properties.contains(Property.Final); } + public final boolean isUnmodifiableOnRestore() { + return properties.contains(Property.UnmodifiableOnRestore); + } + public final boolean isInternalIndex() { return properties.contains(Property.InternalIndex); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java index df2ce9096bfc1..26390faa20dc1 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java @@ -10,6 +10,7 @@ import org.apache.lucene.store.RandomAccessInput; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; +import org.opensearch.search.startree.StarTreeNodeCollector; import java.io.IOException; import java.io.UncheckedIOException; @@ -200,7 +201,21 @@ public StarTreeNode getChildForDimensionValue(Long dimensionValue) throws IOExce StarTreeNode resultStarTreeNode = null; if (null != dimensionValue) { - resultStarTreeNode = binarySearchChild(dimensionValue); + resultStarTreeNode = binarySearchChild(dimensionValue, null); + } + return resultStarTreeNode; + } + + @Override + public StarTreeNode getChildForDimensionValue(Long dimensionValue, StarTreeNode lastMatchedChild) throws IOException { + // there will be no children for leaf nodes + if (isLeaf()) { + return null; + } + + StarTreeNode resultStarTreeNode = null; + if (null != dimensionValue) { + resultStarTreeNode = binarySearchChild(dimensionValue, lastMatchedChild); } return resultStarTreeNode; } @@ -240,21 +255,29 @@ private static FixedLengthStarTreeNode matchStarTreeNodeTypeOrNull(FixedLengthSt * @return The child node if found, null otherwise * @throws IOException If there's an error reading from the input */ - private FixedLengthStarTreeNode binarySearchChild(long dimensionValue) throws IOException { + private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, StarTreeNode lastMatchedNode) throws IOException { int low = firstChildId; - // if the current node is star node, increment the low to reduce the search space - if (matchStarTreeNodeTypeOrNull(new FixedLengthStarTreeNode(in, firstChildId), StarTreeNodeType.STAR) != null) { - low++; - } - int high = getInt(LAST_CHILD_ID_OFFSET); // if the current node is null node, decrement the high to reduce the search space if (matchStarTreeNodeTypeOrNull(new FixedLengthStarTreeNode(in, high), StarTreeNodeType.NULL) != null) { high--; } + if (lastMatchedNode instanceof FixedLengthStarTreeNode) { + int lastMatchedNodeId = ((FixedLengthStarTreeNode) lastMatchedNode).nodeId(); + // Start the binary search from node after the last matched as low. + if ((lastMatchedNodeId + 1) <= high) { + low = lastMatchedNodeId + 1; + } else { + return null; + } + } else if (matchStarTreeNodeTypeOrNull(new FixedLengthStarTreeNode(in, low), StarTreeNodeType.STAR) != null) { + // if the current node is star node, increment the low to reduce the search space + low++; + } + while (low <= high) { int mid = low + (high - low) / 2; FixedLengthStarTreeNode midNode = new FixedLengthStarTreeNode(in, mid); @@ -271,6 +294,100 @@ private FixedLengthStarTreeNode binarySearchChild(long dimensionValue) throws IO return null; } + @Override + public void collectChildrenInRange(long low, long high, StarTreeNodeCollector collector) throws IOException { + if (low <= high) { + FixedLengthStarTreeNode lowStarTreeNode = binarySearchChild(low, true, null); + if (lowStarTreeNode != null) { + FixedLengthStarTreeNode highStarTreeNode = binarySearchChild(high, false, lowStarTreeNode); + if (highStarTreeNode != null) { + for (int lowNodeId = lowStarTreeNode.nodeId(); lowNodeId <= highStarTreeNode.nodeId(); ++lowNodeId) { + collector.collectStarTreeNode(new FixedLengthStarTreeNode(in, lowNodeId)); + } + } else if (lowStarTreeNode.getDimensionValue() <= high) { // Low StarTreeNode is the last default node for that dimension. + collector.collectStarTreeNode(lowStarTreeNode); + } + } + } + } + + /** + * + * @param dimensionValue : The dimension to match. + * @param matchNextHighest : If true then we try to return @dimensionValue or the next Highest. Else, we return @dimensionValue or the next Lowest. + * @param lastMatchedNode : If not null, we begin the binary search from the node after this. + * @return : Matched node or null. + * @throws IOException : + */ + private FixedLengthStarTreeNode binarySearchChild(long dimensionValue, boolean matchNextHighest, StarTreeNode lastMatchedNode) + throws IOException { + + int low = firstChildId; + int tempLow = low; + int starNodeId, nullNodeId; + starNodeId = nullNodeId = Integer.MIN_VALUE; + + // if the current node is star node, increment the tempLow to reduce the search space + if (matchStarTreeNodeTypeOrNull(new FixedLengthStarTreeNode(in, tempLow), StarTreeNodeType.STAR) != null) { + starNodeId = tempLow; + tempLow++; + } + + int high = getInt(LAST_CHILD_ID_OFFSET); + int tempHigh = high; + // if the current node is null node, decrement the tempHigh to reduce the search space + if (matchStarTreeNodeTypeOrNull(new FixedLengthStarTreeNode(in, tempHigh), StarTreeNodeType.NULL) != null) { + nullNodeId = tempHigh; + tempHigh--; + } + + if (lastMatchedNode instanceof FixedLengthStarTreeNode) { + int lastMatchedNodeId = ((FixedLengthStarTreeNode) lastMatchedNode).nodeId(); + // Start the binary search from node after the last matched as low. + if ((lastMatchedNodeId + 1) <= tempHigh) { + tempLow = lastMatchedNodeId + 1; + } else { + return null; + } + } + + while (tempLow <= tempHigh) { + int mid = tempLow + (tempHigh - tempLow) / 2; + FixedLengthStarTreeNode midNode = new FixedLengthStarTreeNode(in, mid); + long midDimensionValue = midNode.getDimensionValue(); + + if (midDimensionValue == dimensionValue) { + return midNode; + } else { + if (midDimensionValue < dimensionValue) { // Going to the right from mid to search next + tempLow = mid + 1; + // We are going out of bounds for this dimension on the right side. + if (tempLow > high || tempLow == nullNodeId) { + return matchNextHighest ? null : midNode; + } else { + FixedLengthStarTreeNode nodeGreaterThanMid = new FixedLengthStarTreeNode(in, tempLow); + if (nodeGreaterThanMid.getDimensionValue() > dimensionValue) { + return matchNextHighest ? nodeGreaterThanMid : midNode; + } + } + } else { // Going to the left from mid to search next + tempHigh = mid - 1; + // We are going out of bounds for this dimension on the left side. + if (tempHigh < low || tempHigh == starNodeId) { + return matchNextHighest ? midNode : null; + } else { + FixedLengthStarTreeNode nodeLessThanMid = new FixedLengthStarTreeNode(in, tempHigh); + if (nodeLessThanMid.getDimensionValue() < dimensionValue) { + return matchNextHighest ? midNode : nodeLessThanMid; + } + } + } + } + } + return null; + + } + @Override public Iterator getChildrenIterator() throws IOException { return new Iterator<>() { @@ -297,4 +414,8 @@ public void remove() { } }; } + + public int nodeId() { + return nodeId; + } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java index fce3e30e9ebf6..dfdbdff09a988 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java @@ -9,6 +9,7 @@ package org.opensearch.index.compositeindex.datacube.startree.node; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.search.startree.StarTreeNodeCollector; import java.io.IOException; import java.util.Iterator; @@ -109,6 +110,17 @@ public interface StarTreeNode { */ StarTreeNode getChildForDimensionValue(Long dimensionValue) throws IOException; + StarTreeNode getChildForDimensionValue(Long dimensionValue, StarTreeNode lastMatchedChild) throws IOException; + + /** + * Collects all matching child nodes whose dimension values lie within the range of low and high, both inclusive. + * @param low : Starting of the range ( inclusive ) + * @param high : End of the range ( inclusive ) + * @param collector : Collector to collect the matched child StarTreeNode's + * @throws IOException : + */ + void collectChildrenInRange(long low, long high, StarTreeNodeCollector collector) throws IOException; + /** * Returns the child star node for a node in the star-tree. * diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeQueryHelper.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeQueryHelper.java deleted file mode 100644 index e46cf6f56b36e..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeQueryHelper.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.utils; - -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SegmentReader; -import org.apache.lucene.search.CollectionTerminatedException; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.FixedBitSet; -import org.opensearch.common.lucene.Lucene; -import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; -import org.opensearch.index.codec.composite.CompositeIndexReader; -import org.opensearch.index.compositeindex.datacube.Dimension; -import org.opensearch.index.compositeindex.datacube.Metric; -import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; -import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; -import org.opensearch.index.mapper.CompositeDataCubeFieldType; -import org.opensearch.index.query.MatchAllQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.TermQueryBuilder; -import org.opensearch.search.aggregations.AggregatorFactory; -import org.opensearch.search.aggregations.LeafBucketCollector; -import org.opensearch.search.aggregations.LeafBucketCollectorBase; -import org.opensearch.search.aggregations.metrics.MetricAggregatorFactory; -import org.opensearch.search.aggregations.support.ValuesSource; -import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.internal.SearchContext; -import org.opensearch.search.startree.StarTreeFilter; -import org.opensearch.search.startree.StarTreeQueryContext; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -/** - * Helper class for building star-tree query - * - * @opensearch.internal - * @opensearch.experimental - */ -public class StarTreeQueryHelper { - - /** - * Checks if the search context can be supported by star-tree - */ - public static boolean isStarTreeSupported(SearchContext context) { - return context.aggregations() != null && context.mapperService().isCompositeIndexPresent() && context.parsedPostFilter() == null; - } - - /** - * Gets StarTreeQueryContext from the search context and source builder. - * Returns null if the query and aggregation cannot be supported. - */ - public static StarTreeQueryContext getStarTreeQueryContext(SearchContext context, SearchSourceBuilder source) throws IOException { - // Current implementation assumes only single star-tree is supported - CompositeDataCubeFieldType compositeMappedFieldType = (CompositeDataCubeFieldType) context.mapperService() - .getCompositeFieldTypes() - .iterator() - .next(); - CompositeIndexFieldInfo starTree = new CompositeIndexFieldInfo( - compositeMappedFieldType.name(), - compositeMappedFieldType.getCompositeIndexType() - ); - - for (AggregatorFactory aggregatorFactory : context.aggregations().factories().getFactories()) { - MetricStat metricStat = validateStarTreeMetricSupport(compositeMappedFieldType, aggregatorFactory); - if (metricStat == null) { - return null; - } - } - - // need to cache star tree values only for multiple aggregations - boolean cacheStarTreeValues = context.aggregations().factories().getFactories().length > 1; - int cacheSize = cacheStarTreeValues ? context.indexShard().segments(false).size() : -1; - - return StarTreeQueryHelper.tryCreateStarTreeQueryContext(starTree, compositeMappedFieldType, source.query(), cacheSize); - } - - /** - * Uses query builder and composite index info to form star-tree query context - */ - private static StarTreeQueryContext tryCreateStarTreeQueryContext( - CompositeIndexFieldInfo compositeIndexFieldInfo, - CompositeDataCubeFieldType compositeFieldType, - QueryBuilder queryBuilder, - int cacheStarTreeValuesSize - ) { - Map queryMap; - if (queryBuilder == null || queryBuilder instanceof MatchAllQueryBuilder) { - queryMap = null; - } else if (queryBuilder instanceof TermQueryBuilder) { - // TODO: Add support for keyword fields - if (compositeFieldType.getDimensions().stream().anyMatch(d -> d.getDocValuesType() != DocValuesType.SORTED_NUMERIC)) { - // return null for non-numeric fields - return null; - } - - List supportedDimensions = compositeFieldType.getDimensions() - .stream() - .map(Dimension::getField) - .collect(Collectors.toList()); - queryMap = getStarTreePredicates(queryBuilder, supportedDimensions); - if (queryMap == null) { - return null; - } - } else { - return null; - } - return new StarTreeQueryContext(compositeIndexFieldInfo, queryMap, cacheStarTreeValuesSize); - } - - /** - * Parse query body to star-tree predicates - * @param queryBuilder to match star-tree supported query shape - * @return predicates to match - */ - private static Map getStarTreePredicates(QueryBuilder queryBuilder, List supportedDimensions) { - TermQueryBuilder tq = (TermQueryBuilder) queryBuilder; - String field = tq.fieldName(); - if (!supportedDimensions.contains(field)) { - return null; - } - long inputQueryVal = Long.parseLong(tq.value().toString()); - - // Create a map with the field and the value - Map predicateMap = new HashMap<>(); - predicateMap.put(field, inputQueryVal); - return predicateMap; - } - - private static MetricStat validateStarTreeMetricSupport( - CompositeDataCubeFieldType compositeIndexFieldInfo, - AggregatorFactory aggregatorFactory - ) { - if (aggregatorFactory instanceof MetricAggregatorFactory && aggregatorFactory.getSubFactories().getFactories().length == 0) { - String field; - Map> supportedMetrics = compositeIndexFieldInfo.getMetrics() - .stream() - .collect(Collectors.toMap(Metric::getField, Metric::getMetrics)); - - MetricStat metricStat = ((MetricAggregatorFactory) aggregatorFactory).getMetricStat(); - field = ((MetricAggregatorFactory) aggregatorFactory).getField(); - - if (field != null && supportedMetrics.containsKey(field) && supportedMetrics.get(field).contains(metricStat)) { - return metricStat; - } - } - return null; - } - - public static CompositeIndexFieldInfo getSupportedStarTree(SearchContext context) { - StarTreeQueryContext starTreeQueryContext = context.getStarTreeQueryContext(); - return (starTreeQueryContext != null) ? starTreeQueryContext.getStarTree() : null; - } - - public static StarTreeValues getStarTreeValues(LeafReaderContext context, CompositeIndexFieldInfo starTree) throws IOException { - SegmentReader reader = Lucene.segmentReader(context.reader()); - if (!(reader.getDocValuesReader() instanceof CompositeIndexReader)) { - return null; - } - CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); - return (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(starTree); - } - - /** - * Get the star-tree leaf collector - * This collector computes the aggregation prematurely and invokes an early termination collector - */ - public static LeafBucketCollector getStarTreeLeafCollector( - SearchContext context, - ValuesSource.Numeric valuesSource, - LeafReaderContext ctx, - LeafBucketCollector sub, - CompositeIndexFieldInfo starTree, - String metric, - Consumer valueConsumer, - Runnable finalConsumer - ) throws IOException { - StarTreeValues starTreeValues = getStarTreeValues(ctx, starTree); - assert starTreeValues != null; - String fieldName = ((ValuesSource.Numeric.FieldData) valuesSource).getIndexFieldName(); - String metricName = StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues(starTree.getField(), fieldName, metric); - - assert starTreeValues != null; - SortedNumericStarTreeValuesIterator valuesIterator = (SortedNumericStarTreeValuesIterator) starTreeValues.getMetricValuesIterator( - metricName - ); - // Obtain a FixedBitSet of matched star tree document IDs - FixedBitSet filteredValues = getStarTreeFilteredValues(context, ctx, starTreeValues); - assert filteredValues != null; - - int numBits = filteredValues.length(); // Get the number of the filtered values (matching docs) - if (numBits > 0) { - // Iterate over the filtered values - for (int bit = filteredValues.nextSetBit(0); bit != DocIdSetIterator.NO_MORE_DOCS; bit = (bit + 1 < numBits) - ? filteredValues.nextSetBit(bit + 1) - : DocIdSetIterator.NO_MORE_DOCS) { - // Advance to the entryId in the valuesIterator - if (valuesIterator.advanceExact(bit) == false) { - continue; // Skip if no more entries - } - - // Iterate over the values for the current entryId - for (int i = 0, count = valuesIterator.entryValueCount(); i < count; i++) { - long value = valuesIterator.nextValue(); - valueConsumer.accept(value); // Apply the consumer operation (e.g., max, sum) - } - } - } - - // Call the final consumer after processing all entries - finalConsumer.run(); - - // Return a LeafBucketCollector that terminates collection - return new LeafBucketCollectorBase(sub, valuesSource.doubleValues(ctx)) { - @Override - public void collect(int doc, long bucket) { - throw new CollectionTerminatedException(); - } - }; - } - - /** - * Get the filtered values for the star-tree query - * Cache the results in case of multiple aggregations (if cache is initialized) - * @return FixedBitSet of matched document IDs - */ - public static FixedBitSet getStarTreeFilteredValues(SearchContext context, LeafReaderContext ctx, StarTreeValues starTreeValues) - throws IOException { - FixedBitSet result = context.getStarTreeQueryContext().getStarTreeValues(ctx); - if (result == null) { - result = StarTreeFilter.getStarTreeResult(starTreeValues, context.getStarTreeQueryContext().getQueryMap()); - context.getStarTreeQueryContext().setStarTreeValues(ctx, result); - } - return result; - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java index 4b4bfa6a915eb..595965c98ea07 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java @@ -26,6 +26,11 @@ public SortedNumericStarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { super(docIdSetIterator); } + @Override + public long value() throws IOException { + return nextValue(); + } + public long nextValue() throws IOException { return ((SortedNumericDocValues) docIdSetIterator).nextValue(); } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedSetStarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedSetStarTreeValuesIterator.java index 0cddffe5877e9..f312dc48a9fd3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedSetStarTreeValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedSetStarTreeValuesIterator.java @@ -29,6 +29,16 @@ public SortedSetStarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { super(docIdSetIterator); } + @Override + public long value() throws IOException { + return nextOrd(); + } + + @Override + public boolean advanceExact(int target) throws IOException { + return ((SortedSetDocValues) docIdSetIterator).advanceExact(target); + } + public long nextOrd() throws IOException { return ((SortedSetDocValues) docIdSetIterator).nextOrd(); } @@ -56,4 +66,5 @@ public TermsEnum termsEnum() throws IOException { public TermsEnum intersect(CompiledAutomaton automaton) throws IOException { return ((SortedSetDocValues) docIdSetIterator).intersect(automaton); } + } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java index 32866f3e50092..d9ee67fe1b0d5 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java @@ -45,4 +45,9 @@ public int advance(int target) throws IOException { public long cost() { return docIdSetIterator.cost(); } + + public abstract long value() throws IOException; + + public abstract boolean advanceExact(int target) throws IOException; + } diff --git a/server/src/main/java/org/opensearch/index/fieldvisitor/CustomFieldsVisitor.java b/server/src/main/java/org/opensearch/index/fieldvisitor/CustomFieldsVisitor.java index df4d398b2b181..8e6799f6bf74c 100644 --- a/server/src/main/java/org/opensearch/index/fieldvisitor/CustomFieldsVisitor.java +++ b/server/src/main/java/org/opensearch/index/fieldvisitor/CustomFieldsVisitor.java @@ -52,6 +52,11 @@ public CustomFieldsVisitor(Set fields, boolean loadSource) { this.fields = fields; } + public CustomFieldsVisitor(Set fields, boolean loadSource, String[] includes, String[] excludes) { + super(loadSource, includes, excludes); + this.fields = fields; + } + @Override public Status needsField(FieldInfo fieldInfo) { if (super.needsField(fieldInfo) == Status.YES) { diff --git a/server/src/main/java/org/opensearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/opensearch/index/fieldvisitor/FieldsVisitor.java index 91ca07d753cc6..92328745f020e 100644 --- a/server/src/main/java/org/opensearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/opensearch/index/fieldvisitor/FieldsVisitor.java @@ -34,6 +34,7 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.util.BytesRef; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.index.mapper.IdFieldMapper; @@ -66,17 +67,29 @@ public class FieldsVisitor extends StoredFieldVisitor { private final boolean loadSource; private final String sourceFieldName; private final Set requiredFields; + private final String[] sourceIncludes; + private final String[] sourceExcludes; protected BytesReference source; protected String id; protected Map> fieldsValues; public FieldsVisitor(boolean loadSource) { - this(loadSource, SourceFieldMapper.NAME); + this(loadSource, SourceFieldMapper.NAME, null, null); + } + + public FieldsVisitor(boolean loadSource, String[] includes, String[] excludes) { + this(loadSource, SourceFieldMapper.NAME, includes, excludes); } public FieldsVisitor(boolean loadSource, String sourceFieldName) { + this(loadSource, sourceFieldName, null, null); + } + + public FieldsVisitor(boolean loadSource, String sourceFieldName, String[] includes, String[] excludes) { this.loadSource = loadSource; this.sourceFieldName = sourceFieldName; + this.sourceIncludes = includes != null ? includes : Strings.EMPTY_ARRAY; + this.sourceExcludes = excludes != null ? excludes : Strings.EMPTY_ARRAY; requiredFields = new HashSet<>(); reset(); } @@ -162,6 +175,22 @@ public BytesReference source() { return source; } + /** + * Returns the array containing the source fields to include + * @return String[] sourceIncludes + */ + public String[] includes() { + return sourceIncludes; + } + + /** + * Returns the array containing the source fields to exclude + * @return String[] sourceExcludes + */ + public String[] excludes() { + return sourceExcludes; + } + public String id() { return id; } diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index b921afb3157a8..7b1b2615f996d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -93,7 +93,12 @@ public static class Defaults { @Override public MappedFieldType keyedFieldType(String key) { - return new FlatObjectFieldType(this.name() + DOT_SYMBOL + key, this.name(), valueFieldType, valueAndPathFieldType); + return new FlatObjectFieldType( + Strings.isNullOrEmpty(key) ? this.name() : (this.name() + DOT_SYMBOL + key), + this.name(), + valueFieldType, + valueAndPathFieldType + ); } /** @@ -177,7 +182,7 @@ public FlatObjectFieldType( new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap() ); - assert rootFieldName == null || (name.length() > rootFieldName.length() && name.startsWith(rootFieldName)); + assert rootFieldName == null || (name.length() >= rootFieldName.length() && name.startsWith(rootFieldName)); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; this.rootFieldName = rootFieldName; @@ -473,8 +478,7 @@ public Query existsQuery(QueryShardContext context) { String searchKey; String searchField; if (isSubField()) { - searchKey = this.rootFieldName; - searchField = name(); + return rangeQuery(null, null, true, true, context); } else { if (hasDocValues()) { return new FieldExistsQuery(name()); diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 702e5db50e841..3b704513eaa22 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -1309,7 +1309,7 @@ public static boolean hasDecimalPart(Object number) { /** * Returns -1, 0, or 1 if the value is lower than, equal to, or greater than 0 */ - static double signum(Object value) { + public static double signum(Object value) { if (value instanceof Number) { double doubleValue = ((Number) value).doubleValue(); return Math.signum(doubleValue); diff --git a/server/src/main/java/org/opensearch/index/query/BaseQueryRewriteContext.java b/server/src/main/java/org/opensearch/index/query/BaseQueryRewriteContext.java new file mode 100644 index 0000000000000..7cfaf9edb4709 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/query/BaseQueryRewriteContext.java @@ -0,0 +1,140 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.query; + +import org.opensearch.client.Client; +import org.opensearch.common.util.concurrent.CountDown; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.LongSupplier; + +/** + * BaseQueryRewriteContext is a base implementation of the QueryRewriteContext interface. + * It provides core functionality for query rewriting operations in OpenSearch. + * + * This class manages the context for query rewriting, including handling of asynchronous actions, + * access to content registries, and time-related operations. + */ +public class BaseQueryRewriteContext implements QueryRewriteContext { + private final NamedXContentRegistry xContentRegistry; + private final NamedWriteableRegistry writeableRegistry; + protected final Client client; + protected final LongSupplier nowInMillis; + private final List>> asyncActions = new ArrayList<>(); + private final boolean validate; + + public BaseQueryRewriteContext( + NamedXContentRegistry xContentRegistry, + NamedWriteableRegistry writeableRegistry, + Client client, + LongSupplier nowInMillis + ) { + this(xContentRegistry, writeableRegistry, client, nowInMillis, false); + } + + public BaseQueryRewriteContext( + NamedXContentRegistry xContentRegistry, + NamedWriteableRegistry writeableRegistry, + Client client, + LongSupplier nowInMillis, + boolean validate + ) { + + this.xContentRegistry = xContentRegistry; + this.writeableRegistry = writeableRegistry; + this.client = client; + this.nowInMillis = nowInMillis; + this.validate = validate; + } + + /** + * The registry used to build new {@link XContentParser}s. Contains registered named parsers needed to parse the query. + */ + public NamedXContentRegistry getXContentRegistry() { + return xContentRegistry; + } + + /** + * Returns the time in milliseconds that is shared across all resources involved. Even across shards and nodes. + */ + public long nowInMillis() { + return nowInMillis.getAsLong(); + } + + public NamedWriteableRegistry getWriteableRegistry() { + return writeableRegistry; + } + + /** + * Returns an instance of {@link QueryShardContext} if available of null otherwise + */ + public QueryShardContext convertToShardContext() { + return null; + } + + /** + * Registers an async action that must be executed before the next rewrite round in order to make progress. + * This should be used if a rewriteabel needs to fetch some external resources in order to be executed ie. a document + * from an index. + */ + public void registerAsyncAction(BiConsumer> asyncAction) { + asyncActions.add(asyncAction); + } + + /** + * Returns true if there are any registered async actions. + */ + public boolean hasAsyncActions() { + return asyncActions.isEmpty() == false; + } + + /** + * Executes all registered async actions and notifies the listener once it's done. The value that is passed to the listener is always + * null. The list of registered actions is cleared once this method returns. + */ + public void executeAsyncActions(ActionListener listener) { + if (asyncActions.isEmpty()) { + listener.onResponse(null); + return; + } + + CountDown countDown = new CountDown(asyncActions.size()); + ActionListener internalListener = new ActionListener() { + @Override + public void onResponse(Object o) { + if (countDown.countDown()) { + listener.onResponse(null); + } + } + + @Override + public void onFailure(Exception e) { + if (countDown.fastForward()) { + listener.onFailure(e); + } + } + }; + // make a copy to prevent concurrent modification exception + List>> biConsumers = new ArrayList<>(asyncActions); + asyncActions.clear(); + for (BiConsumer> action : biConsumers) { + action.accept(client, internalListener); + } + } + + public boolean validate() { + return validate; + } +} diff --git a/server/src/main/java/org/opensearch/index/query/QueryBuilders.java b/server/src/main/java/org/opensearch/index/query/QueryBuilders.java index 387d21830aa38..1debba73136b2 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryBuilders.java +++ b/server/src/main/java/org/opensearch/index/query/QueryBuilders.java @@ -50,6 +50,7 @@ import java.io.IOException; import java.util.Collection; import java.util.List; +import java.util.Map; /** * Utility class to create search queries. @@ -780,4 +781,13 @@ public static GeoShapeQueryBuilder geoDisjointQuery(String name, String indexedS public static ExistsQueryBuilder existsQuery(String name) { return new ExistsQueryBuilder(name); } + + /** + * A query that contains a template with holder that should be resolved by search processors + * + * @param content The content of the template + */ + public static TemplateQueryBuilder templateQuery(Map content) { + return new TemplateQueryBuilder(content); + } } diff --git a/server/src/main/java/org/opensearch/index/query/QueryCoordinatorContext.java b/server/src/main/java/org/opensearch/index/query/QueryCoordinatorContext.java new file mode 100644 index 0000000000000..c99a952ee42e3 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/query/QueryCoordinatorContext.java @@ -0,0 +1,93 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.query; + +import org.opensearch.client.Client; +import org.opensearch.common.annotation.PublicApi; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.search.pipeline.PipelinedRequest; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiConsumer; + +/** + * The QueryCoordinatorContext class implements the QueryRewriteContext interface and provides + * additional functionality for coordinating query rewriting in OpenSearch. + * + * This class acts as a wrapper around a QueryRewriteContext instance and a PipelinedRequest, + * allowing access to both rewrite context methods and pass over search request information. + * + * @since 2.19.0 + */ +@PublicApi(since = "2.19.0") +public class QueryCoordinatorContext implements QueryRewriteContext { + private final QueryRewriteContext rewriteContext; + private final PipelinedRequest searchRequest; + + public QueryCoordinatorContext(QueryRewriteContext rewriteContext, PipelinedRequest searchRequest) { + this.rewriteContext = rewriteContext; + this.searchRequest = searchRequest; + } + + @Override + public NamedXContentRegistry getXContentRegistry() { + return rewriteContext.getXContentRegistry(); + } + + @Override + public long nowInMillis() { + return rewriteContext.nowInMillis(); + } + + @Override + public NamedWriteableRegistry getWriteableRegistry() { + return rewriteContext.getWriteableRegistry(); + } + + @Override + public QueryShardContext convertToShardContext() { + return rewriteContext.convertToShardContext(); + } + + @Override + public void registerAsyncAction(BiConsumer> asyncAction) { + rewriteContext.registerAsyncAction(asyncAction); + } + + @Override + public boolean hasAsyncActions() { + return rewriteContext.hasAsyncActions(); + } + + @Override + public void executeAsyncActions(ActionListener listener) { + rewriteContext.executeAsyncActions(listener); + } + + @Override + public boolean validate() { + return rewriteContext.validate(); + } + + @Override + public QueryCoordinatorContext convertToCoordinatorContext() { + return this; + } + + public Map getContextVariables() { + + // Read from pipeline context + Map contextVariables = new HashMap<>(searchRequest.getPipelineProcessingContext().getAttributes()); + + return contextVariables; + } +} diff --git a/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java index 15a6d0b5a774e..aec5914066ab5 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/opensearch/index/query/QueryRewriteContext.java @@ -33,16 +33,12 @@ import org.opensearch.client.Client; import org.opensearch.common.annotation.PublicApi; -import org.opensearch.common.util.concurrent.CountDown; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import java.util.ArrayList; -import java.util.List; import java.util.function.BiConsumer; -import java.util.function.LongSupplier; /** * Context object used to rewrite {@link QueryBuilder} instances into simplified version. @@ -50,60 +46,27 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class QueryRewriteContext { - private final NamedXContentRegistry xContentRegistry; - private final NamedWriteableRegistry writeableRegistry; - protected final Client client; - protected final LongSupplier nowInMillis; - private final List>> asyncActions = new ArrayList<>(); - private final boolean validate; - - public QueryRewriteContext( - NamedXContentRegistry xContentRegistry, - NamedWriteableRegistry writeableRegistry, - Client client, - LongSupplier nowInMillis - ) { - this(xContentRegistry, writeableRegistry, client, nowInMillis, false); - } - - public QueryRewriteContext( - NamedXContentRegistry xContentRegistry, - NamedWriteableRegistry writeableRegistry, - Client client, - LongSupplier nowInMillis, - boolean validate - ) { - - this.xContentRegistry = xContentRegistry; - this.writeableRegistry = writeableRegistry; - this.client = client; - this.nowInMillis = nowInMillis; - this.validate = validate; - } - +public interface QueryRewriteContext { /** * The registry used to build new {@link XContentParser}s. Contains registered named parsers needed to parse the query. */ - public NamedXContentRegistry getXContentRegistry() { - return xContentRegistry; - } + NamedXContentRegistry getXContentRegistry(); /** * Returns the time in milliseconds that is shared across all resources involved. Even across shards and nodes. */ - public long nowInMillis() { - return nowInMillis.getAsLong(); - } + long nowInMillis(); - public NamedWriteableRegistry getWriteableRegistry() { - return writeableRegistry; - } + NamedWriteableRegistry getWriteableRegistry(); /** * Returns an instance of {@link QueryShardContext} if available of null otherwise */ - public QueryShardContext convertToShardContext() { + default QueryShardContext convertToShardContext() { + return null; + } + + default QueryCoordinatorContext convertToCoordinatorContext() { return null; } @@ -112,51 +75,18 @@ public QueryShardContext convertToShardContext() { * This should be used if a rewriteabel needs to fetch some external resources in order to be executed ie. a document * from an index. */ - public void registerAsyncAction(BiConsumer> asyncAction) { - asyncActions.add(asyncAction); - } + void registerAsyncAction(BiConsumer> asyncAction); /** * Returns true if there are any registered async actions. */ - public boolean hasAsyncActions() { - return asyncActions.isEmpty() == false; - } + boolean hasAsyncActions(); /** * Executes all registered async actions and notifies the listener once it's done. The value that is passed to the listener is always * null. The list of registered actions is cleared once this method returns. */ - public void executeAsyncActions(ActionListener listener) { - if (asyncActions.isEmpty()) { - listener.onResponse(null); - } else { - CountDown countDown = new CountDown(asyncActions.size()); - ActionListener internalListener = new ActionListener() { - @Override - public void onResponse(Object o) { - if (countDown.countDown()) { - listener.onResponse(null); - } - } - - @Override - public void onFailure(Exception e) { - if (countDown.fastForward()) { - listener.onFailure(e); - } - } - }; - // make a copy to prevent concurrent modification exception - List>> biConsumers = new ArrayList<>(asyncActions); - asyncActions.clear(); - for (BiConsumer> action : biConsumers) { - action.accept(client, internalListener); - } - } - } + void executeAsyncActions(ActionListener listener); - public boolean validate() { - return validate; - } + boolean validate(); } diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java index d717f10b17d9c..62c8c0e25596c 100644 --- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java @@ -76,6 +76,7 @@ import org.opensearch.search.aggregations.support.AggregationUsageService; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.search.lookup.SearchLookup; +import org.opensearch.search.startree.StarTreeQueryContext; import org.opensearch.transport.RemoteClusterAware; import java.io.IOException; @@ -100,7 +101,7 @@ * @opensearch.api */ @PublicApi(since = "1.0.0") -public class QueryShardContext extends QueryRewriteContext { +public class QueryShardContext extends BaseQueryRewriteContext { private final ScriptService scriptService; private final IndexSettings indexSettings; @@ -128,6 +129,8 @@ public class QueryShardContext extends QueryRewriteContext { private boolean keywordIndexOrDocValuesEnabled; private boolean isInnerHitQuery; + private StarTreeQueryContext starTreeQueryContext; + public QueryShardContext( int shardId, IndexSettings indexSettings, @@ -379,6 +382,14 @@ public > IFD getForField(MappedFieldType fieldType ); } + public StarTreeQueryContext getStarTreeQueryContext() { + return starTreeQueryContext; + } + + public void setStarTreeQueryContext(StarTreeQueryContext starTreeQueryContext) { + this.starTreeQueryContext = starTreeQueryContext; + } + public void addNamedQuery(String name, Query query) { if (query != null) { namedQueries.put(name, query); diff --git a/server/src/main/java/org/opensearch/index/query/TemplateQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/TemplateQueryBuilder.java new file mode 100644 index 0000000000000..85d119ab704ec --- /dev/null +++ b/server/src/main/java/org/opensearch/index/query/TemplateQueryBuilder.java @@ -0,0 +1,198 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.query; + +import org.apache.lucene.search.Query; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * A query builder that constructs a query based on a template and context variables. + * This query is designed to be rewritten with variables from search processors. + */ + +public class TemplateQueryBuilder extends AbstractQueryBuilder { + public static final String NAME = "template"; + public static final String queryName = "template"; + private final Map content; + + /** + * Constructs a new TemplateQueryBuilder with the given content. + * + * @param content The template content as a map. + */ + public TemplateQueryBuilder(Map content) { + this.content = content; + } + + /** + * Creates a TemplateQueryBuilder from XContent. + * + * @param parser The XContentParser to read from. + * @return A new TemplateQueryBuilder instance. + * @throws IOException If there's an error parsing the content. + */ + public static TemplateQueryBuilder fromXContent(XContentParser parser) throws IOException { + return new TemplateQueryBuilder(parser.map()); + } + + /** + * Constructs a TemplateQueryBuilder from a stream input. + * + * @param in The StreamInput to read from. + * @throws IOException If there's an error reading from the stream. + */ + public TemplateQueryBuilder(StreamInput in) throws IOException { + super(in); + this.content = in.readMap(); + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeMap(content); + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(NAME, content); + } + + @Override + protected Query doToQuery(QueryShardContext context) throws IOException { + throw new IllegalStateException( + "Template queries cannot be converted directly to a query. Template Query must be rewritten first during doRewrite." + ); + } + + @Override + protected boolean doEquals(TemplateQueryBuilder other) { + return Objects.equals(this.content, other.content); + } + + @Override + protected int doHashCode() { + return Objects.hash(content); + } + + @Override + public String getWriteableName() { + return NAME; + } + + /** + * Gets the content of this template query. + * + * @return The template content as a map. + */ + public Map getContent() { + return content; + } + + /** + * Rewrites the template query by substituting variables from the context. + * + * @param queryCoordinatorContext The context for query rewriting. + * @return A rewritten QueryBuilder. + * @throws IOException If there's an error during rewriting. + */ + @Override + protected QueryBuilder doRewrite(QueryRewriteContext queryCoordinatorContext) throws IOException { + // the queryRewrite is expected at QueryCoordinator level + if (!(queryCoordinatorContext instanceof QueryCoordinatorContext)) { + throw new IllegalStateException( + "Template Query must be rewritten at the coordinator node. Rewriting at shard level is not supported." + ); + } + + QueryCoordinatorContext convertedQueryCoordinateContext = (QueryCoordinatorContext) queryCoordinatorContext; + Map contextVariables = convertedQueryCoordinateContext.getContextVariables(); + String queryString; + + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + builder.map(this.content); + queryString = builder.toString(); + } + + // Convert Map to Map with proper JSON escaping + Map variablesMap = null; + if (contextVariables != null) { + variablesMap = contextVariables.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> { + try { + return JsonXContent.contentBuilder().value(entry.getValue()).toString(); + } catch (IOException e) { + throw new RuntimeException("Error converting contextVariables to JSON string", e); + } + })); + } + String newQueryContent = replaceVariables(queryString, variablesMap); + + try { + XContentParser parser = XContentType.JSON.xContent() + .createParser(queryCoordinatorContext.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, newQueryContent); + + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser); + + QueryBuilder newQueryBuilder = parseInnerQueryBuilder(parser); + + return newQueryBuilder; + + } catch (Exception e) { + throw new IllegalArgumentException("Failed to rewrite template query: " + newQueryContent, e); + } + } + + private String replaceVariables(String template, Map variables) { + if (template == null || template.equals("null")) { + throw new IllegalArgumentException("Template string cannot be null. A valid template must be provided."); + } + if (template.isEmpty() || template.equals("{}")) { + throw new IllegalArgumentException("Template string cannot be empty. A valid template must be provided."); + } + if (variables == null || variables.isEmpty()) { + return template; + } + + StringBuilder result = new StringBuilder(); + int start = 0; + while (true) { + int startVar = template.indexOf("\"${", start); + if (startVar == -1) { + result.append(template.substring(start)); + break; + } + result.append(template, start, startVar); + int endVar = template.indexOf("}\"", startVar); + if (endVar == -1) { + throw new IllegalArgumentException("Unclosed variable in template: " + template.substring(startVar)); + } + String varName = template.substring(startVar + 3, endVar); + String replacement = variables.get(varName); + if (replacement == null) { + throw new IllegalArgumentException("Variable not found: " + varName); + } + result.append(replacement); + start = endVar + 2; + } + return result.toString(); + } + +} diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index b9bad5527e3f4..67fab720d95dd 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -122,6 +122,7 @@ import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.merge.MergeStats; +import org.opensearch.index.query.BaseQueryRewriteContext; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.recovery.RecoveryStats; @@ -1937,7 +1938,7 @@ public QueryRewriteContext getValidationRewriteContext(LongSupplier nowInMillis) * Returns a new {@link QueryRewriteContext} with the given {@code now} provider */ private QueryRewriteContext getRewriteContext(LongSupplier nowInMillis, boolean validate) { - return new QueryRewriteContext(xContentRegistry, namedWriteableRegistry, client, nowInMillis, validate); + return new BaseQueryRewriteContext(xContentRegistry, namedWriteableRegistry, client, nowInMillis, validate); } /** diff --git a/server/src/main/java/org/opensearch/ingest/IngestService.java b/server/src/main/java/org/opensearch/ingest/IngestService.java index 0315a960dae92..4d65509345614 100644 --- a/server/src/main/java/org/opensearch/ingest/IngestService.java +++ b/server/src/main/java/org/opensearch/ingest/IngestService.java @@ -44,7 +44,7 @@ import org.opensearch.action.index.IndexRequest; import org.opensearch.action.ingest.DeletePipelineRequest; import org.opensearch.action.ingest.PutPipelineRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.AckedClusterStateUpdateTask; import org.opensearch.cluster.ClusterChangedEvent; diff --git a/server/src/main/java/org/opensearch/plugins/PluginInfo.java b/server/src/main/java/org/opensearch/plugins/PluginInfo.java index 4ff699e8017ba..323e061aea567 100644 --- a/server/src/main/java/org/opensearch/plugins/PluginInfo.java +++ b/server/src/main/java/org/opensearch/plugins/PluginInfo.java @@ -36,8 +36,8 @@ import com.fasterxml.jackson.core.json.JsonReadFeature; import org.opensearch.Version; -import org.opensearch.bootstrap.JarHell; import org.opensearch.common.annotation.PublicApi; +import org.opensearch.common.bootstrap.JarHell; import org.opensearch.common.xcontent.json.JsonXContentParser; import org.opensearch.core.common.Strings; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/plugins/PluginsService.java b/server/src/main/java/org/opensearch/plugins/PluginsService.java index 9bc1f1334122e..72b8ada94a0d1 100644 --- a/server/src/main/java/org/opensearch/plugins/PluginsService.java +++ b/server/src/main/java/org/opensearch/plugins/PluginsService.java @@ -43,7 +43,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.admin.cluster.node.info.PluginsAndModules; -import org.opensearch.bootstrap.JarHell; +import org.opensearch.common.bootstrap.JarHell; import org.opensearch.common.collect.Tuple; import org.opensearch.common.inject.Module; import org.opensearch.common.lifecycle.LifecycleComponent; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestDeleteDanglingIndexAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestDeleteDanglingIndexAction.java index 21003f565be44..ed4eec9dab12c 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestDeleteDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestDeleteDanglingIndexAction.java @@ -33,7 +33,7 @@ package org.opensearch.rest.action.admin.cluster.dangling; import org.opensearch.action.admin.indices.dangling.delete.DeleteDanglingIndexRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.core.rest.RestStatus; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestImportDanglingIndexAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestImportDanglingIndexAction.java index ea1e25717b9c0..ea6125f5a80fe 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestImportDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/dangling/RestImportDanglingIndexAction.java @@ -33,7 +33,7 @@ package org.opensearch.rest.action.admin.cluster.dangling; import org.opensearch.action.admin.indices.dangling.import_index.ImportDanglingIndexRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.common.logging.DeprecationLogger; import org.opensearch.core.rest.RestStatus; diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestOpenIndexAction.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestOpenIndexAction.java index e246efaa9af01..8ccd5f8e9450b 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestOpenIndexAction.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestOpenIndexAction.java @@ -50,7 +50,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; -import static org.opensearch.action.support.master.AcknowledgedRequest.DEFAULT_TASK_EXECUTION_TIMEOUT; +import static org.opensearch.action.support.clustermanager.AcknowledgedRequest.DEFAULT_TASK_EXECUTION_TIMEOUT; import static org.opensearch.rest.RestRequest.Method.POST; /** diff --git a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestResizeHandler.java b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestResizeHandler.java index 66f1e98228ba3..412693d0b99c1 100644 --- a/server/src/main/java/org/opensearch/rest/action/admin/indices/RestResizeHandler.java +++ b/server/src/main/java/org/opensearch/rest/action/admin/indices/RestResizeHandler.java @@ -53,7 +53,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; -import static org.opensearch.action.support.master.AcknowledgedRequest.DEFAULT_TASK_EXECUTION_TIMEOUT; +import static org.opensearch.action.support.clustermanager.AcknowledgedRequest.DEFAULT_TASK_EXECUTION_TIMEOUT; import static org.opensearch.rest.RestRequest.Method.POST; import static org.opensearch.rest.RestRequest.Method.PUT; diff --git a/server/src/main/java/org/opensearch/script/ScriptService.java b/server/src/main/java/org/opensearch/script/ScriptService.java index d3c8861dbc5d7..5ff779948b2da 100644 --- a/server/src/main/java/org/opensearch/script/ScriptService.java +++ b/server/src/main/java/org/opensearch/script/ScriptService.java @@ -38,7 +38,7 @@ import org.opensearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.opensearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.opensearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.AckedClusterStateUpdateTask; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterState; diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index 40e0293f88f07..24bdb2d28cba2 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -86,6 +86,7 @@ import org.opensearch.index.query.SpanOrQueryBuilder; import org.opensearch.index.query.SpanTermQueryBuilder; import org.opensearch.index.query.SpanWithinQueryBuilder; +import org.opensearch.index.query.TemplateQueryBuilder; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.index.query.TermsQueryBuilder; import org.opensearch.index.query.TermsSetQueryBuilder; @@ -1172,7 +1173,7 @@ private void registerQueryParsers(List plugins) { registerQuery( new QuerySpec<>(MatchBoolPrefixQueryBuilder.NAME, MatchBoolPrefixQueryBuilder::new, MatchBoolPrefixQueryBuilder::fromXContent) ); - + registerQuery(new QuerySpec<>(TemplateQueryBuilder.NAME, TemplateQueryBuilder::new, TemplateQueryBuilder::fromXContent)); if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) { registerQuery(new QuerySpec<>(GeoShapeQueryBuilder.NAME, GeoShapeQueryBuilder::new, GeoShapeQueryBuilder::fromXContent)); } diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index e892a2f1a7620..866cf0d62b033 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -77,7 +77,6 @@ import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.IndexService; import org.opensearch.index.IndexSettings; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.engine.Engine; import org.opensearch.index.mapper.DerivedFieldResolver; import org.opensearch.index.mapper.DerivedFieldResolverFactory; @@ -85,6 +84,7 @@ import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.MatchNoneQueryBuilder; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryCoordinatorContext; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.Rewriteable; @@ -126,6 +126,7 @@ import org.opensearch.search.internal.ShardSearchContextId; import org.opensearch.search.internal.ShardSearchRequest; import org.opensearch.search.lookup.SearchLookup; +import org.opensearch.search.pipeline.PipelinedRequest; import org.opensearch.search.profile.Profilers; import org.opensearch.search.query.QueryPhase; import org.opensearch.search.query.QuerySearchRequest; @@ -139,6 +140,7 @@ import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortOrder; import org.opensearch.search.startree.StarTreeQueryContext; +import org.opensearch.search.startree.StarTreeQueryHelper; import org.opensearch.search.suggest.Suggest; import org.opensearch.search.suggest.completion.CompletionSuggestion; import org.opensearch.tasks.TaskResourceTrackingService; @@ -1546,15 +1548,11 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc if (this.indicesService.getCompositeIndexSettings() != null && this.indicesService.getCompositeIndexSettings().isStarTreeIndexCreationEnabled() && StarTreeQueryHelper.isStarTreeSupported(context)) { - try { - StarTreeQueryContext starTreeQueryContext = StarTreeQueryHelper.getStarTreeQueryContext(context, source); - if (starTreeQueryContext != null) { - context.starTreeQueryContext(starTreeQueryContext); - logger.debug("can use star tree"); - } else { - logger.debug("cannot use star tree"); - } - } catch (IOException ignored) {} + StarTreeQueryContext starTreeQueryContext = new StarTreeQueryContext(context, source.query()); + boolean consolidated = starTreeQueryContext.consolidateAllFilters(context); + if (consolidated) { + queryShardContext.setStarTreeQueryContext(starTreeQueryContext); + } } } @@ -1775,8 +1773,8 @@ private void rewriteAndFetchShardRequest(IndexShard shard, ShardSearchRequest re /** * Returns a new {@link QueryRewriteContext} with the given {@code now} provider */ - public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis) { - return indicesService.getRewriteContext(nowInMillis); + public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis, PipelinedRequest searchRequest) { + return new QueryCoordinatorContext(indicesService.getRewriteContext(nowInMillis), searchRequest); } /** diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregator.java index 2970c5ca851e7..29e2384a29ec1 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/AvgAggregator.java @@ -44,7 +44,6 @@ import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.index.fielddata.SortedNumericDoubleValues; @@ -56,12 +55,13 @@ import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeQueryHelper; import java.io.IOException; import java.util.Map; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getStarTreeFilteredValues; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getSupportedStarTree; +import static org.opensearch.search.startree.StarTreeQueryHelper.getStarTreeFilteredValues; +import static org.opensearch.search.startree.StarTreeQueryHelper.getSupportedStarTree; /** * Aggregate all docs into an average @@ -106,7 +106,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc if (valuesSource == null) { return LeafBucketCollector.NO_OP_COLLECTOR; } - CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context); + CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context.getQueryShardContext()); if (supportedStarTree != null) { return getStarTreeLeafCollector(ctx, sub, supportedStarTree); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregator.java index 257109bca54bb..4f5243bf4d62b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MaxAggregator.java @@ -43,7 +43,6 @@ import org.opensearch.common.util.DoubleArray; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.fielddata.NumericDoubleValues; import org.opensearch.index.fielddata.SortedNumericDoubleValues; import org.opensearch.search.DocValueFormat; @@ -55,6 +54,7 @@ import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeQueryHelper; import java.io.IOException; import java.util.Arrays; @@ -62,7 +62,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getSupportedStarTree; +import static org.opensearch.search.startree.StarTreeQueryHelper.getSupportedStarTree; /** * Aggregate all docs into a max value @@ -128,7 +128,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc } } - CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context); + CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context.getQueryShardContext()); if (supportedStarTree != null) { return getStarTreeCollector(ctx, sub, supportedStarTree); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregator.java index a9f20bdeb5fd5..9292475db81f2 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/MinAggregator.java @@ -43,7 +43,6 @@ import org.opensearch.common.util.DoubleArray; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.fielddata.NumericDoubleValues; import org.opensearch.index.fielddata.SortedNumericDoubleValues; import org.opensearch.search.DocValueFormat; @@ -55,13 +54,14 @@ import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeQueryHelper; import java.io.IOException; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getSupportedStarTree; +import static org.opensearch.search.startree.StarTreeQueryHelper.getSupportedStarTree; /** * Aggregate all docs into a min value @@ -127,7 +127,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc } } - CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context); + CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context.getQueryShardContext()); if (supportedStarTree != null) { return getStarTreeCollector(ctx, sub, supportedStarTree); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java index 3d237a94c5699..bd1c3cc11224e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java @@ -39,7 +39,6 @@ import org.opensearch.common.util.DoubleArray; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.fielddata.SortedNumericDoubleValues; import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.Aggregator; @@ -49,11 +48,12 @@ import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeQueryHelper; import java.io.IOException; import java.util.Map; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getSupportedStarTree; +import static org.opensearch.search.startree.StarTreeQueryHelper.getSupportedStarTree; /** * Aggregate all docs into a single sum value @@ -96,7 +96,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc return LeafBucketCollector.NO_OP_COLLECTOR; } - CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context); + CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context.getQueryShardContext()); if (supportedStarTree != null) { return getStarTreeCollector(ctx, sub, supportedStarTree); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java index a156ec49983fa..1fd16190da69b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/ValueCountAggregator.java @@ -39,7 +39,6 @@ import org.opensearch.common.util.LongArray; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.compositeindex.datacube.MetricStat; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.fielddata.MultiGeoPointValues; import org.opensearch.index.fielddata.SortedBinaryDocValues; import org.opensearch.search.aggregations.Aggregator; @@ -49,11 +48,12 @@ import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeQueryHelper; import java.io.IOException; import java.util.Map; -import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper.getSupportedStarTree; +import static org.opensearch.search.startree.StarTreeQueryHelper.getSupportedStarTree; /** * A field data based aggregator that counts the number of values a specific field has within the aggregation context. @@ -94,7 +94,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc if (valuesSource instanceof ValuesSource.Numeric) { - CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context); + CompositeIndexFieldInfo supportedStarTree = getSupportedStarTree(this.context.getQueryShardContext()); if (supportedStarTree != null) { return getStarTreeCollector(ctx, sub, supportedStarTree); } diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java index fb21eaff5f857..90dfc1e086602 100644 --- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java @@ -305,7 +305,7 @@ public SearchSourceBuilder(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_2_18_0)) { searchPipeline = in.readOptionalString(); } - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_19_0)) { verbosePipeline = in.readBoolean(); } } @@ -391,7 +391,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_2_18_0)) { out.writeOptionalString(searchPipeline); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_19_0)) { out.writeBoolean(verbosePipeline); } } diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java index 11a1b9a97235b..df37b7dbfda98 100644 --- a/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/opensearch/search/fetch/FetchPhase.java @@ -221,7 +221,7 @@ public int compareTo(DocIdToIndex o) { } } - private FieldsVisitor createStoredFieldsVisitor(SearchContext context, Map> storedToRequestedFields) { + protected FieldsVisitor createStoredFieldsVisitor(SearchContext context, Map> storedToRequestedFields) { StoredFieldsContext storedFieldsContext = context.storedFieldsContext(); if (storedFieldsContext == null) { @@ -230,7 +230,11 @@ private FieldsVisitor createStoredFieldsVisitor(SearchContext context, Map searchExtBuilderList + + ) { + super( + hits, + aggregations, + suggest, + timedOut, + terminatedEarly, + profileResults, + numReducePhases, + searchExtBuilderList, + Collections.emptyList() + ); + } + public InternalSearchResponse(StreamInput in) throws IOException { super( new SearchHits(in), @@ -149,11 +173,11 @@ private static void writeSearchExtBuildersOnOrAfter(StreamOutput out, List readProcessorResultOnOrAfter(StreamInput in) throws IOException { - return (in.getVersion().onOrAfter(Version.V_3_0_0)) ? in.readList(ProcessorExecutionDetail::new) : Collections.emptyList(); + return (in.getVersion().onOrAfter(Version.V_2_19_0)) ? in.readList(ProcessorExecutionDetail::new) : Collections.emptyList(); } private static void writeProcessorResultOnOrAfter(StreamOutput out, List processorResult) throws IOException { - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_19_0)) { out.writeList(processorResult); } } diff --git a/server/src/main/java/org/opensearch/search/internal/SearchContext.java b/server/src/main/java/org/opensearch/search/internal/SearchContext.java index b7ea06d2989e5..0e61dbc76b838 100644 --- a/server/src/main/java/org/opensearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/opensearch/search/internal/SearchContext.java @@ -76,7 +76,6 @@ import org.opensearch.search.query.ReduceableSearchResult; import org.opensearch.search.rescore.RescoreContext; import org.opensearch.search.sort.SortAndFormats; -import org.opensearch.search.startree.StarTreeQueryContext; import org.opensearch.search.suggest.SuggestionSearchContext; import java.util.Collection; @@ -126,7 +125,6 @@ public List toInternalAggregations(Collection co private final AtomicBoolean closed = new AtomicBoolean(false); private InnerHitsContext innerHitsContext; private volatile boolean searchTimedOut; - private StarTreeQueryContext starTreeQueryContext; protected SearchContext() {} @@ -532,12 +530,4 @@ public boolean keywordIndexOrDocValuesEnabled() { return false; } - public SearchContext starTreeQueryContext(StarTreeQueryContext starTreeQueryContext) { - this.starTreeQueryContext = starTreeQueryContext; - return this; - } - - public StarTreeQueryContext getStarTreeQueryContext() { - return this.starTreeQueryContext; - } } diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelineProcessingContext.java b/server/src/main/java/org/opensearch/search/pipeline/PipelineProcessingContext.java index 7e86c30ddbbd9..c7fad1363cf2f 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/PipelineProcessingContext.java +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelineProcessingContext.java @@ -57,4 +57,8 @@ public void addProcessorExecutionDetail(ProcessorExecutionDetail detail) { public List getProcessorExecutionDetails() { return Collections.unmodifiableList(processorExecutionDetails); } + + public Map getAttributes() { + return attributes; + } } diff --git a/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java b/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java index f5ce94946dd32..b35784aef5582 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java +++ b/server/src/main/java/org/opensearch/search/pipeline/PipelinedRequest.java @@ -61,4 +61,8 @@ public void transformSearchPhaseResults( Pipeline getPipeline() { return pipeline; } + + public PipelineProcessingContext getPipelineProcessingContext() { + return requestContext; + } } diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java index 27b837740c0ca..7c71771953da9 100644 --- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java +++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java @@ -16,7 +16,7 @@ import org.opensearch.action.search.DeleteSearchPipelineRequest; import org.opensearch.action.search.PutSearchPipelineRequest; import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.AckedClusterStateUpdateTask; import org.opensearch.cluster.ClusterChangedEvent; diff --git a/server/src/main/java/org/opensearch/search/startree/StarTreeNodeCollector.java b/server/src/main/java/org/opensearch/search/startree/StarTreeNodeCollector.java new file mode 100644 index 0000000000000..9d3dcd6824874 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/StarTreeNodeCollector.java @@ -0,0 +1,25 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; + +/** + * Collects one or more @{@link StarTreeNode}'s + */ +@ExperimentalApi +public interface StarTreeNodeCollector { + /** + * Called to collect a @{@link StarTreeNode} + * @param node : Node to collect + */ + void collectStarTreeNode(StarTreeNode node); + +} diff --git a/server/src/main/java/org/opensearch/search/startree/StarTreeQueryContext.java b/server/src/main/java/org/opensearch/search/startree/StarTreeQueryContext.java index cda3a25b30e53..bd52693bcacad 100644 --- a/server/src/main/java/org/opensearch/search/startree/StarTreeQueryContext.java +++ b/server/src/main/java/org/opensearch/search/startree/StarTreeQueryContext.java @@ -12,68 +12,151 @@ import org.apache.lucene.util.FixedBitSet; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.mapper.CompositeDataCubeFieldType; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.search.aggregations.AggregatorFactory; +import org.opensearch.search.aggregations.metrics.MetricAggregatorFactory; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.filter.StarTreeFilter; +import org.opensearch.search.startree.filter.provider.StarTreeFilterProvider; +import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** - * Query class for querying star tree data structure. - * - * @opensearch.experimental + * Stores the star tree related context of a search request. */ @ExperimentalApi public class StarTreeQueryContext { - /** - * Star tree field info - * This is used to get the star tree data structure - */ - private final CompositeIndexFieldInfo starTree; + private final CompositeDataCubeFieldType compositeMappedFieldType; /** - * Map of field name to a value to be queried for that field - * This is used to filter the data based on the query + * Cache for leaf results + * This is used to cache the results for each leaf reader context + * to avoid reading the filtered values from the leaf reader context multiple times */ - private final Map queryMap; - - /** - * Cache for leaf results - * This is used to cache the results for each leaf reader context - * to avoid reading the filtered values from the leaf reader context multiple times - */ + // TODO : Change caching to be based on aggregation specific filters. private final FixedBitSet[] starTreeValues; - public StarTreeQueryContext(CompositeIndexFieldInfo starTree, Map queryMap, int numSegmentsCache) { - this.starTree = starTree; - this.queryMap = queryMap; - if (numSegmentsCache > -1) { - starTreeValues = new FixedBitSet[numSegmentsCache]; + private final QueryBuilder baseQueryBuilder; + private StarTreeFilter baseStarTreeFilter; + + // TODO : Implement storing and aggregating aggregation specific filters. + + public StarTreeQueryContext(SearchContext context, QueryBuilder baseQueryBuilder) { + this.baseQueryBuilder = baseQueryBuilder; + // TODO : We need to select the most appropriate one from multiple star tree field types. + compositeMappedFieldType = (CompositeDataCubeFieldType) context.mapperService().getCompositeFieldTypes().iterator().next(); + // need to cache star tree values only for multiple aggregations + boolean cacheStarTreeValues = context.aggregations().factories().getFactories().length > 1; + int cacheSize = cacheStarTreeValues ? context.indexShard().segments(false).size() : -1; + if (cacheSize > -1) { + starTreeValues = new FixedBitSet[cacheSize]; + } else { + starTreeValues = null; + } + } + + public StarTreeQueryContext(CompositeDataCubeFieldType compositeMappedFieldType, QueryBuilder baseQueryBuilder, int cacheSize) { + this.compositeMappedFieldType = compositeMappedFieldType; + this.baseQueryBuilder = baseQueryBuilder; + if (cacheSize > -1) { + starTreeValues = new FixedBitSet[cacheSize]; } else { starTreeValues = null; } } public CompositeIndexFieldInfo getStarTree() { - return starTree; + return new CompositeIndexFieldInfo(compositeMappedFieldType.name(), compositeMappedFieldType.getCompositeIndexType()); } - public Map getQueryMap() { - return queryMap; + public FixedBitSet getStarTreeValue(LeafReaderContext ctx) { + return starTreeValues != null ? starTreeValues[ctx.ord] : null; } public FixedBitSet[] getStarTreeValues() { return starTreeValues; } - public FixedBitSet getStarTreeValues(LeafReaderContext ctx) { + public void setStarTreeValues(LeafReaderContext ctx, FixedBitSet values) { if (starTreeValues != null) { - return starTreeValues[ctx.ord]; + starTreeValues[ctx.ord] = values; + } + } + + /** + * Generates the Base StarTreeFilter and then recursively merges + * any aggregation specific STF. + * @return true if recursively all filters were consolidated, else false. + */ + public boolean consolidateAllFilters(SearchContext context) { + // Validate the fields and metrics required by aggregations are supported in star tree + for (AggregatorFactory aggregatorFactory : context.aggregations().factories().getFactories()) { + MetricStat metricStat = validateStarTreeMetricSupport(compositeMappedFieldType, aggregatorFactory); + if (metricStat == null) { + return false; + } + } + + // Generate the base Star Tree Filter + if (baseQueryBuilder != null) { + baseStarTreeFilter = getStarTreeFilter(context, baseQueryBuilder, compositeMappedFieldType); + return baseStarTreeFilter != null; // Base Query is not supported by star tree filter. + } + // TODO : Generate StarTreeFilter specific to aggregations by merging base and their parents. + return true; + } + + public StarTreeFilter getBaseQueryStarTreeFilter() { + if (baseStarTreeFilter == null) { + return new StarTreeFilter(Collections.emptyMap()); + } + return baseStarTreeFilter; + } + + // TODO : Push this validation down to a common method in AggregatorFactory or an equivalent place. + private MetricStat validateStarTreeMetricSupport( + CompositeDataCubeFieldType compositeIndexFieldInfo, + AggregatorFactory aggregatorFactory + ) { + if (aggregatorFactory instanceof MetricAggregatorFactory && aggregatorFactory.getSubFactories().getFactories().length == 0) { + String field; + Map> supportedMetrics = compositeIndexFieldInfo.getMetrics() + .stream() + .collect(Collectors.toMap(Metric::getField, Metric::getMetrics)); + + MetricStat metricStat = ((MetricAggregatorFactory) aggregatorFactory).getMetricStat(); + field = ((MetricAggregatorFactory) aggregatorFactory).getField(); + + if (field != null && supportedMetrics.containsKey(field) && supportedMetrics.get(field).contains(metricStat)) { + return metricStat; + } } return null; } - public void setStarTreeValues(LeafReaderContext ctx, FixedBitSet values) { - if (starTreeValues != null) { - starTreeValues[ctx.ord] = values; + private StarTreeFilter getStarTreeFilter( + SearchContext context, + QueryBuilder queryBuilder, + CompositeDataCubeFieldType compositeMappedFieldType + ) { + StarTreeFilterProvider starTreeFilterProvider = StarTreeFilterProvider.SingletonFactory.getProvider(queryBuilder); + // The query builder's support is not implemented. + if (starTreeFilterProvider == null) { + return null; + } + try { + return starTreeFilterProvider.getFilter(context, queryBuilder, compositeMappedFieldType); + } catch (IOException e) { + throw new RuntimeException(e); } } + } diff --git a/server/src/main/java/org/opensearch/search/startree/StarTreeQueryHelper.java b/server/src/main/java/org/opensearch/search/startree/StarTreeQueryHelper.java new file mode 100644 index 0000000000000..e42b3ac980287 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/StarTreeQueryHelper.java @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.search.CollectionTerminatedException; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.FixedBitSet; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; +import org.opensearch.index.codec.composite.CompositeIndexReader; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.search.aggregations.LeafBucketCollector; +import org.opensearch.search.aggregations.LeafBucketCollectorBase; +import org.opensearch.search.aggregations.support.ValuesSource; +import org.opensearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.List; +import java.util.function.Consumer; + +/** + * Helper class for building star-tree query + * + * @opensearch.internal + * @opensearch.experimental + */ +public class StarTreeQueryHelper { + + private static StarTreeValues starTreeValues; + + /** + * Checks if the search context can be supported by star-tree + */ + public static boolean isStarTreeSupported(SearchContext context) { + return context.aggregations() != null && context.mapperService().isCompositeIndexPresent() && context.parsedPostFilter() == null; + } + + public static CompositeIndexFieldInfo getSupportedStarTree(QueryShardContext context) { + StarTreeQueryContext starTreeQueryContext = context.getStarTreeQueryContext(); + return (starTreeQueryContext != null) ? starTreeQueryContext.getStarTree() : null; + } + + public static StarTreeValues getStarTreeValues(LeafReaderContext context, CompositeIndexFieldInfo starTree) throws IOException { + SegmentReader reader = Lucene.segmentReader(context.reader()); + if (!(reader.getDocValuesReader() instanceof CompositeIndexReader)) { + return null; + } + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + return (StarTreeValues) starTreeDocValuesReader.getCompositeIndexValues(starTree); + } + + /** + * Get the star-tree leaf collector + * This collector computes the aggregation prematurely and invokes an early termination collector + */ + public static LeafBucketCollector getStarTreeLeafCollector( + SearchContext context, + ValuesSource.Numeric valuesSource, + LeafReaderContext ctx, + LeafBucketCollector sub, + CompositeIndexFieldInfo starTree, + String metric, + Consumer valueConsumer, + Runnable finalConsumer + ) throws IOException { + StarTreeValues starTreeValues = getStarTreeValues(ctx, starTree); + assert starTreeValues != null; + String fieldName = ((ValuesSource.Numeric.FieldData) valuesSource).getIndexFieldName(); + String metricName = StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues(starTree.getField(), fieldName, metric); + + assert starTreeValues != null; + SortedNumericStarTreeValuesIterator valuesIterator = (SortedNumericStarTreeValuesIterator) starTreeValues.getMetricValuesIterator( + metricName + ); + // Obtain a FixedBitSet of matched star tree document IDs + FixedBitSet filteredValues = getStarTreeFilteredValues(context, ctx, starTreeValues); + assert filteredValues != null; + + int numBits = filteredValues.length(); // Get the number of the filtered values (matching docs) + if (numBits > 0) { + // Iterate over the filtered values + for (int bit = filteredValues.nextSetBit(0); bit != DocIdSetIterator.NO_MORE_DOCS; bit = (bit + 1 < numBits) + ? filteredValues.nextSetBit(bit + 1) + : DocIdSetIterator.NO_MORE_DOCS) { + // Advance to the entryId in the valuesIterator + if (valuesIterator.advanceExact(bit) == false) { + continue; // Skip if no more entries + } + + // Iterate over the values for the current entryId + for (int i = 0, count = valuesIterator.entryValueCount(); i < count; i++) { + long value = valuesIterator.nextValue(); + valueConsumer.accept(value); // Apply the consumer operation (e.g., max, sum) + } + } + } + + // Call the final consumer after processing all entries + finalConsumer.run(); + + // Return a LeafBucketCollector that terminates collection + return new LeafBucketCollectorBase(sub, valuesSource.doubleValues(ctx)) { + @Override + public void collect(int doc, long bucket) { + throw new CollectionTerminatedException(); + } + }; + } + + /** + * Get the filtered values for the star-tree query + * Cache the results in case of multiple aggregations (if cache is initialized) + * @return FixedBitSet of matched document IDs + */ + public static FixedBitSet getStarTreeFilteredValues(SearchContext context, LeafReaderContext ctx, StarTreeValues starTreeValues) + throws IOException { + FixedBitSet result = context.getQueryShardContext().getStarTreeQueryContext().getStarTreeValue(ctx); + if (result == null) { + result = StarTreeTraversalUtil.getStarTreeResult( + starTreeValues, + context.getQueryShardContext().getStarTreeQueryContext().getBaseQueryStarTreeFilter(), + context + ); + } + context.getQueryShardContext().getStarTreeQueryContext().setStarTreeValues(ctx, result); + return result; + } + + public static Dimension getMatchingDimensionOrError(String dimensionName, List orderedDimensions) { + Dimension matchingDimension = getMatchingDimensionOrNull(dimensionName, orderedDimensions); + if (matchingDimension == null) { + throw new IllegalStateException("No matching dimension found for [" + dimensionName + "]"); + } + return matchingDimension; + } + + public static Dimension getMatchingDimensionOrNull(String dimensionName, List orderedDimensions) { + List matchingDimensions = orderedDimensions.stream().filter(x -> x.getField().equals(dimensionName)).toList(); + if (matchingDimensions.size() != 1) { + return null; + } + return matchingDimensions.get(0); + } + +} diff --git a/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java b/server/src/main/java/org/opensearch/search/startree/StarTreeTraversalUtil.java similarity index 78% rename from server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java rename to server/src/main/java/org/opensearch/search/startree/StarTreeTraversalUtil.java index f7fa210691678..eb7ef3fbb489a 100644 --- a/server/src/main/java/org/opensearch/search/startree/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/search/startree/StarTreeTraversalUtil.java @@ -19,6 +19,9 @@ import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.filter.DimensionFilter; +import org.opensearch.search.startree.filter.StarTreeFilter; import java.io.IOException; import java.util.ArrayDeque; @@ -27,7 +30,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Queue; import java.util.Set; import java.util.stream.Collectors; @@ -40,16 +42,20 @@ * @opensearch.experimental * @opensearch.internal */ -public class StarTreeFilter { - private static final Logger logger = LogManager.getLogger(StarTreeFilter.class); +public class StarTreeTraversalUtil { + private static final Logger logger = LogManager.getLogger(StarTreeTraversalUtil.class); - /** - * First go over the star tree and try to match as many dimensions as possible - * For the remaining columns, use star-tree doc values to match them - */ - public static FixedBitSet getStarTreeResult(StarTreeValues starTreeValues, Map predicateEvaluators) throws IOException { - Map queryMap = predicateEvaluators != null ? predicateEvaluators : Collections.emptyMap(); - StarTreeResult starTreeResult = traverseStarTree(starTreeValues, queryMap); + public static FixedBitSet getStarTreeResult(StarTreeValues starTreeValues, StarTreeFilter starTreeFilter, SearchContext searchContext) + throws IOException { + + // Initialising all dimension filters for this segment + for (String dimension : starTreeFilter.getDimensions()) { + for (DimensionFilter dimensionFilter : starTreeFilter.getFiltersForDimension(dimension)) { + dimensionFilter.initialiseForSegment(starTreeValues, searchContext); + } + } + + StarTreeResult starTreeResult = traverseStarTree(starTreeValues, starTreeFilter); // Initialize FixedBitSet with size maxMatchedDoc + 1 FixedBitSet bitSet = new FixedBitSet(starTreeResult.maxMatchedDoc + 1); @@ -74,11 +80,9 @@ public static FixedBitSet getStarTreeResult(StarTreeValues starTreeValues, Map dimensionFilters = starTreeFilter.getFiltersForDimension(remainingPredicateColumn); // Clear the temporary bit set before reuse tempBitSet.clear(0, starTreeResult.maxMatchedDoc + 1); @@ -88,14 +92,12 @@ public static FixedBitSet getStarTreeResult(StarTreeValues starTreeValues, Map queryMap) throws IOException { + private static StarTreeResult traverseStarTree(StarTreeValues starTreeValues, StarTreeFilter starTreeFilter) throws IOException { DocIdSetBuilder docsWithField = new DocIdSetBuilder(starTreeValues.getStarTreeDocumentCount()); DocIdSetBuilder.BulkAdder adder; Set globalRemainingPredicateColumns = null; @@ -128,7 +126,7 @@ private static StarTreeResult traverseStarTree(StarTreeValues starTreeValues, Ma Queue queue = new ArrayDeque<>(); queue.add(starTree); int currentDimensionId = -1; - Set remainingPredicateColumns = new HashSet<>(queryMap.keySet()); + Set remainingPredicateColumns = new HashSet<>(starTreeFilter.getDimensions()); int matchedDocsCountInStarTree = 0; int maxDocNum = -1; StarTreeNode starTreeNode; @@ -169,12 +167,15 @@ private static StarTreeResult traverseStarTree(StarTreeValues starTreeValues, Ma } if (remainingPredicateColumns.contains(childDimension)) { - long queryValue = queryMap.get(childDimension); // Get the query value directly from the map - StarTreeNode matchingChild = starTreeNode.getChildForDimensionValue(queryValue); - if (matchingChild != null) { - queue.add(matchingChild); - foundLeafNode |= matchingChild.isLeaf(); + List dimensionFilters = starTreeFilter.getFiltersForDimension(childDimension); + final boolean[] tempFoundLeafNodes = new boolean[1]; + for (DimensionFilter dimensionFilter : dimensionFilters) { + dimensionFilter.matchStarTreeNodes(starTreeNode, starTreeValues, node -> { + queue.add(node); + tempFoundLeafNodes[0] |= node.isLeaf(); + }); } + foundLeafNode |= tempFoundLeafNodes[0]; } else { if (starNode != null) { queue.add(starNode); diff --git a/server/src/main/java/org/opensearch/search/startree/filter/DimensionFilter.java b/server/src/main/java/org/opensearch/search/startree/filter/DimensionFilter.java new file mode 100644 index 0000000000000..a5cfc24889e1f --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/DimensionFilter.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree.filter; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeNodeCollector; + +import java.io.IOException; + +/** + * Contains the logic to filter over a dimension either in StarTree Index or it's Dimension DocValues + */ +@ExperimentalApi +public interface DimensionFilter { + + /** + * Converts parsed user values to ordinals based on segment and other init actions can be performed. + * @param starTreeValues : Segment specific star tree root node and other metadata + * @param searchContext : Search context + * @throws IOException : + */ + void initialiseForSegment(StarTreeValues starTreeValues, SearchContext searchContext) throws IOException; + + /** + * Called when matching a dimension values in the star tree index. + * @param parentNode : StarTreeNode below which the dimension to be filtered is present. + * @param starTreeValues : Segment specific star tree root node and other metadata + * @param collector : Collector which collates the matched StarTreeNode's + * @throws IOException : + */ + void matchStarTreeNodes(StarTreeNode parentNode, StarTreeValues starTreeValues, StarTreeNodeCollector collector) throws IOException; + + /** + * Called when a dimension is not found in star tree index and needs to matched by iterating its docValues + * @param ordinal : Value to Match + * @param starTreeValues : Segment specific star tree root node and other metadata + * @return : true if matches, else false. + */ + boolean matchDimValue(long ordinal, StarTreeValues starTreeValues); + + /** + * Represents how to match a value when comparing during StarTreeTraversal + */ + @ExperimentalApi + enum MatchType { + GT, + LT, + GTE, + LTE, + EXACT + } +} diff --git a/server/src/main/java/org/opensearch/search/startree/filter/ExactMatchDimFilter.java b/server/src/main/java/org/opensearch/search/startree/filter/ExactMatchDimFilter.java new file mode 100644 index 0000000000000..f0c07f011e264 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/ExactMatchDimFilter.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree.filter; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeNodeCollector; +import org.opensearch.search.startree.StarTreeQueryHelper; +import org.opensearch.search.startree.filter.provider.DimensionFilterMapper; + +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.TreeSet; + +/** + * Handles Term and Terms query like search in StarTree Dimension filtering. + */ +@ExperimentalApi +public class ExactMatchDimFilter implements DimensionFilter { + + private final String dimensionName; + + private final List rawValues; + + // Order is essential for successive binary search + private TreeSet convertedOrdinals; + + public ExactMatchDimFilter(String dimensionName, List valuesToMatch) { + this.dimensionName = dimensionName; + this.rawValues = valuesToMatch; + } + + @Override + public void initialiseForSegment(StarTreeValues starTreeValues, SearchContext searchContext) { + convertedOrdinals = new TreeSet<>(); + Dimension matchedDim = StarTreeQueryHelper.getMatchingDimensionOrError( + dimensionName, + starTreeValues.getStarTreeField().getDimensionsOrder() + ); + DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType( + searchContext.mapperService().fieldType(dimensionName) + ); + for (Object rawValue : rawValues) { + Optional ordinal = dimensionFilterMapper.getMatchingOrdinal( + matchedDim.getField(), + rawValue, + starTreeValues, + DimensionFilter.MatchType.EXACT + ); + // Numeric type returning negative ordinal ( same as their value ) is valid + // Whereas Keyword type returning -ve ordinal indicates it doesn't exist in Star Tree Dimension values. + ordinal.ifPresent(aLong -> convertedOrdinals.add(aLong)); + } + } + + @Override + public void matchStarTreeNodes(StarTreeNode parentNode, StarTreeValues starTreeValues, StarTreeNodeCollector collector) + throws IOException { + if (parentNode != null) { + StarTreeNode lastMatchedNode = null; + for (long ordinal : convertedOrdinals) { + lastMatchedNode = parentNode.getChildForDimensionValue(ordinal, lastMatchedNode); + if (lastMatchedNode != null) { + collector.collectStarTreeNode(lastMatchedNode); + } + } + } + } + + @Override + public boolean matchDimValue(long ordinal, StarTreeValues starTreeValues) { + return convertedOrdinals.contains(ordinal); + } +} diff --git a/server/src/main/java/org/opensearch/search/startree/filter/MatchNoneFilter.java b/server/src/main/java/org/opensearch/search/startree/filter/MatchNoneFilter.java new file mode 100644 index 0000000000000..3066b4d7a8a3f --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/MatchNoneFilter.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree.filter; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeNodeCollector; + +/** + * Filter which matches no StarTreeNodes. + */ +@ExperimentalApi +public class MatchNoneFilter implements DimensionFilter { + @Override + public void initialiseForSegment(StarTreeValues starTreeValues, SearchContext searchContext) { + // Nothing to do as we won't match anything. + } + + @Override + public void matchStarTreeNodes(StarTreeNode parentNode, StarTreeValues starTreeValues, StarTreeNodeCollector collector) { + // Don't match any star tree node. + } + + @Override + public boolean matchDimValue(long ordinal, StarTreeValues starTreeValues) { + return false; + } +} diff --git a/server/src/main/java/org/opensearch/search/startree/filter/RangeMatchDimFilter.java b/server/src/main/java/org/opensearch/search/startree/filter/RangeMatchDimFilter.java new file mode 100644 index 0000000000000..fecf1a9ebf76b --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/RangeMatchDimFilter.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree.filter; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeNodeCollector; +import org.opensearch.search.startree.filter.provider.DimensionFilterMapper; + +import java.io.IOException; +import java.util.Optional; + +/** + * Performs range match based on the params of @{@link org.opensearch.index.query.RangeQueryBuilder} + * Also, contains logic to skip performing range search if it's sure that it won't be found in Star Tree. + */ +@ExperimentalApi +public class RangeMatchDimFilter implements DimensionFilter { + + private final String dimensionName; + + private final Object low; + private final Object high; + private final boolean includeLow; + private final boolean includeHigh; + + private Long lowOrdinal; + private Long highOrdinal; + + private boolean skipRangeCollection = false; + + public RangeMatchDimFilter(String dimensionName, Object low, Object high, boolean includeLow, boolean includeHigh) { + this.dimensionName = dimensionName; + this.low = low; + this.high = high; + this.includeLow = includeLow; + this.includeHigh = includeHigh; + } + + @Override + public void initialiseForSegment(StarTreeValues starTreeValues, SearchContext searchContext) { + skipRangeCollection = false; + DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType( + searchContext.mapperService().fieldType(dimensionName) + ); + lowOrdinal = 0L; + if (low != null) { + MatchType lowMatchType = includeLow ? MatchType.GTE : MatchType.GT; + Optional lowOrdinalFound = dimensionFilterMapper.getMatchingOrdinal(dimensionName, low, starTreeValues, lowMatchType); + if (lowOrdinalFound.isPresent()) { + lowOrdinal = lowOrdinalFound.get(); + } else { + // This is only valid for Non-numeric fields. + // High can't be found since nothing >= low exists. + lowOrdinal = highOrdinal = Long.MAX_VALUE; + skipRangeCollection = true; + return; + } + } + highOrdinal = Long.MAX_VALUE; + if (high != null) { + MatchType highMatchType = includeHigh ? MatchType.LTE : MatchType.LT; + Optional highOrdinalFound = dimensionFilterMapper.getMatchingOrdinal(dimensionName, high, starTreeValues, highMatchType); + highOrdinalFound.ifPresent(ord -> highOrdinal = ord); + } + } + + @Override + public void matchStarTreeNodes(StarTreeNode parentNode, StarTreeValues starTreeValues, StarTreeNodeCollector collector) + throws IOException { + if (parentNode != null && !skipRangeCollection) { + parentNode.collectChildrenInRange(lowOrdinal, highOrdinal, collector); + } + } + + @Override + public boolean matchDimValue(long ordinal, StarTreeValues starTreeValues) { + return lowOrdinal <= ordinal && ordinal <= highOrdinal; + } + +} diff --git a/server/src/main/java/org/opensearch/search/startree/filter/StarTreeFilter.java b/server/src/main/java/org/opensearch/search/startree/filter/StarTreeFilter.java new file mode 100644 index 0000000000000..38a1f092adc6f --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/StarTreeFilter.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree.filter; + +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Container for intermediate/consolidated dimension filters that will be applied for a query in star tree traversal. + */ +@ExperimentalApi +public class StarTreeFilter { + + private final Map> dimensionFilterMap; + + public StarTreeFilter(Map> dimensionFilterMap) { + this.dimensionFilterMap = dimensionFilterMap; + } + + public List getFiltersForDimension(String dimension) { + return dimensionFilterMap.get(dimension); + } + + public Set getDimensions() { + return dimensionFilterMap.keySet(); + } + // TODO : Implement Merging of 2 Star Tree Filters + // This would also involve merging 2 different types of dimension filters. + // It also brings in the challenge of sorting input values in user query for efficient merging. + // Merging Range with Term and Range with Range and so on. + // All these will be implemented post OS 2.19 + +} diff --git a/server/src/main/java/org/opensearch/search/startree/filter/package-info.java b/server/src/main/java/org/opensearch/search/startree/filter/package-info.java new file mode 100644 index 0000000000000..565996d9bfb21 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Star Tree Dimension Filters */ +package org.opensearch.search.startree.filter; diff --git a/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java b/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java new file mode 100644 index 0000000000000..04b5963259d52 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java @@ -0,0 +1,413 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree.filter.provider; + +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.sandbox.document.HalfFloatPoint; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.lucene.BytesRefs; +import org.opensearch.common.lucene.Lucene; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator; +import org.opensearch.index.mapper.KeywordFieldMapper.KeywordFieldType; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.mapper.NumberFieldMapper.NumberFieldType; +import org.opensearch.search.startree.filter.DimensionFilter; +import org.opensearch.search.startree.filter.ExactMatchDimFilter; +import org.opensearch.search.startree.filter.MatchNoneFilter; +import org.opensearch.search.startree.filter.RangeMatchDimFilter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.BYTE; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.DOUBLE; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.FLOAT; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.HALF_FLOAT; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.INTEGER; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.LONG; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.SHORT; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.hasDecimalPart; +import static org.opensearch.index.mapper.NumberFieldMapper.NumberType.signum; + +/** + * Generates the @{@link DimensionFilter} raw values and the @{@link MappedFieldType} of the dimension. + */ +@ExperimentalApi +public interface DimensionFilterMapper { + /** + * Generates @{@link ExactMatchDimFilter} from Term/Terms query input. + * @param mappedFieldType: + * @param rawValues: + * @return : + */ + DimensionFilter getExactMatchFilter(MappedFieldType mappedFieldType, List rawValues); + + /** + * Generates @{@link RangeMatchDimFilter} from Range query input. + * @param mappedFieldType: + * @param rawLow: + * @param rawHigh: + * @param includeLow: + * @param includeHigh: + * @return : + */ + DimensionFilter getRangeMatchFilter( + MappedFieldType mappedFieldType, + Object rawLow, + Object rawHigh, + boolean includeLow, + boolean includeHigh + ); + + /** + * Called during conversion from parsedUserInput to segmentOrdinal for every segment. + * @param dimensionName: + * @param value: + * @param starTreeValues: + * @param matchType: + * @return : + */ + Optional getMatchingOrdinal( + String dimensionName, + Object value, + StarTreeValues starTreeValues, + DimensionFilter.MatchType matchType + ); + + /** + * Singleton Factory for @{@link DimensionFilterMapper} + */ + class Factory { + + private static final Map NUMERIC_SINGLETON_MAPPINGS = Map.of( + BYTE, + new IntegerFieldMapperNumeric(), + SHORT, + new IntegerFieldMapperNumeric(), + INTEGER, + new IntegerFieldMapperNumeric(), + LONG, + new SignedLongFieldMapperNumeric(), + HALF_FLOAT, + new HalfFloatFieldMapperNumeric(), + FLOAT, + new FloatFieldMapperNumeric(), + DOUBLE, + new DoubleFieldMapperNumeric() + ); + + public static DimensionFilterMapper fromMappedFieldType(MappedFieldType mappedFieldType) { + if (mappedFieldType instanceof KeywordFieldType) { + return new KeywordFieldMapper(); + } else if (mappedFieldType instanceof NumberFieldType) { + NumberFieldMapper.NumberType numberType = ((NumberFieldMapper.NumberFieldType) mappedFieldType).numberType(); + return NUMERIC_SINGLETON_MAPPINGS.get(numberType); + } else { + return null; + } + } + } + +} + +abstract class NumericMapper implements DimensionFilterMapper { + + @Override + public Optional getMatchingOrdinal( + String dimensionName, + Object value, + StarTreeValues starTreeValues, + DimensionFilter.MatchType matchType + ) { + // Casting to long ensures that all numeric fields have been converted to equivalent long at request parsing time. + return Optional.of((long) value); + } +} + +abstract class NumericNonDecimalMapper extends NumericMapper { + + @Override + public DimensionFilter getExactMatchFilter(MappedFieldType mappedFieldType, List rawValues) { + NumberFieldType numberFieldType = (NumberFieldType) mappedFieldType; + List convertedValues = new ArrayList<>(rawValues.size()); + for (Object rawValue : rawValues) { + convertedValues.add(numberFieldType.numberType().parse(rawValue, true).longValue()); + } + return new ExactMatchDimFilter(mappedFieldType.name(), convertedValues); + } + + @Override + public DimensionFilter getRangeMatchFilter( + MappedFieldType mappedFieldType, + Object rawLow, + Object rawHigh, + boolean includeLow, + boolean includeHigh + ) { + NumberFieldType numberFieldType = (NumberFieldType) mappedFieldType; + + Long parsedLow = rawLow == null ? defaultMinimum() : numberFieldType.numberType().parse(rawLow, true).longValue(); + Long parsedHigh = rawHigh == null ? defaultMaximum() : numberFieldType.numberType().parse(rawHigh, true).longValue(); + + boolean lowerTermHasDecimalPart = hasDecimalPart(parsedLow); + if ((lowerTermHasDecimalPart == false && includeLow == false) || (lowerTermHasDecimalPart && signum(parsedLow) > 0)) { + if (parsedLow.equals(defaultMaximum())) { + return new MatchNoneFilter(); + } + ++parsedLow; + } + boolean upperTermHasDecimalPart = hasDecimalPart(parsedHigh); + if ((upperTermHasDecimalPart == false && includeHigh == false) || (upperTermHasDecimalPart && signum(parsedHigh) < 0)) { + if (parsedHigh.equals(defaultMinimum())) { + return new MatchNoneFilter(); + } + --parsedHigh; + } + return new RangeMatchDimFilter(mappedFieldType.name(), parsedLow, parsedHigh, true, true); + } + + abstract Long defaultMinimum(); + + abstract Long defaultMaximum(); + +} + +class IntegerFieldMapperNumeric extends NumericNonDecimalMapper { + @Override + Long defaultMinimum() { + return (long) Integer.MIN_VALUE; + } + + @Override + Long defaultMaximum() { + return (long) Integer.MAX_VALUE; + } +} + +class SignedLongFieldMapperNumeric extends NumericNonDecimalMapper { + @Override + Long defaultMinimum() { + return Long.MIN_VALUE; + } + + @Override + Long defaultMaximum() { + return Long.MAX_VALUE; + } +} + +abstract class NumericDecimalFieldMapper extends NumericMapper { + + @Override + public DimensionFilter getExactMatchFilter(MappedFieldType mappedFieldType, List rawValues) { + NumberFieldType numberFieldType = (NumberFieldType) mappedFieldType; + List convertedValues = new ArrayList<>(rawValues.size()); + for (Object rawValue : rawValues) { + convertedValues.add(convertToDocValues(numberFieldType.numberType().parse(rawValue, true))); + } + return new ExactMatchDimFilter(mappedFieldType.name(), convertedValues); + } + + @Override + public DimensionFilter getRangeMatchFilter( + MappedFieldType mappedFieldType, + Object rawLow, + Object rawHigh, + boolean includeLow, + boolean includeHigh + ) { + NumberFieldType numberFieldType = (NumberFieldType) mappedFieldType; + Number l = Long.MIN_VALUE; + Number u = Long.MAX_VALUE; + if (rawLow != null) { + l = numberFieldType.numberType().parse(rawLow, false); + if (includeLow == false) { + l = getNextHigh(l); + } + l = convertToDocValues(l); + } + if (rawHigh != null) { + u = numberFieldType.numberType().parse(rawHigh, false); + if (includeHigh == false) { + u = getNextLow(u); + } + u = convertToDocValues(u); + } + return new RangeMatchDimFilter(numberFieldType.name(), l, u, true, true); + } + + abstract long convertToDocValues(Number parsedValue); + + abstract Number getNextLow(Number parsedValue); + + abstract Number getNextHigh(Number parsedValue); + +} + +class HalfFloatFieldMapperNumeric extends NumericDecimalFieldMapper { + @Override + long convertToDocValues(Number parsedValue) { + return HalfFloatPoint.halfFloatToSortableShort((Float) parsedValue); + } + + @Override + Number getNextLow(Number parsedValue) { + return HalfFloatPoint.nextDown((Float) parsedValue); + } + + @Override + Number getNextHigh(Number parsedValue) { + return HalfFloatPoint.nextUp((Float) parsedValue); + } +} + +class FloatFieldMapperNumeric extends NumericDecimalFieldMapper { + @Override + long convertToDocValues(Number parsedValue) { + return NumericUtils.floatToSortableInt((Float) parsedValue); + } + + @Override + Number getNextLow(Number parsedValue) { + return FloatPoint.nextDown((Float) parsedValue); + } + + @Override + Number getNextHigh(Number parsedValue) { + return FloatPoint.nextUp((Float) parsedValue); + } +} + +class DoubleFieldMapperNumeric extends NumericDecimalFieldMapper { + @Override + long convertToDocValues(Number parsedValue) { + return NumericUtils.doubleToSortableLong((Double) parsedValue); + } + + @Override + Number getNextLow(Number parsedValue) { + return DoublePoint.nextDown((Double) parsedValue); + } + + @Override + Number getNextHigh(Number parsedValue) { + return DoublePoint.nextUp((Double) parsedValue); + } +} + +class KeywordFieldMapper implements DimensionFilterMapper { + + @Override + public DimensionFilter getExactMatchFilter(MappedFieldType mappedFieldType, List rawValues) { + KeywordFieldType keywordFieldType = (KeywordFieldType) mappedFieldType; + List convertedValues = new ArrayList<>(rawValues.size()); + for (Object rawValue : rawValues) { + convertedValues.add(parseRawKeyword(mappedFieldType.name(), rawValue, keywordFieldType)); + } + return new ExactMatchDimFilter(mappedFieldType.name(), convertedValues); + } + + @Override + public DimensionFilter getRangeMatchFilter( + MappedFieldType mappedFieldType, + Object rawLow, + Object rawHigh, + boolean includeLow, + boolean includeHigh + ) { + KeywordFieldType keywordFieldType = (KeywordFieldType) mappedFieldType; + return new RangeMatchDimFilter( + mappedFieldType.name(), + parseRawKeyword(mappedFieldType.name(), rawLow, keywordFieldType), + parseRawKeyword(mappedFieldType.name(), rawHigh, keywordFieldType), + includeLow, + includeHigh + ); + } + + @Override + public Optional getMatchingOrdinal( + String dimensionName, + Object value, + StarTreeValues starTreeValues, + DimensionFilter.MatchType matchType + ) { + SortedSetStarTreeValuesIterator sortedSetIterator = (SortedSetStarTreeValuesIterator) starTreeValues.getDimensionValuesIterator( + dimensionName + ); + try { + if (matchType == DimensionFilter.MatchType.EXACT) { + long ordinal = sortedSetIterator.lookupTerm((BytesRef) value); + return ordinal >= 0 ? Optional.of(ordinal) : Optional.empty(); + } else { + TermsEnum termsEnum = sortedSetIterator.termsEnum(); + TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil((BytesRef) value); + // We reached the end and couldn't match anything, else we found a term which matches. + // LT || LTE + // If we found a term just greater, then return ordinal of the term just before it. + // Checking if we are in bounds for satisfying LT + // Checking if we are in bounds for satisfying LT + switch (matchType) { + case GTE: + return seekStatus == TermsEnum.SeekStatus.END ? Optional.empty() : Optional.of(termsEnum.ord()); + case GT: + return switch (seekStatus) { + case END -> Optional.empty(); + case FOUND -> ((termsEnum.ord() + 1) < sortedSetIterator.getValueCount()) + ? Optional.of(termsEnum.ord() + 1) + : Optional.empty(); + case NOT_FOUND -> Optional.of(termsEnum.ord()); + }; + case LTE: + if (seekStatus == TermsEnum.SeekStatus.NOT_FOUND) { + return ((termsEnum.ord() - 1) >= 0) ? Optional.of(termsEnum.ord() - 1) : Optional.empty(); + } else { + return Optional.of(termsEnum.ord()); + } + case LT: + if (seekStatus == TermsEnum.SeekStatus.END) { + return Optional.of(termsEnum.ord()); + } else { + return ((termsEnum.ord() - 1) >= 0) ? Optional.of(termsEnum.ord() - 1) : Optional.empty(); + } + default: + throw new IllegalStateException("unexpected matchType " + matchType); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + // TODO : Think around making TermBasedFT#indexedValueForSearch() accessor public for reuse here. + private Object parseRawKeyword(String field, Object rawValue, KeywordFieldType keywordFieldType) { + Object parsedValue = null; + if (rawValue != null) { + if (keywordFieldType.getTextSearchInfo().getSearchAnalyzer() == Lucene.KEYWORD_ANALYZER) { + parsedValue = BytesRefs.toBytesRef(rawValue); + } else { + if (rawValue instanceof BytesRef) { + rawValue = ((BytesRef) rawValue).utf8ToString(); + } + parsedValue = keywordFieldType.getTextSearchInfo().getSearchAnalyzer().normalize(field, rawValue.toString()); + } + } + return parsedValue; + } + +} diff --git a/server/src/main/java/org/opensearch/search/startree/filter/provider/StarTreeFilterProvider.java b/server/src/main/java/org/opensearch/search/startree/filter/provider/StarTreeFilterProvider.java new file mode 100644 index 0000000000000..9932c974cefd6 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/provider/StarTreeFilterProvider.java @@ -0,0 +1,174 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree.filter.provider; + +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.mapper.CompositeDataCubeFieldType; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.RangeQueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeQueryHelper; +import org.opensearch.search.startree.filter.StarTreeFilter; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Converts a {@link QueryBuilder} into a {@link StarTreeFilter} by generating the appropriate @{@link org.opensearch.search.startree.filter.DimensionFilter} + * for the fields provided in the user query. + */ +@ExperimentalApi +public interface StarTreeFilterProvider { + + /** + * Returns the {@link StarTreeFilter} generated from the {@link QueryBuilder} + * @param context: + * @param rawFilter: + * @param compositeFieldType: + * @return : {@link StarTreeFilter} if the query shape is supported, else null. + * @throws IOException : + */ + StarTreeFilter getFilter(SearchContext context, QueryBuilder rawFilter, CompositeDataCubeFieldType compositeFieldType) + throws IOException; + + StarTreeFilterProvider MATCH_ALL_PROVIDER = (context, rawFilter, compositeFieldType) -> new StarTreeFilter(Collections.emptyMap()); + + /** + * Singleton instances for most {@link StarTreeFilterProvider} + */ + class SingletonFactory { + + private static final Map, StarTreeFilterProvider> QUERY_BUILDERS_TO_STF_PROVIDER = Map.of( + MatchAllQueryBuilder.class, + MATCH_ALL_PROVIDER, + TermQueryBuilder.class, + new TermStarTreeFilterProvider(), + TermsQueryBuilder.class, + new TermsStarTreeFilterProvider(), + RangeQueryBuilder.class, + new RangeStarTreeFilterProvider() + ); + + public static StarTreeFilterProvider getProvider(QueryBuilder query) { + if (query != null) { + return QUERY_BUILDERS_TO_STF_PROVIDER.get(query.getClass()); + } + return MATCH_ALL_PROVIDER; + } + + } + + /** + * Converts @{@link TermQueryBuilder} into @{@link org.opensearch.search.startree.filter.ExactMatchDimFilter} + */ + class TermStarTreeFilterProvider implements StarTreeFilterProvider { + @Override + public StarTreeFilter getFilter(SearchContext context, QueryBuilder rawFilter, CompositeDataCubeFieldType compositeFieldType) + throws IOException { + TermQueryBuilder termQueryBuilder = (TermQueryBuilder) rawFilter; + String field = termQueryBuilder.fieldName(); + MappedFieldType mappedFieldType = context.mapperService().fieldType(field); + DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType(mappedFieldType); + Dimension matchedDimension = StarTreeQueryHelper.getMatchingDimensionOrNull(field, compositeFieldType.getDimensions()); + if (matchedDimension == null || mappedFieldType == null || dimensionFilterMapper == null) { + return null; // Indicates Aggregators to fallback to default implementation. + } else { + // FIXME : DocValuesType validation is field type specific and not query builder specific should happen elsewhere. + Query query = termQueryBuilder.toQuery(context.getQueryShardContext()); + if (query instanceof MatchNoDocsQuery) { + return new StarTreeFilter(Collections.emptyMap()); + } else { + return new StarTreeFilter( + Map.of( + field, + List.of(dimensionFilterMapper.getExactMatchFilter(mappedFieldType, List.of(termQueryBuilder.value()))) + ) + ); + } + } + } + } + + /** + * Converts @{@link TermsQueryBuilder} into @{@link org.opensearch.search.startree.filter.ExactMatchDimFilter} + */ + class TermsStarTreeFilterProvider implements StarTreeFilterProvider { + @Override + public StarTreeFilter getFilter(SearchContext context, QueryBuilder rawFilter, CompositeDataCubeFieldType compositeFieldType) + throws IOException { + TermsQueryBuilder termsQueryBuilder = (TermsQueryBuilder) rawFilter; + String field = termsQueryBuilder.fieldName(); + Dimension matchedDimension = StarTreeQueryHelper.getMatchingDimensionOrNull(field, compositeFieldType.getDimensions()); + MappedFieldType mappedFieldType = context.mapperService().fieldType(field); + DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType(mappedFieldType); + if (matchedDimension == null || mappedFieldType == null || dimensionFilterMapper == null) { + return null; // Indicates Aggregators to fallback to default implementation. + } else { + Query query = termsQueryBuilder.toQuery(context.getQueryShardContext()); + if (query instanceof MatchNoDocsQuery) { + return new StarTreeFilter(Collections.emptyMap()); + } else { + return new StarTreeFilter( + Map.of(field, List.of(dimensionFilterMapper.getExactMatchFilter(mappedFieldType, termsQueryBuilder.values()))) + ); + } + } + } + } + + /** + * Converts @{@link RangeQueryBuilder} into @{@link org.opensearch.search.startree.filter.RangeMatchDimFilter} + */ + class RangeStarTreeFilterProvider implements StarTreeFilterProvider { + + @Override + public StarTreeFilter getFilter(SearchContext context, QueryBuilder rawFilter, CompositeDataCubeFieldType compositeFieldType) + throws IOException { + RangeQueryBuilder rangeQueryBuilder = (RangeQueryBuilder) rawFilter; + String field = rangeQueryBuilder.fieldName(); + Dimension matchedDimension = StarTreeQueryHelper.getMatchingDimensionOrNull(field, compositeFieldType.getDimensions()); + MappedFieldType mappedFieldType = context.mapperService().fieldType(field); + DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType(mappedFieldType); + if (matchedDimension == null || mappedFieldType == null || dimensionFilterMapper == null) { + return null; + } else { + Query query = rangeQueryBuilder.toQuery(context.getQueryShardContext()); + if (query instanceof MatchNoDocsQuery) { + return new StarTreeFilter(Collections.emptyMap()); + } else { + return new StarTreeFilter( + Map.of( + field, + List.of( + dimensionFilterMapper.getRangeMatchFilter( + mappedFieldType, + rangeQueryBuilder.from(), + rangeQueryBuilder.to(), + rangeQueryBuilder.includeLower(), + rangeQueryBuilder.includeUpper() + ) + ) + ) + ); + } + } + } + + } + +} diff --git a/server/src/main/java/org/opensearch/search/startree/filter/provider/package-info.java b/server/src/main/java/org/opensearch/search/startree/filter/provider/package-info.java new file mode 100644 index 0000000000000..b6ff423e1f140 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/provider/package-info.java @@ -0,0 +1,10 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** Star Tree Filter and Dimension Filter Providers */ +package org.opensearch.search.startree.filter.provider; diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java index 4b5bd951f80a0..29ced9d5f0f0c 100644 --- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java @@ -77,6 +77,7 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.common.regex.Regex; import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.ArrayUtils; @@ -122,12 +123,10 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; -import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED; import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY; import static org.opensearch.common.util.IndexUtils.filterIndices; @@ -164,8 +163,6 @@ public class RestoreService implements ClusterStateApplier { private static final Set USER_UNMODIFIABLE_SETTINGS = unmodifiableSet( newHashSet( - SETTING_NUMBER_OF_SHARDS, - SETTING_VERSION_CREATED, SETTING_INDEX_UUID, SETTING_CREATION_DATE, SETTING_HISTORY_UUID, @@ -180,7 +177,7 @@ public class RestoreService implements ClusterStateApplier { private static final String REMOTE_STORE_INDEX_SETTINGS_REGEX = "index.remote_store.*"; static { - Set unremovable = new HashSet<>(USER_UNMODIFIABLE_SETTINGS.size() + 4); + Set unremovable = new HashSet<>(USER_UNMODIFIABLE_SETTINGS.size() + 3); unremovable.addAll(USER_UNMODIFIABLE_SETTINGS); unremovable.add(SETTING_NUMBER_OF_REPLICAS); unremovable.add(SETTING_AUTO_EXPAND_REPLICAS); @@ -202,6 +199,8 @@ public class RestoreService implements ClusterStateApplier { private final ClusterSettings clusterSettings; + private final IndexScopedSettings indexScopedSettings; + private final IndicesService indicesService; private final Supplier clusterInfoSupplier; @@ -234,6 +233,7 @@ public RestoreService( this.clusterSettings = clusterService.getClusterSettings(); this.shardLimitValidator = shardLimitValidator; this.indicesService = indicesService; + this.indexScopedSettings = createIndexService.getIndexScopedSettings(); this.clusterInfoSupplier = clusterInfoSupplier; this.dataToFileCacheSizeRatioSupplier = dataToFileCacheSizeRatioSupplier; @@ -835,6 +835,11 @@ private IndexMetadata updateIndexSettings( snapshot, "cannot remove setting [" + ignoredSetting + "] on restore" ); + } else if (indexScopedSettings.isUnmodifiableOnRestoreSetting(ignoredSetting)) { + throw new SnapshotRestoreException( + snapshot, + "cannot remove UnmodifiableOnRestore setting [" + ignoredSetting + "] on restore" + ); } else { keyFilters.add(ignoredSetting); } @@ -853,7 +858,7 @@ private IndexMetadata updateIndexSettings( } Predicate settingsFilter = k -> { - if (USER_UNREMOVABLE_SETTINGS.contains(k) == false) { + if (USER_UNREMOVABLE_SETTINGS.contains(k) == false && !indexScopedSettings.isUnmodifiableOnRestoreSetting(k)) { for (String filterKey : keyFilters) { if (k.equals(filterKey)) { return false; @@ -872,6 +877,11 @@ private IndexMetadata updateIndexSettings( .put(normalizedChangeSettings.filter(k -> { if (USER_UNMODIFIABLE_SETTINGS.contains(k)) { throw new SnapshotRestoreException(snapshot, "cannot modify setting [" + k + "] on restore"); + } else if (indexScopedSettings.isUnmodifiableOnRestoreSetting(k)) { + throw new SnapshotRestoreException( + snapshot, + "cannot modify UnmodifiableOnRestore setting [" + k + "] on restore" + ); } else { return true; } diff --git a/libs/common/src/test/resources/org/opensearch/bootstrap/test.policy b/server/src/main/resources/org/opensearch/bootstrap/test.policy similarity index 100% rename from libs/common/src/test/resources/org/opensearch/bootstrap/test.policy rename to server/src/main/resources/org/opensearch/bootstrap/test.policy diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java index 129475e0fa3fb..5573cdc290207 100644 --- a/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java +++ b/server/src/test/java/org/opensearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java @@ -32,8 +32,8 @@ package org.opensearch.action.admin.cluster.reroute; +import org.opensearch.action.support.clustermanager.AcknowledgedRequest; import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; -import org.opensearch.action.support.master.AcknowledgedRequest; import org.opensearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand; import org.opensearch.cluster.routing.allocation.command.AllocateReplicaAllocationCommand; import org.opensearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand; diff --git a/server/src/test/java/org/opensearch/action/search/TransportMultiSearchActionTests.java b/server/src/test/java/org/opensearch/action/search/TransportMultiSearchActionTests.java index 48970e2b96add..45980e7137ce4 100644 --- a/server/src/test/java/org/opensearch/action/search/TransportMultiSearchActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/TransportMultiSearchActionTests.java @@ -49,7 +49,9 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.search.internal.InternalSearchResponse; +import org.opensearch.tasks.CancellableTask; import org.opensearch.tasks.Task; +import org.opensearch.tasks.TaskListener; import org.opensearch.tasks.TaskManager; import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; @@ -62,7 +64,9 @@ import java.util.IdentityHashMap; import java.util.List; import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -289,4 +293,118 @@ public void testDefaultMaxConcurrentSearches() { assertThat(result, equalTo(1)); } + public void testCancellation() { + // Initialize dependencies of TransportMultiSearchAction + Settings settings = Settings.builder().put("node.name", TransportMultiSearchActionTests.class.getSimpleName()).build(); + ActionFilters actionFilters = mock(ActionFilters.class); + when(actionFilters.filters()).thenReturn(new ActionFilter[0]); + ThreadPool threadPool = new ThreadPool(settings); + TransportService transportService = new TransportService( + Settings.EMPTY, + mock(Transport.class), + threadPool, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, + boundAddress -> DiscoveryNode.createLocal(settings, boundAddress.publishAddress(), UUIDs.randomBase64UUID()), + null, + Collections.emptySet(), + NoopTracer.INSTANCE + ) { + @Override + public TaskManager getTaskManager() { + return taskManager; + } + }; + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("test")).build()); + + // Keep track of the number of concurrent searches started by multi search api, + // and if there are more searches than is allowed create an error and remember that. + int maxAllowedConcurrentSearches = 1; // Allow 1 search at a time. + AtomicInteger counter = new AtomicInteger(); + AtomicReference errorHolder = new AtomicReference<>(); + // randomize whether or not requests are executed asynchronously + ExecutorService executorService = threadPool.executor(ThreadPool.Names.GENERIC); + final Set requests = Collections.newSetFromMap(Collections.synchronizedMap(new IdentityHashMap<>())); + CountDownLatch countDownLatch = new CountDownLatch(1); + CancellableTask[] parentTask = new CancellableTask[1]; + NodeClient client = new NodeClient(settings, threadPool) { + @Override + public void search(final SearchRequest request, final ActionListener listener) { + if (parentTask[0] != null && parentTask[0].isCancelled()) { + fail("Should not execute search after parent task is cancelled"); + } + try { + countDownLatch.await(10, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + requests.add(request); + executorService.execute(() -> { + counter.decrementAndGet(); + listener.onResponse( + new SearchResponse( + InternalSearchResponse.empty(), + null, + 0, + 0, + 0, + 0L, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ) + ); + }); + } + + @Override + public String getLocalNodeId() { + return "local_node_id"; + } + }; + + TransportMultiSearchAction action = new TransportMultiSearchAction( + threadPool, + actionFilters, + transportService, + clusterService, + 10, + System::nanoTime, + client + ); + + // Execute the multi search api and fail if we find an error after executing: + try { + /* + * Allow for a large number of search requests in a single batch as previous implementations could stack overflow if the number + * of requests in a single batch was large + */ + int numSearchRequests = scaledRandomIntBetween(1024, 8192); + MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); + multiSearchRequest.maxConcurrentSearchRequests(maxAllowedConcurrentSearches); + for (int i = 0; i < numSearchRequests; i++) { + multiSearchRequest.add(new SearchRequest()); + } + MultiSearchResponse[] responses = new MultiSearchResponse[1]; + Exception[] exceptions = new Exception[1]; + parentTask[0] = (CancellableTask) action.execute(multiSearchRequest, new TaskListener<>() { + @Override + public void onResponse(Task task, MultiSearchResponse items) { + responses[0] = items; + } + + @Override + public void onFailure(Task task, Exception e) { + exceptions[0] = e; + } + }); + parentTask[0].cancel("Giving up"); + countDownLatch.countDown(); + + assertNull(responses[0]); + assertNull(exceptions[0]); + } finally { + assertTrue(OpenSearchTestCase.terminate(threadPool)); + } + } } diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponseTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponseTests.java index 68b7d97fa1427..1d839cc5f2c16 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponseTests.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/ShardsAcknowledgedResponseTests.java @@ -32,7 +32,6 @@ package org.opensearch.action.support.clustermanager; import org.opensearch.Version; -import org.opensearch.action.support.master.ShardsAcknowledgedResponse; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index 99e259c8170f3..05ae67d10f4cb 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -34,7 +34,7 @@ import org.opensearch.Version; import org.opensearch.action.admin.indices.alias.Alias; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.applicationtemplates.ClusterStateSystemTemplateLoader; import org.opensearch.cluster.applicationtemplates.SystemTemplateMetadata; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceTests.java index 562e293083633..ebce050c67539 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/TemplateUpgradeServiceTests.java @@ -35,7 +35,7 @@ import org.opensearch.Version; import org.opensearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.AdminClient; import org.opensearch.client.Client; import org.opensearch.client.IndicesAdminClient; diff --git a/server/src/test/java/org/opensearch/common/settings/ScopedSettingsTests.java b/server/src/test/java/org/opensearch/common/settings/ScopedSettingsTests.java index 7780481c9deff..55e3cfa34040b 100644 --- a/server/src/test/java/org/opensearch/common/settings/ScopedSettingsTests.java +++ b/server/src/test/java/org/opensearch/common/settings/ScopedSettingsTests.java @@ -789,6 +789,30 @@ public void testIsFinal() { assertTrue(settings.isFinalSetting("foo.group.key")); } + public void testIsUnmodifiableOnRestore() { + ClusterSettings settings = new ClusterSettings( + Settings.EMPTY, + new HashSet<>( + Arrays.asList( + Setting.intSetting("foo.int", 1, Property.UnmodifiableOnRestore, Property.IndexScope, Property.NodeScope), + Setting.groupSetting("foo.group.", Property.UnmodifiableOnRestore, Property.IndexScope, Property.NodeScope), + Setting.groupSetting("foo.list.", Property.UnmodifiableOnRestore, Property.IndexScope, Property.NodeScope), + Setting.intSetting("foo.int.baz", 1, Property.IndexScope, Property.NodeScope) + ) + ) + ); + + assertFalse(settings.isUnmodifiableOnRestoreSetting("foo.int.baz")); + assertTrue(settings.isUnmodifiableOnRestoreSetting("foo.int")); + + assertFalse(settings.isUnmodifiableOnRestoreSetting("foo.list")); + assertTrue(settings.isUnmodifiableOnRestoreSetting("foo.list.0.key")); + assertTrue(settings.isUnmodifiableOnRestoreSetting("foo.list.key")); + + assertFalse(settings.isUnmodifiableOnRestoreSetting("foo.group")); + assertTrue(settings.isUnmodifiableOnRestoreSetting("foo.group.key")); + } + public void testDiff() throws IOException { Setting fooBarBaz = Setting.intSetting("foo.bar.baz", 1, Property.NodeScope); Setting fooBar = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope); diff --git a/server/src/test/java/org/opensearch/common/settings/SettingTests.java b/server/src/test/java/org/opensearch/common/settings/SettingTests.java index c3c399a9d88b2..a0788b0c83e11 100644 --- a/server/src/test/java/org/opensearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/opensearch/common/settings/SettingTests.java @@ -1439,6 +1439,22 @@ public void testRejectConflictingDynamicAndFinalProperties() { assertThat(ex.getMessage(), containsString("final setting [foo.bar] cannot be dynamic")); } + public void testRejectConflictingDynamicAndUnmodifiableOnRestoreProperties() { + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> Setting.simpleString("foo.bar", Property.UnmodifiableOnRestore, Property.Dynamic) + ); + assertThat(ex.getMessage(), containsString("UnmodifiableOnRestore setting [foo.bar] cannot be dynamic")); + } + + public void testRejectNonIndexScopedUnmodifiableOnRestoreSetting() { + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> Setting.simpleString("foo.bar", Property.UnmodifiableOnRestore) + ); + assertThat(e, hasToString(containsString("non-index-scoped setting [foo.bar] can not have property [UnmodifiableOnRestore]"))); + } + public void testRejectNonIndexScopedNotCopyableOnResizeSetting() { final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeSearchTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeSearchTests.java new file mode 100644 index 0000000000000..de3c6da078921 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeSearchTests.java @@ -0,0 +1,332 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.fileformats.node; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.StarTreeWriter; +import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; +import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.search.startree.StarTreeNodeCollector; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.ALL; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FixedLengthStarTreeNodeSearchTests extends OpenSearchTestCase { + + public void testExactMatch() { + long[] randomSorted = random().longs(100, Long.MIN_VALUE, Long.MAX_VALUE).toArray(); + Arrays.sort(randomSorted); + for (boolean createStarNode : new boolean[] { true, false }) { + for (boolean createNullNode : new boolean[] { true, false }) { + createStarTreeForDimension(new long[] { -1, 1, 2, 5 }, createStarNode, createNullNode, List.of(fixedLengthStarTreeNode -> { + try { + boolean result = true; + FixedLengthStarTreeNode lastMatchedNode; + lastMatchedNode = (FixedLengthStarTreeNode) fixedLengthStarTreeNode.getChildForDimensionValue(-1L); + result &= -1 == lastMatchedNode.getDimensionValue(); + // Leaf Node should return null + result &= null == lastMatchedNode.getChildForDimensionValue(5L); + result &= null == lastMatchedNode.getChildForDimensionValue(5L, lastMatchedNode); + // Asserting Last Matched Node works as expected + lastMatchedNode = (FixedLengthStarTreeNode) fixedLengthStarTreeNode.getChildForDimensionValue(1L, lastMatchedNode); + result &= 1 == lastMatchedNode.getDimensionValue(); + lastMatchedNode = (FixedLengthStarTreeNode) fixedLengthStarTreeNode.getChildForDimensionValue(5L, lastMatchedNode); + result &= 5 == lastMatchedNode.getDimensionValue(); + // Asserting null is returned when last matched node is after the value to search. + lastMatchedNode = (FixedLengthStarTreeNode) fixedLengthStarTreeNode.getChildForDimensionValue(2L, lastMatchedNode); + result &= null == lastMatchedNode; + // When dimension value is null + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(null); + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(null, null); + // non-existing dimensionValue + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(4L); + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(randomLongBetween(6, Long.MAX_VALUE)); + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(randomLongBetween(Long.MIN_VALUE, -2)); + return result; + } catch (IOException e) { + throw new RuntimeException(e); + } + })); + createStarTreeForDimension(new long[] { 1 }, createStarNode, createNullNode, List.of(fixedLengthStarTreeNode -> { + try { + boolean result = true; + result &= 1 == fixedLengthStarTreeNode.getChildForDimensionValue(1L).getDimensionValue(); + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(2L); + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(randomLongBetween(2, Long.MAX_VALUE)); + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(randomLongBetween(Long.MIN_VALUE, 0)); + return result; + } catch (IOException e) { + throw new RuntimeException(e); + } + })); + createStarTreeForDimension(new long[] {}, createStarNode, createNullNode, List.of(fixedLengthStarTreeNode -> { + try { + boolean result = true; + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(1L); + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(randomLongBetween(0, Long.MAX_VALUE)); + result &= null == fixedLengthStarTreeNode.getChildForDimensionValue(randomLongBetween(Long.MIN_VALUE, 0)); + return result; + } catch (IOException e) { + throw new RuntimeException(e); + } + })); + createStarTreeForDimension(randomSorted, createStarNode, createNullNode, List.of(fixedLengthStarTreeNode -> { + boolean result = true; + for (int i = 1; i <= 100; i++) { + try { + ArrayBasedCollector collector = new ArrayBasedCollector(); + long key = randomLong(); + FixedLengthStarTreeNode node = (FixedLengthStarTreeNode) fixedLengthStarTreeNode.getChildForDimensionValue(key); + long match = Arrays.binarySearch(randomSorted, key); + if (match >= 0) { + assertNotNull(node); + assertEquals(key, node.getDimensionValue()); + } else { + assertEquals(0, collector.collectedNodeCount()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return result; + })); + } + } + } + + public void testRangeMatch() { + long[] randomSorted = random().longs(100, Long.MIN_VALUE, Long.MAX_VALUE).toArray(); + Arrays.sort(randomSorted); + for (boolean createStarNode : new boolean[] { true, false }) { + for (boolean createNullNode : new boolean[] { true, false }) { + createStarTreeForDimension( + new long[] { -10, -1, 1, 2, 5, 9, 25 }, + createStarNode, + createNullNode, + List.of(fixedLengthStarTreeNode -> { + try { + boolean result = true; + ArrayBasedCollector collector; + // Whole range + collector = new ArrayBasedCollector(); + fixedLengthStarTreeNode.collectChildrenInRange(-20, 26, collector); + result &= collector.matchValues(new long[] { -10, -1, 1, 2, 5, 9, 25 }); + // Subset matched from left + collector = new ArrayBasedCollector(); + fixedLengthStarTreeNode.collectChildrenInRange(-2, 1, collector); + result &= collector.matchValues(new long[] { -1, 1 }); + // Subset matched from right + collector = new ArrayBasedCollector(); + fixedLengthStarTreeNode.collectChildrenInRange(6, 100, collector); + result &= collector.matchValues(new long[] { 9, 25 }); + // No match on left + collector = new ArrayBasedCollector(); + fixedLengthStarTreeNode.collectChildrenInRange(-30, -20, collector); + result &= collector.collectedNodeCount() == 0; + // No match on right + collector = new ArrayBasedCollector(); + fixedLengthStarTreeNode.collectChildrenInRange(30, 50, collector); + result &= collector.collectedNodeCount() == 0; + // Low > High + collector = new ArrayBasedCollector(); + fixedLengthStarTreeNode.collectChildrenInRange(50, 10, collector); + result &= collector.collectedNodeCount() == 0; + // Match leftmost + collector = new ArrayBasedCollector(); + fixedLengthStarTreeNode.collectChildrenInRange(-30, -10, collector); + result &= collector.matchValues(new long[] { -10 }); + // Match rightmost + collector = new ArrayBasedCollector(); + fixedLengthStarTreeNode.collectChildrenInRange(10, 25, collector); + result &= collector.matchValues(new long[] { 25 }); + // Match contains interval which has nothing + collector = new ArrayBasedCollector(); + fixedLengthStarTreeNode.collectChildrenInRange(10, 24, collector); + result &= collector.collectedNodeCount() == 0; + // Match contains interval which has nothing + collector = new ArrayBasedCollector(); + fixedLengthStarTreeNode.collectChildrenInRange(6, 24, collector); + result &= collector.matchValues(new long[] { 9 }); + return result; + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + ); + createStarTreeForDimension(randomSorted, createStarNode, createNullNode, List.of(fixedLengthStarTreeNode -> { + boolean result = true; + TreeSet treeSet = Arrays.stream(randomSorted).boxed().collect(Collectors.toCollection(TreeSet::new)); + for (int i = 1; i <= 100; i++) { + try { + ArrayBasedCollector collector = new ArrayBasedCollector(); + long low = randomLong(), high = randomLong(); + fixedLengthStarTreeNode.collectChildrenInRange(low, high, collector); + if (low < high) { + Long lowValue = treeSet.ceiling(low); + if (lowValue != null) { + Long highValue = treeSet.floor(high); + if (highValue != null && highValue >= lowValue) { + collector.matchValues( + Arrays.copyOfRange( + randomSorted, + Arrays.binarySearch(randomSorted, lowValue), + Arrays.binarySearch(randomSorted, highValue) + ) + ); + } else if (lowValue <= high) { + collector.matchValues(new long[] { lowValue }); + } else { + assertEquals(0, collector.collectedNodeCount()); + } + } else { + assertEquals(0, collector.collectedNodeCount()); + } + } else if (low == high) { + collector.matchValues(new long[] { low }); + } else { + assertEquals(0, collector.collectedNodeCount()); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return result; + })); + } + } + } + + private void createStarTreeForDimension( + long[] dimensionValues, + boolean createStarNode, + boolean createNullNode, + List> predicates + ) { + + try (Directory directory = newFSDirectory(createTempDir())) { + + long starTreeDataLength; + + try (IndexOutput dataOut = directory.createOutput("star-tree-data", IOContext.DEFAULT)) { + StarTreeWriter starTreeWriter = new StarTreeWriter(); + int starNodeLengthContribution = 0; + + InMemoryTreeNode rootNode = new InMemoryTreeNode( + 0, + randomInt(), + randomInt(), + randomFrom((byte) 0, (byte) -1, (byte) 1), + -1 + ); + rootNode.setChildDimensionId(1); + rootNode.setAggregatedDocId(randomInt()); + + if (createStarNode && dimensionValues.length > 1) { + InMemoryTreeNode starChild = new InMemoryTreeNode( + rootNode.getDimensionId() + 1, + randomInt(), + randomInt(), + (byte) -1, + -1 + ); + starChild.setChildDimensionId(-1); + starChild.setAggregatedDocId(randomInt()); + rootNode.addChildNode(starChild, (long) ALL); + starNodeLengthContribution++; + } + + for (long dimensionValue : dimensionValues) { + InMemoryTreeNode defaultNode = new InMemoryTreeNode( + rootNode.getDimensionId() + 1, + randomInt(), + randomInt(), + (byte) 0, + dimensionValue + ); + defaultNode.setChildDimensionId(-1); + defaultNode.setAggregatedDocId(randomInt()); + rootNode.addChildNode(defaultNode, dimensionValue); + } + + if (createNullNode) { + InMemoryTreeNode nullNode = new InMemoryTreeNode(rootNode.getDimensionId() + 1, randomInt(), randomInt(), (byte) 1, -1); + nullNode.setChildDimensionId(-1); + nullNode.setAggregatedDocId(randomInt()); + rootNode.addChildNode(nullNode, null); + } + + starTreeDataLength = starTreeWriter.writeStarTree( + dataOut, + rootNode, + starNodeLengthContribution + rootNode.getChildren().size() + 1, + "star-tree" + ); + + // asserting on the actual length of the star tree data file + assertEquals(starTreeDataLength, (33L * rootNode.getChildren().size()) + (starNodeLengthContribution * 33L) + 33L); + } + + for (Predicate predicate : predicates) { + try (IndexInput dataIn = directory.openInput("star-tree-data", IOContext.READONCE)) { + StarTreeMetadata starTreeMetadata = mock(StarTreeMetadata.class); + when(starTreeMetadata.getDataLength()).thenReturn(starTreeDataLength); + when(starTreeMetadata.getDataStartFilePointer()).thenReturn(0L); + FixedLengthStarTreeNode effectiveRoot = (FixedLengthStarTreeNode) StarTreeFactory.createStarTree( + dataIn, + starTreeMetadata + ); + assertTrue(predicate.test(effectiveRoot)); + } + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + + } + + private static class ArrayBasedCollector implements StarTreeNodeCollector { + + private final List nodes = new ArrayList<>(); + + @Override + public void collectStarTreeNode(StarTreeNode node) { + nodes.add(node); + } + + public boolean matchValues(long[] values) throws IOException { + boolean matches = true; + for (int i = 0; i < values.length; i++) { + matches &= nodes.get(i).getDimensionValue() == values[i]; + } + return matches; + } + + public int collectedNodeCount() { + return nodes.size(); + } + + } + +} diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java index febdb17201edd..e3f2f6e5ea27c 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java @@ -75,6 +75,7 @@ import org.opensearch.index.mapper.DateFieldMapper.Resolution; import org.opensearch.index.mapper.MappedFieldType.Relation; import org.opensearch.index.mapper.ParseContext.Document; +import org.opensearch.index.query.BaseQueryRewriteContext; import org.opensearch.index.query.DateRangeIncludingNowQuery; import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; @@ -99,7 +100,7 @@ public class DateFieldTypeTests extends FieldTypeTestCase { private static final long nowInMillis = 0; public void testIsFieldWithinRangeEmptyReader() throws IOException { - QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis); + QueryRewriteContext context = new BaseQueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis); IndexReader reader = new MultiReader(); DateFieldType ft = new DateFieldType("my_date"); assertEquals( @@ -136,7 +137,7 @@ public void isFieldWithinRangeTestCase(DateFieldType ft) throws IOException { doTestIsFieldWithinQuery(ft, reader, DateTimeZone.UTC, null); doTestIsFieldWithinQuery(ft, reader, DateTimeZone.UTC, alternateFormat); - QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis); + QueryRewriteContext context = new BaseQueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis); // Fields with no value indexed. DateFieldType ft2 = new DateFieldType("my_date2"); @@ -148,7 +149,7 @@ public void isFieldWithinRangeTestCase(DateFieldType ft) throws IOException { private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader, DateTimeZone zone, DateMathParser alternateFormat) throws IOException { - QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis); + QueryRewriteContext context = new BaseQueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis); assertEquals( Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", randomBoolean(), randomBoolean(), null, null, context) diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java index 118f58cf5e855..adb2c8b1ffe2a 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java @@ -17,6 +17,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.opensearch.common.TriFunction; +import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.xcontent.ToXContent; @@ -27,6 +28,7 @@ import java.io.IOException; import java.util.List; +import java.util.Set; import static org.opensearch.index.mapper.FlatObjectFieldMapper.CONTENT_TYPE; import static org.opensearch.index.mapper.FlatObjectFieldMapper.VALUE_AND_PATH_SUFFIX; @@ -397,7 +399,17 @@ public void testFetchDocValues() throws IOException { Throwable throwable = assertThrows(IllegalArgumentException.class, () -> ft.docValueFormat(null, null)); assertEquals("Field [field] of type [flat_object] does not support doc_value in root field", throwable.getMessage()); } + } + + public void testPatternMatch() throws IOException { + MapperService mapperService = createMapperService( + fieldMapping(b -> b.startObject("properties").startObject("foo").field("type", "flat_object").endObject().endObject()) + ); + QueryShardContext queryShardContext = createQueryShardContext(mapperService); + Set fields = queryShardContext.simpleMatchToIndexNames("field.*"); + assertEquals(1, fields.size()); + assertEquals(Sets.newHashSet("field.foo"), fields); } @Override diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java index 600f04edcd552..2ab1ad689a0b9 100644 --- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldTypeTests.java @@ -418,8 +418,12 @@ public void testExistsQuery() { ft.getValueFieldType(), ft.getValueAndPathFieldType() ); - assertEquals(new TermQuery(new Term("field", "field.bar")), dynamicMappedFieldType.existsQuery(null)); - + Automaton termAutomaton = PrefixQuery.toAutomaton(new BytesRef("field.bar=")); + Automaton dvAutomaton = PrefixQuery.toAutomaton(new BytesRef("field.field.bar=")); + Query indexQuery = new AutomatonQuery(new Term("field" + VALUE_AND_PATH_SUFFIX), termAutomaton, true); + Query dvQuery = new AutomatonQuery(new Term("field" + VALUE_AND_PATH_SUFFIX), dvAutomaton, true, DOC_VALUES_REWRITE); + Query expected = new IndexOrDocValuesQuery(indexQuery, dvQuery); + assertEquals(expected, dynamicMappedFieldType.existsQuery(MOCK_QSC_ENABLE_INDEX_DOC_VALUES)); } { @@ -1176,8 +1180,8 @@ public void testRangeQuery() { ); continue; } - boolean nullLowerTerm = true;// randomBoolean(); - boolean nullUpperTerm = true;// nullLowerTerm == false || randomBoolean(); + boolean nullLowerTerm = randomBoolean(); + boolean nullUpperTerm = nullLowerTerm == false || randomBoolean(); Automaton a1 = PrefixQuery.toAutomaton(new BytesRef("field.field1=")); Automaton a2 = TermRangeQuery.toAutomaton( diff --git a/server/src/test/java/org/opensearch/index/mapper/StoredNumericValuesTests.java b/server/src/test/java/org/opensearch/index/mapper/StoredNumericValuesTests.java index 07b64a9cdc58f..64b8719191718 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StoredNumericValuesTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StoredNumericValuesTests.java @@ -41,9 +41,11 @@ import org.opensearch.common.lucene.Lucene; import org.opensearch.common.util.set.Sets; import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.Strings; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.index.fieldvisitor.CustomFieldsVisitor; +import org.opensearch.index.fieldvisitor.FieldsVisitor; import org.opensearch.index.mapper.MapperService.MergeReason; import org.opensearch.test.OpenSearchSingleNodeTestCase; @@ -200,4 +202,36 @@ public void testBytesAndNumericRepresentation() throws Exception { reader.close(); writer.close(); } + + public void testFieldsVisitorValidateIncludesExcludes() throws Exception { + Set fieldNames = Sets.newHashSet( + "field1", + "field2", + "field3", + "field4", + "field5", + "field6", + "field7", + "field8", + "field9", + "field10", + "field11" + ); + String[] includes = { "field1", "field2", "field3" }; + String[] excludes = { "field7", "field8" }; + + CustomFieldsVisitor fieldsVisitor = new CustomFieldsVisitor(fieldNames, false, includes, excludes); + + assertArrayEquals(fieldsVisitor.includes(), includes); + assertArrayEquals(fieldsVisitor.excludes(), excludes); + + FieldsVisitor fieldsVisitor1 = new FieldsVisitor(false, includes, excludes); + assertArrayEquals(fieldsVisitor1.includes(), includes); + assertArrayEquals(fieldsVisitor1.excludes(), excludes); + + FieldsVisitor fieldsVisitor2 = new FieldsVisitor(false); + assertArrayEquals(fieldsVisitor2.includes(), Strings.EMPTY_ARRAY); + assertArrayEquals(fieldsVisitor2.excludes(), Strings.EMPTY_ARRAY); + + } } diff --git a/server/src/test/java/org/opensearch/index/query/RewriteableTests.java b/server/src/test/java/org/opensearch/index/query/RewriteableTests.java index 6385a57f9f370..6e58023ecc7e2 100644 --- a/server/src/test/java/org/opensearch/index/query/RewriteableTests.java +++ b/server/src/test/java/org/opensearch/index/query/RewriteableTests.java @@ -45,7 +45,7 @@ public class RewriteableTests extends OpenSearchTestCase { public void testRewrite() throws IOException { - QueryRewriteContext context = new QueryRewriteContext(null, null, null, null); + QueryRewriteContext context = new BaseQueryRewriteContext(null, null, null, null); TestRewriteable rewrite = Rewriteable.rewrite( new TestRewriteable(randomIntBetween(0, Rewriteable.MAX_REWRITE_ROUNDS)), context, @@ -65,7 +65,7 @@ public void testRewrite() throws IOException { } public void testRewriteAndFetch() throws ExecutionException, InterruptedException { - QueryRewriteContext context = new QueryRewriteContext(null, null, null, null); + BaseQueryRewriteContext context = new BaseQueryRewriteContext(null, null, null, null); PlainActionFuture future = new PlainActionFuture<>(); Rewriteable.rewriteAndFetch(new TestRewriteable(randomIntBetween(0, Rewriteable.MAX_REWRITE_ROUNDS), true), context, future); TestRewriteable rewrite = future.get(); @@ -83,7 +83,7 @@ public void testRewriteAndFetch() throws ExecutionException, InterruptedExceptio } public void testRewriteList() throws IOException { - QueryRewriteContext context = new QueryRewriteContext(null, null, null, null); + BaseQueryRewriteContext context = new BaseQueryRewriteContext(null, null, null, null); List rewriteableList = new ArrayList<>(); int numInstances = randomIntBetween(1, 10); rewriteableList.add(new TestRewriteable(randomIntBetween(1, Rewriteable.MAX_REWRITE_ROUNDS))); diff --git a/server/src/test/java/org/opensearch/index/query/TemplateQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/TemplateQueryBuilderTests.java new file mode 100644 index 0000000000000..4ea01818ca32e --- /dev/null +++ b/server/src/test/java/org/opensearch/index/query/TemplateQueryBuilderTests.java @@ -0,0 +1,834 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.query; + +import org.opensearch.client.Client; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.search.SearchModule; +import org.opensearch.search.builder.SearchSourceBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.function.BiConsumer; + +import static org.opensearch.index.query.TemplateQueryBuilder.NAME; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TemplateQueryBuilderTests extends OpenSearchTestCase { + + /** + * Tests the fromXContent method of TemplateQueryBuilder. + * Verifies that a TemplateQueryBuilder can be correctly created from XContent. + */ + public void testFromXContent() throws IOException { + /* + { + "template": { + "term": { + "message": { + "value": "foo" + } + } + } + } + */ + Map template = new HashMap<>(); + Map term = new HashMap<>(); + Map message = new HashMap<>(); + + message.put("value", "foo"); + term.put("message", message); + template.put("term", term); + + XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().map(template); + + XContentParser contentParser = createParser(xContentBuilder); + contentParser.nextToken(); + TemplateQueryBuilder templateQueryBuilder = TemplateQueryBuilder.fromXContent(contentParser); + + assertEquals(NAME, templateQueryBuilder.getWriteableName()); + assertEquals(template, templateQueryBuilder.getContent()); + + SearchSourceBuilder source = new SearchSourceBuilder().query(templateQueryBuilder); + assertEquals(source.toString(), "{\"query\":{\"template\":{\"term\":{\"message\":{\"value\":\"foo\"}}}}}"); + } + + /** + * Tests the query source generation of TemplateQueryBuilder. + * Verifies that the correct query source is generated from a TemplateQueryBuilder. + */ + public void testQuerySource() { + + Map template = new HashMap<>(); + Map term = new HashMap<>(); + Map message = new HashMap<>(); + + message.put("value", "foo"); + term.put("message", message); + template.put("term", term); + QueryBuilder incomingQuery = new TemplateQueryBuilder(template); + SearchSourceBuilder source = new SearchSourceBuilder().query(incomingQuery); + assertEquals(source.toString(), "{\"query\":{\"template\":{\"term\":{\"message\":{\"value\":\"foo\"}}}}}"); + } + + /** + * Tests parsing a TemplateQueryBuilder from a JSON string. + * Verifies that the parsed query matches the expected structure and can be serialized and deserialized. + */ + public void testFromJson() throws IOException { + String jsonString = "{\n" + + " \"geo_shape\": {\n" + + " \"location\": {\n" + + " \"shape\": {\n" + + " \"type\": \"Envelope\",\n" + + " \"coordinates\": \"${modelPredictionOutcome}\"\n" + + " },\n" + + " \"relation\": \"intersects\"\n" + + " },\n" + + " \"ignore_unmapped\": false,\n" + + " \"boost\": 42.0\n" + + " }\n" + + "}"; + + XContentParser parser = XContentType.JSON.xContent() + .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, jsonString); + parser.nextToken(); + TemplateQueryBuilder parsed = TemplateQueryBuilder.fromXContent(parser); + + // Check if the parsed query is an instance of TemplateQueryBuilder + assertNotNull(parsed); + assertTrue(parsed instanceof TemplateQueryBuilder); + + // Check if the content of the parsed query matches the expected content + Map expectedContent = new HashMap<>(); + Map geoShape = new HashMap<>(); + Map location = new HashMap<>(); + Map shape = new HashMap<>(); + + shape.put("type", "Envelope"); + shape.put("coordinates", "${modelPredictionOutcome}"); + location.put("shape", shape); + location.put("relation", "intersects"); + geoShape.put("location", location); + geoShape.put("ignore_unmapped", false); + geoShape.put("boost", 42.0); + expectedContent.put("geo_shape", geoShape); + + Map actualContent = new HashMap<>(); + actualContent.put("template", expectedContent); + assertEquals(expectedContent, parsed.getContent()); + + // Test that the query can be serialized and deserialized + BytesStreamOutput out = new BytesStreamOutput(); + parsed.writeTo(out); + StreamInput in = out.bytes().streamInput(); + TemplateQueryBuilder deserializedQuery = new TemplateQueryBuilder(in); + assertEquals(parsed.getContent(), deserializedQuery.getContent()); + + // Test that the query can be converted to XContent + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + parsed.doXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + Map expectedJson = new HashMap<>(); + Map template = new HashMap<>(); + template.put("geo_shape", geoShape); + expectedJson.put("template", template); + + XContentParser jsonParser = XContentType.JSON.xContent() + .createParser(xContentRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, builder.toString()); + Map actualJson = jsonParser.map(); + + assertEquals(expectedJson, actualJson); + } + + /** + * Tests the constructor and getter methods of TemplateQueryBuilder. + * Verifies that the content and writeable name are correctly set and retrieved. + */ + public void testConstructorAndGetters() { + Map content = new HashMap<>(); + content.put("key", "value"); + TemplateQueryBuilder builder = new TemplateQueryBuilder(content); + + assertEquals(content, builder.getContent()); + assertEquals(NAME, builder.getWriteableName()); + } + + /** + * Tests the equals and hashCode methods of TemplateQueryBuilder. + * Verifies that two builders with the same content are equal and have the same hash code, + * while builders with different content are not equal and have different hash codes. + */ + public void testEqualsAndHashCode() { + Map content1 = new HashMap<>(); + content1.put("key", "value"); + TemplateQueryBuilder builder1 = new TemplateQueryBuilder(content1); + + Map content2 = new HashMap<>(); + content2.put("key", "value"); + TemplateQueryBuilder builder2 = new TemplateQueryBuilder(content2); + + Map content3 = new HashMap<>(); + content3.put("key", "different_value"); + TemplateQueryBuilder builder3 = new TemplateQueryBuilder(content3); + + assertTrue(builder1.equals(builder2)); + assertTrue(builder1.hashCode() == builder2.hashCode()); + assertFalse(builder1.equals(builder3)); + assertFalse(builder1.hashCode() == builder3.hashCode()); + } + + /** + * Tests the doToQuery method of TemplateQueryBuilder. + * Verifies that calling doToQuery throws an IllegalStateException. + */ + public void testDoToQuery() { + Map content = new HashMap<>(); + content.put("key", "value"); + TemplateQueryBuilder builder = new TemplateQueryBuilder(content); + + QueryShardContext mockContext = mock(QueryShardContext.class); + expectThrows(IllegalStateException.class, () -> builder.doToQuery(mockContext)); + } + + /** + * Tests the serialization and deserialization of TemplateQueryBuilder. + * Verifies that a builder can be written to a stream and read back correctly. + */ + public void testStreamRoundTrip() throws IOException { + Map content = new HashMap<>(); + content.put("key", "value"); + TemplateQueryBuilder original = new TemplateQueryBuilder(content); + + BytesStreamOutput out = new BytesStreamOutput(); + original.writeTo(out); + + StreamInput in = out.bytes().streamInput(); + TemplateQueryBuilder deserialized = new TemplateQueryBuilder(in); + + assertEquals(original, deserialized); + } + + /** + * Tests the doRewrite method of TemplateQueryBuilder with a simple term query. + * Verifies that the template is correctly rewritten to a TermQueryBuilder. + */ + public void testDoRewrite() throws IOException { + + Map template = new HashMap<>(); + Map term = new HashMap<>(); + Map message = new HashMap<>(); + + message.put("value", "foo"); + term.put("message", message); + template.put("term", term); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + TermQueryBuilder termQueryBuilder = new TermQueryBuilder("message", "foo"); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + + Map contextVariables = new HashMap<>(); + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + + TermQueryBuilder newQuery = (TermQueryBuilder) templateQueryBuilder.doRewrite(queryRewriteContext); + + assertEquals(newQuery, termQueryBuilder); + assertEquals( + "{\n" + + " \"term\" : {\n" + + " \"message\" : {\n" + + " \"value\" : \"foo\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + newQuery.toString() + ); + } + + /** + * Tests the doRewrite method of TemplateQueryBuilder with a string variable. + * Verifies that the template is correctly rewritten with the variable substituted. + */ + public void testDoRewriteWithString() throws IOException { + + Map template = new HashMap<>(); + Map term = new HashMap<>(); + Map message = new HashMap<>(); + + message.put("value", "${response}"); + term.put("message", message); + template.put("term", term); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + TermQueryBuilder termQueryBuilder = new TermQueryBuilder("message", "foo"); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + + Map contextVariables = new HashMap<>(); + contextVariables.put("response", "foo"); + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + + TermQueryBuilder newQuery = (TermQueryBuilder) templateQueryBuilder.doRewrite(queryRewriteContext); + + assertEquals(newQuery, termQueryBuilder); + assertEquals( + "{\n" + + " \"term\" : {\n" + + " \"message\" : {\n" + + " \"value\" : \"foo\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + newQuery.toString() + ); + } + + /** + * Tests the doRewrite method of TemplateQueryBuilder with a list variable. + * Verifies that the template is correctly rewritten with the list variable substituted. + */ + public void testDoRewriteWithList() throws IOException { + ArrayList termsList = new ArrayList<>(); + termsList.add("foo"); + termsList.add("bar"); + + Map template = new HashMap<>(); + Map terms = new HashMap<>(); + + terms.put("message", "${response}"); + template.put("terms", terms); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + TermsQueryBuilder termsQueryBuilder = new TermsQueryBuilder("message", termsList); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + + Map contextVariables = new HashMap<>(); + contextVariables.put("response", termsList); + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + NamedXContentRegistry TEST_XCONTENT_REGISTRY_FOR_QUERY = new NamedXContentRegistry( + new SearchModule(Settings.EMPTY, List.of()).getNamedXContents() + ); + when(queryRewriteContext.getXContentRegistry()).thenReturn(TEST_XCONTENT_REGISTRY_FOR_QUERY); + TermsQueryBuilder newQuery = (TermsQueryBuilder) templateQueryBuilder.doRewrite(queryRewriteContext); + assertEquals(newQuery, termsQueryBuilder); + assertEquals( + "{\n" + + " \"terms\" : {\n" + + " \"message\" : [\n" + + " \"foo\",\n" + + " \"bar\"\n" + + " ],\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}", + newQuery.toString() + ); + } + + /** + * Tests the doRewrite method of TemplateQueryBuilder with a geo_distance query. + * Verifies that the template is correctly rewritten for a geo_distance query. + */ + public void testDoRewriteWithGeoDistanceQuery() throws IOException { + Map template = new HashMap<>(); + Map geoDistance = new HashMap<>(); + + geoDistance.put("distance", "12km"); + geoDistance.put("pin.location", "${geoPoint}"); + template.put("geo_distance", geoDistance); + + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + GeoPoint geoPoint = new GeoPoint(40, -70); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + Map contextVariables = new HashMap<>(); + contextVariables.put("geoPoint", geoPoint); + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + + GeoDistanceQueryBuilder expectedQuery = new GeoDistanceQueryBuilder("pin.location"); + expectedQuery.point(geoPoint).distance("12km"); + + QueryBuilder newQuery = templateQueryBuilder.doRewrite(queryRewriteContext); + assertEquals(expectedQuery, newQuery); + assertEquals( + "{\n" + + " \"geo_distance\" : {\n" + + " \"pin.location\" : [\n" + + " -70.0,\n" + + " 40.0\n" + + " ],\n" + + " \"distance\" : 12000.0,\n" + + " \"distance_type\" : \"arc\",\n" + + " \"validation_method\" : \"STRICT\",\n" + + " \"ignore_unmapped\" : false,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}", + newQuery.toString() + ); + } + + /** + * Tests the doRewrite method of TemplateQueryBuilder with a range query. + * Verifies that the template is correctly rewritten for a range query. + */ + public void testDoRewriteWithRangeQuery() throws IOException { + Map template = new HashMap<>(); + Map range = new HashMap<>(); + Map age = new HashMap<>(); + + age.put("gte", "${minAge}"); + age.put("lte", "${maxAge}"); + range.put("age", age); + template.put("range", range); + + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + Map contextVariables = new HashMap<>(); + contextVariables.put("minAge", 25); + contextVariables.put("maxAge", 35); + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + + RangeQueryBuilder expectedQuery = new RangeQueryBuilder("age"); + expectedQuery.gte(25).lte(35); + + QueryBuilder newQuery = templateQueryBuilder.doRewrite(queryRewriteContext); + assertEquals(expectedQuery, newQuery); + assertEquals( + "{\n" + + " \"range\" : {\n" + + " \"age\" : {\n" + + " \"from\" : 25,\n" + + " \"to\" : 35,\n" + + " \"include_lower\" : true,\n" + + " \"include_upper\" : true,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + newQuery.toString() + ); + } + + /** + * Tests the doRewrite method of TemplateQueryBuilder with a nested map variable. + * Verifies that the template is correctly rewritten with the nested map variable substituted. + */ + public void testDoRewriteWithNestedMap() throws IOException { + Map template = new HashMap<>(); + Map bool = new HashMap<>(); + List> must = new ArrayList<>(); + Map match = new HashMap<>(); + Map textEntry = new HashMap<>(); + + textEntry.put("text_entry", "${keyword}"); + match.put("match", textEntry); + must.add(match); + bool.put("must", must); + + List> should = new ArrayList<>(); + Map shouldMatch1 = new HashMap<>(); + Map shouldTextEntry1 = new HashMap<>(); + shouldTextEntry1.put("text_entry", "life"); + shouldMatch1.put("match", shouldTextEntry1); + should.add(shouldMatch1); + + Map shouldMatch2 = new HashMap<>(); + Map shouldTextEntry2 = new HashMap<>(); + shouldTextEntry2.put("text_entry", "grace"); + shouldMatch2.put("match", shouldTextEntry2); + should.add(shouldMatch2); + + bool.put("should", should); + bool.put("minimum_should_match", 1); + + Map filter = new HashMap<>(); + Map term = new HashMap<>(); + term.put("play_name", "Romeo and Juliet"); + filter.put("term", term); + bool.put("filter", filter); + + template.put("bool", bool); + + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + Map contextVariables = new HashMap<>(); + contextVariables.put("keyword", "love"); + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + + BoolQueryBuilder expectedQuery = new BoolQueryBuilder().must(new MatchQueryBuilder("text_entry", "love")) + .should(new MatchQueryBuilder("text_entry", "life")) + .should(new MatchQueryBuilder("text_entry", "grace")) + .filter(new TermQueryBuilder("play_name", "Romeo and Juliet")) + .minimumShouldMatch(1); + + QueryBuilder newQuery = templateQueryBuilder.doRewrite(queryRewriteContext); + assertEquals(expectedQuery, newQuery); + assertEquals( + "{\n" + + " \"bool\" : {\n" + + " \"must\" : [\n" + + " {\n" + + " \"match\" : {\n" + + " \"text_entry\" : {\n" + + " \"query\" : \"love\",\n" + + " \"operator\" : \"OR\",\n" + + " \"prefix_length\" : 0,\n" + + " \"max_expansions\" : 50,\n" + + " \"fuzzy_transpositions\" : true,\n" + + " \"lenient\" : false,\n" + + " \"zero_terms_query\" : \"NONE\",\n" + + " \"auto_generate_synonyms_phrase_query\" : true,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"filter\" : [\n" + + " {\n" + + " \"term\" : {\n" + + " \"play_name\" : {\n" + + " \"value\" : \"Romeo and Juliet\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"should\" : [\n" + + " {\n" + + " \"match\" : {\n" + + " \"text_entry\" : {\n" + + " \"query\" : \"life\",\n" + + " \"operator\" : \"OR\",\n" + + " \"prefix_length\" : 0,\n" + + " \"max_expansions\" : 50,\n" + + " \"fuzzy_transpositions\" : true,\n" + + " \"lenient\" : false,\n" + + " \"zero_terms_query\" : \"NONE\",\n" + + " \"auto_generate_synonyms_phrase_query\" : true,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " },\n" + + " {\n" + + " \"match\" : {\n" + + " \"text_entry\" : {\n" + + " \"query\" : \"grace\",\n" + + " \"operator\" : \"OR\",\n" + + " \"prefix_length\" : 0,\n" + + " \"max_expansions\" : 50,\n" + + " \"fuzzy_transpositions\" : true,\n" + + " \"lenient\" : false,\n" + + " \"zero_terms_query\" : \"NONE\",\n" + + " \"auto_generate_synonyms_phrase_query\" : true,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}", + newQuery.toString() + ); + } + + /** + * Tests the doRewrite method with an invalid query type. + * Verifies that an IOException is thrown when an invalid query type is used. + */ + public void testDoRewriteWithInvalidQueryType() throws IOException { + Map template = new HashMap<>(); + template.put("invalid_query_type", new HashMap<>()); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + when(queryRewriteContext.getContextVariables()).thenReturn(new HashMap<>()); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> templateQueryBuilder.doRewrite(queryRewriteContext) + ); + assertTrue(exception.getMessage().contains("Failed to rewrite template query")); + } + + /** + * Tests the doRewrite method with a malformed JSON query. + * Verifies that an IOException is thrown when the query JSON is malformed. + */ + public void testDoRewriteWithMalformedJson() throws IOException { + Map template = new HashMap<>(); + template.put("malformed_json", "{ this is not valid JSON }"); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + when(queryRewriteContext.getContextVariables()).thenReturn(new HashMap<>()); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> templateQueryBuilder.doRewrite(queryRewriteContext) + ); + assertTrue(exception.getMessage().contains("Failed to rewrite template query")); + } + + /** + * Tests the doRewrite method with an invalid matchall query. + * Verifies that an IOException is thrown when an invalid matchall query is used. + */ + public void testDoRewriteWithInvalidMatchAllQuery() throws IOException { + Map template = new HashMap<>(); + template.put("matchall_1", new HashMap<>()); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + when(queryRewriteContext.getContextVariables()).thenReturn(new HashMap<>()); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> templateQueryBuilder.doRewrite(queryRewriteContext) + ); + assertTrue(exception.getMessage().contains("Failed to rewrite template query")); + } + + /** + * Tests the doRewrite method with a missing required field in a query. + * Verifies that an IOException is thrown when a required field is missing. + */ + public void testDoRewriteWithMissingRequiredField() throws IOException { + Map template = new HashMap<>(); + template.put("term", "value");// Missing the required field for term query + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + when(queryRewriteContext.getContextVariables()).thenReturn(new HashMap<>()); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> templateQueryBuilder.doRewrite(queryRewriteContext) + ); + assertTrue(exception.getMessage().contains("Failed to rewrite template query")); + } + + /** + * Tests the doRewrite method with a malformed variable substitution. + * Verifies that an IOException is thrown when a malformed variable is used. + */ + public void testDoRewriteWithMalformedVariableSubstitution() throws IOException { + + Map template = new HashMap<>(); + Map terms = new HashMap<>(); + + terms.put("message", "${response}"); + template.put("terms", terms); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + + Map contextVariables = new HashMap<>(); + contextVariables.put("response", "should be a list but this is a string"); + + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> templateQueryBuilder.doRewrite(queryRewriteContext) + ); + + assertTrue(exception.getMessage().contains("Failed to rewrite template query")); + } + + /** + * Tests the doRewrite method with a variable not found. + * Verifies that an IOException is thrown when a malformed variable is used. + */ + public void testDoRewriteWithNotFoundVariableSubstitution() throws IOException { + + Map template = new HashMap<>(); + Map term = new HashMap<>(); + Map message = new HashMap<>(); + + message.put("value", "${response}"); + term.put("message", message); + template.put("term", term); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + + Map contextVariables = new HashMap<>(); + contextVariables.put("response1", "foo"); + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> templateQueryBuilder.doRewrite(queryRewriteContext) + ); + assertTrue(exception.getMessage().contains("Variable not found")); + } + + /** + * Tests the doRewrite method of TemplateQueryBuilder with a missing bracket variable. + * Verifies that the exception is thrown + */ + public void testDoRewriteWithMissingBracketVariable() throws IOException { + + Map template = new HashMap<>(); + Map term = new HashMap<>(); + Map message = new HashMap<>(); + + message.put("value", "${response"); + term.put("message", message); + template.put("term", term); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + + Map contextVariables = new HashMap<>(); + contextVariables.put("response", "foo"); + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> templateQueryBuilder.doRewrite(queryRewriteContext) + ); + assertTrue(exception.getMessage().contains("Unclosed variable in template")); + } + + /** + * Tests the replaceVariables method when the template is null. + * Verifies that an IllegalArgumentException is thrown with the appropriate error message. + */ + + public void testReplaceVariablesWithNullTemplate() { + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder((Map) null); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + Map contextVariables = new HashMap<>(); + contextVariables.put("response", "foo"); + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> templateQueryBuilder.doRewrite(queryRewriteContext) + ); + assertEquals("Template string cannot be null. A valid template must be provided.", exception.getMessage()); + } + + /** + * Tests the replaceVariables method when the template is empty. + * Verifies that an IllegalArgumentException is thrown with the appropriate error message. + */ + + public void testReplaceVariablesWithEmptyTemplate() { + Map template = new HashMap<>(); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + Map contextVariables = new HashMap<>(); + contextVariables.put("response", "foo"); + when(queryRewriteContext.getContextVariables()).thenReturn(contextVariables); + + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> templateQueryBuilder.doRewrite(queryRewriteContext) + ); + assertEquals("Template string cannot be empty. A valid template must be provided.", exception.getMessage()); + + } + + /** + * Tests the replaceVariables method when the variables map is null. + * Verifies that the method returns the original template unchanged, + * since a null variables map is treated as no replacement. + */ + public void testReplaceVariablesWithNullVariables() throws IOException { + + Map template = new HashMap<>(); + Map term = new HashMap<>(); + Map message = new HashMap<>(); + + message.put("value", "foo"); + term.put("message", message); + template.put("term", term); + TemplateQueryBuilder templateQueryBuilder = new TemplateQueryBuilder(template); + TermQueryBuilder termQueryBuilder = new TermQueryBuilder("message", "foo"); + + QueryCoordinatorContext queryRewriteContext = mockQueryRewriteContext(); + + when(queryRewriteContext.getContextVariables()).thenReturn(null); + + TermQueryBuilder newQuery = (TermQueryBuilder) templateQueryBuilder.doRewrite(queryRewriteContext); + + assertEquals(newQuery, termQueryBuilder); + assertEquals( + "{\n" + + " \"term\" : {\n" + + " \"message\" : {\n" + + " \"value\" : \"foo\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + newQuery.toString() + ); + } + + /** + * Helper method to create a mock QueryCoordinatorContext for testing. + */ + private QueryCoordinatorContext mockQueryRewriteContext() { + QueryCoordinatorContext queryRewriteContext = mock(QueryCoordinatorContext.class); + final CountDownLatch inProgressLatch = new CountDownLatch(1); + doAnswer(invocation -> { + BiConsumer> biConsumer = invocation.getArgument(0); + biConsumer.accept( + null, + ActionListener.wrap( + response -> inProgressLatch.countDown(), + err -> fail("Failed to set query tokens supplier: " + err.getMessage()) + ) + ); + return null; + }).when(queryRewriteContext).registerAsyncAction(any()); + + NamedXContentRegistry TEST_XCONTENT_REGISTRY_FOR_QUERY = new NamedXContentRegistry( + new SearchModule(Settings.EMPTY, List.of()).getNamedXContents() + ); + when(queryRewriteContext.getXContentRegistry()).thenReturn(TEST_XCONTENT_REGISTRY_FOR_QUERY); + + return queryRewriteContext; + } +} diff --git a/server/src/test/java/org/opensearch/plugins/PluginsServiceTests.java b/server/src/test/java/org/opensearch/plugins/PluginsServiceTests.java index f5702fa1a7ade..cb549eafc0d21 100644 --- a/server/src/test/java/org/opensearch/plugins/PluginsServiceTests.java +++ b/server/src/test/java/org/opensearch/plugins/PluginsServiceTests.java @@ -38,7 +38,7 @@ import org.apache.lucene.util.Constants; import org.opensearch.LegacyESVersion; import org.opensearch.Version; -import org.opensearch.bootstrap.JarHell; +import org.opensearch.common.bootstrap.JarHell; import org.opensearch.common.collect.Tuple; import org.opensearch.common.io.PathUtils; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index f2b06b0926b81..03a5ba8599f77 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -34,7 +34,7 @@ import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse; import org.opensearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.RepositoryMetadata; import org.opensearch.common.settings.Settings; diff --git a/server/src/test/java/org/opensearch/search/SearchModuleTests.java b/server/src/test/java/org/opensearch/search/SearchModuleTests.java index 9e8e7afe332f1..d78393e917b2f 100644 --- a/server/src/test/java/org/opensearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/opensearch/search/SearchModuleTests.java @@ -608,7 +608,8 @@ public Optional create(IndexSettings indexSettin "terms_set", "wildcard", "wrapper", - "distance_feature" }; + "distance_feature", + "template" }; // add here deprecated queries to make sure we log a deprecation warnings when they are used private static final String[] DEPRECATED_QUERIES = new String[] { "common", "field_masking_span" }; diff --git a/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java b/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java index 3b32e9e4ac6b7..83478e74a3b67 100644 --- a/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java +++ b/server/src/test/java/org/opensearch/search/SearchServiceStarTreeTests.java @@ -16,16 +16,27 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.index.IndexService; -import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.compositeindex.CompositeIndexSettings; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.OrdinalDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; -import org.opensearch.index.mapper.CompositeMappedFieldType; +import org.opensearch.index.mapper.CompositeDataCubeFieldType; +import org.opensearch.index.mapper.StarTreeMapper; import org.opensearch.index.query.MatchAllQueryBuilder; +import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.index.shard.IndexShard; import org.opensearch.indices.IndicesService; import org.opensearch.search.aggregations.AggregationBuilders; -import org.opensearch.search.aggregations.startree.StarTreeFilterTests; +import org.opensearch.search.aggregations.AggregatorFactories; +import org.opensearch.search.aggregations.AggregatorFactory; +import org.opensearch.search.aggregations.SearchContextAggregations; +import org.opensearch.search.aggregations.startree.StarTreeQueryTests; +import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.internal.AliasFilter; import org.opensearch.search.internal.ReaderContext; @@ -35,10 +46,13 @@ import org.opensearch.test.OpenSearchSingleNodeTestCase; import java.io.IOException; -import java.util.Map; +import java.util.Collections; +import java.util.List; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class SearchServiceStarTreeTests extends OpenSearchSingleNodeTestCase { @@ -55,7 +69,9 @@ public void testParseQueryToOriginalOrStarTreeQuery() throws IOException { .indices() .prepareCreate("test") .setSettings(settings) - .setMapping(StarTreeFilterTests.getExpandedMapping(1, false)); + .setMapping( + StarTreeQueryTests.getExpandedMapping(1, false, StarTreeQueryTests.DIMENSION_TYPE_MAP, StarTreeQueryTests.METRIC_TYPE_MAP) + ); createIndex("test", builder); IndicesService indicesService = getInstanceFromNode(IndicesService.class); @@ -73,6 +89,14 @@ public void testParseQueryToOriginalOrStarTreeQuery() throws IOException { null ); + QueryBuilder baseQuery; + SearchContext searchContext = createSearchContext(indexService); + StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration( + 1, + Collections.emptySet(), + StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP + ); + // Case 1: No query or aggregations, should not use star tree SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); assertStarTreeContext(request, sourceBuilder, null, -1); @@ -82,15 +106,24 @@ public void testParseQueryToOriginalOrStarTreeQuery() throws IOException { assertStarTreeContext(request, sourceBuilder, null, -1); // Case 3: MatchAllQuery and aggregations present, should use star tree - sourceBuilder = new SearchSourceBuilder().size(0) - .query(new MatchAllQueryBuilder()) - .aggregation(AggregationBuilders.max("test").field("field")); - CompositeIndexFieldInfo expectedStarTree = new CompositeIndexFieldInfo( - "startree", - CompositeMappedFieldType.CompositeFieldType.STAR_TREE + baseQuery = new MatchAllQueryBuilder(); + sourceBuilder = new SearchSourceBuilder().size(0).query(baseQuery).aggregation(AggregationBuilders.max("test").field("field")); + assertStarTreeContext( + request, + sourceBuilder, + getStarTreeQueryContext( + searchContext, + starTreeFieldConfiguration, + "startree", + -1, + List.of(), + List.of(new Metric("field", List.of(MetricStat.MAX))), + baseQuery, + sourceBuilder, + true + ), + -1 ); - Map expectedQueryMap = null; - assertStarTreeContext(request, sourceBuilder, new StarTreeQueryContext(expectedStarTree, expectedQueryMap, -1), -1); // Case 4: MatchAllQuery and aggregations present, but postFilter specified, should not use star tree sourceBuilder = new SearchSourceBuilder().size(0) @@ -100,25 +133,69 @@ public void testParseQueryToOriginalOrStarTreeQuery() throws IOException { assertStarTreeContext(request, sourceBuilder, null, -1); // Case 5: TermQuery and single aggregation, should use star tree, but not initialize query cache - sourceBuilder = new SearchSourceBuilder().size(0) - .query(new TermQueryBuilder("sndv", 1)) - .aggregation(AggregationBuilders.max("test").field("field")); - expectedQueryMap = Map.of("sndv", 1L); - assertStarTreeContext(request, sourceBuilder, new StarTreeQueryContext(expectedStarTree, expectedQueryMap, -1), -1); + baseQuery = new TermQueryBuilder("sndv", 1); + sourceBuilder = new SearchSourceBuilder().size(0).query(baseQuery).aggregation(AggregationBuilders.max("test").field("field")); + assertStarTreeContext( + request, + sourceBuilder, + getStarTreeQueryContext( + searchContext, + starTreeFieldConfiguration, + "startree", + -1, + List.of(new OrdinalDimension("sndv")), + List.of(new Metric("field", List.of(MetricStat.MAX))), + baseQuery, + sourceBuilder, + true + ), + -1 + ); // Case 6: TermQuery and multiple aggregations present, should use star tree & initialize cache + baseQuery = new TermQueryBuilder("sndv", 1); sourceBuilder = new SearchSourceBuilder().size(0) - .query(new TermQueryBuilder("sndv", 1)) + .query(baseQuery) .aggregation(AggregationBuilders.max("test").field("field")) .aggregation(AggregationBuilders.sum("test2").field("field")); - expectedQueryMap = Map.of("sndv", 1L); - assertStarTreeContext(request, sourceBuilder, new StarTreeQueryContext(expectedStarTree, expectedQueryMap, 0), 0); + assertStarTreeContext( + request, + sourceBuilder, + getStarTreeQueryContext( + searchContext, + starTreeFieldConfiguration, + "startree", + -1, + List.of(new OrdinalDimension("sndv")), + List.of(new Metric("field", List.of(MetricStat.MAX, MetricStat.SUM))), + baseQuery, + sourceBuilder, + true + ), + 0 + ); // Case 7: No query, metric aggregations present, should use star tree sourceBuilder = new SearchSourceBuilder().size(0).aggregation(AggregationBuilders.max("test").field("field")); - assertStarTreeContext(request, sourceBuilder, new StarTreeQueryContext(expectedStarTree, null, -1), -1); + assertStarTreeContext( + request, + sourceBuilder, + getStarTreeQueryContext( + searchContext, + starTreeFieldConfiguration, + "startree", + -1, + List.of(new OrdinalDimension("sndv")), + List.of(new Metric("field", List.of(MetricStat.MAX))), + null, + sourceBuilder, + true + ), + -1 + ); setStarTreeIndexSetting(null); + searchContext.close(); } private void setStarTreeIndexSetting(String value) throws IOException { @@ -139,15 +216,15 @@ private void assertStarTreeContext( SearchService searchService = getInstanceFromNode(SearchService.class); try (ReaderContext reader = searchService.createOrGetReaderContext(request, false)) { SearchContext context = searchService.createContext(reader, request, null, true); - StarTreeQueryContext actualContext = context.getStarTreeQueryContext(); + StarTreeQueryContext actualContext = context.getQueryShardContext().getStarTreeQueryContext(); if (expectedContext == null) { - assertThat(context.getStarTreeQueryContext(), nullValue()); + assertThat(context.getQueryShardContext().getStarTreeQueryContext(), nullValue()); } else { assertThat(actualContext, notNullValue()); assertEquals(expectedContext.getStarTree().getType(), actualContext.getStarTree().getType()); assertEquals(expectedContext.getStarTree().getField(), actualContext.getStarTree().getField()); - assertEquals(expectedContext.getQueryMap(), actualContext.getQueryMap()); + assertEquals(expectedContext.getBaseQueryStarTreeFilter(), actualContext.getBaseQueryStarTreeFilter()); if (expectedCacheUsage > -1) { assertEquals(expectedCacheUsage, actualContext.getStarTreeValues().length); } else { @@ -157,4 +234,39 @@ private void assertStarTreeContext( searchService.doStop(); } } + + private StarTreeQueryContext getStarTreeQueryContext( + SearchContext searchContext, + StarTreeFieldConfiguration starTreeFieldConfiguration, + String compositeFieldName, + int cacheSize, + List dimensions, + List metrics, + QueryBuilder baseQuery, + SearchSourceBuilder sourceBuilder, + boolean assertConsolidation + ) { + AggregatorFactories aggregatorFactories = mock(AggregatorFactories.class); + AggregatorFactory[] aggregatorFactoriesArray = sourceBuilder.aggregations().getAggregatorFactories().stream().map(af -> { + try { + return ((ValuesSourceAggregationBuilder) af).build(searchContext.getQueryShardContext(), null); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).toArray(AggregatorFactory[]::new); + when(aggregatorFactories.getFactories()).thenReturn(aggregatorFactoriesArray); + SearchContextAggregations mockAggregations = mock(SearchContextAggregations.class); + when(mockAggregations.factories()).thenReturn(aggregatorFactories); + searchContext.aggregations(mockAggregations); + CompositeDataCubeFieldType compositeDataCubeFieldType = new StarTreeMapper.StarTreeFieldType( + compositeFieldName, + new StarTreeField(compositeFieldName, dimensions, metrics, starTreeFieldConfiguration) + ); + StarTreeQueryContext starTreeQueryContext = new StarTreeQueryContext(compositeDataCubeFieldType, baseQuery, cacheSize); + boolean consolidated = starTreeQueryContext.consolidateAllFilters(searchContext); + if (assertConsolidation) { + assertTrue(consolidated); + } + return starTreeQueryContext; + } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java index c930d27b068f8..a5724d3c34352 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregatorFactoriesTests.java @@ -45,6 +45,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.env.Environment; +import org.opensearch.index.query.BaseQueryRewriteContext; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryRewriteContext; @@ -255,7 +256,7 @@ public void testRewriteAggregation() throws Exception { BucketScriptPipelineAggregationBuilder pipelineAgg = new BucketScriptPipelineAggregationBuilder("const", new Script("1")); AggregatorFactories.Builder builder = new AggregatorFactories.Builder().addAggregator(filterAggBuilder) .addPipelineAggregator(pipelineAgg); - AggregatorFactories.Builder rewritten = builder.rewrite(new QueryRewriteContext(xContentRegistry, null, null, () -> 0L)); + AggregatorFactories.Builder rewritten = builder.rewrite(new BaseQueryRewriteContext(xContentRegistry, null, null, () -> 0L)); assertNotSame(builder, rewritten); Collection aggregatorFactories = rewritten.getAggregatorFactories(); assertEquals(1, aggregatorFactories.size()); @@ -268,7 +269,9 @@ public void testRewriteAggregation() throws Exception { assertThat(rewrittenFilter, instanceOf(TermsQueryBuilder.class)); // Check that a further rewrite returns the same aggregation factories builder - AggregatorFactories.Builder secondRewritten = rewritten.rewrite(new QueryRewriteContext(xContentRegistry, null, null, () -> 0L)); + AggregatorFactories.Builder secondRewritten = rewritten.rewrite( + new BaseQueryRewriteContext(xContentRegistry, null, null, () -> 0L) + ); assertSame(rewritten, secondRewritten); } @@ -277,7 +280,7 @@ public void testRewritePipelineAggregationUnderAggregation() throws Exception { new RewrittenPipelineAggregationBuilder() ); AggregatorFactories.Builder builder = new AggregatorFactories.Builder().addAggregator(filterAggBuilder); - QueryRewriteContext context = new QueryRewriteContext(xContentRegistry, null, null, () -> 0L); + QueryRewriteContext context = new BaseQueryRewriteContext(xContentRegistry, null, null, () -> 0L); AggregatorFactories.Builder rewritten = builder.rewrite(context); CountDownLatch latch = new CountDownLatch(1); context.executeAsyncActions(new ActionListener() { @@ -304,7 +307,7 @@ public void testRewriteAggregationAtTopLevel() throws Exception { FilterAggregationBuilder filterAggBuilder = new FilterAggregationBuilder("titles", new MatchAllQueryBuilder()); AggregatorFactories.Builder builder = new AggregatorFactories.Builder().addAggregator(filterAggBuilder) .addPipelineAggregator(new RewrittenPipelineAggregationBuilder()); - QueryRewriteContext context = new QueryRewriteContext(xContentRegistry, null, null, () -> 0L); + QueryRewriteContext context = new BaseQueryRewriteContext(xContentRegistry, null, null, () -> 0L); AggregatorFactories.Builder rewritten = builder.rewrite(context); CountDownLatch latch = new CountDownLatch(1); context.executeAsyncActions(new ActionListener() { diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java index 56f7f450dbdfb..770f18f781689 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/FiltersTests.java @@ -36,12 +36,12 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BaseQueryRewriteContext; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.MatchNoneQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.BaseAggregationTestCase; import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; @@ -147,12 +147,12 @@ public void testRewrite() throws IOException { // test non-keyed filter that doesn't rewrite AggregationBuilder original = new FiltersAggregationBuilder("my-agg", new MatchAllQueryBuilder()); original.setMetadata(Collections.singletonMap(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20))); - AggregationBuilder rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + AggregationBuilder rewritten = original.rewrite(new BaseQueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); assertSame(original, rewritten); // test non-keyed filter that does rewrite original = new FiltersAggregationBuilder("my-agg", new BoolQueryBuilder()); - rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + rewritten = original.rewrite(new BaseQueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); assertNotSame(original, rewritten); assertThat(rewritten, instanceOf(FiltersAggregationBuilder.class)); assertEquals("my-agg", ((FiltersAggregationBuilder) rewritten).getName()); @@ -163,12 +163,12 @@ public void testRewrite() throws IOException { // test keyed filter that doesn't rewrite original = new FiltersAggregationBuilder("my-agg", new KeyedFilter("my-filter", new MatchAllQueryBuilder())); - rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + rewritten = original.rewrite(new BaseQueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); assertSame(original, rewritten); // test non-keyed filter that does rewrite original = new FiltersAggregationBuilder("my-agg", new KeyedFilter("my-filter", new BoolQueryBuilder())); - rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + rewritten = original.rewrite(new BaseQueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); assertNotSame(original, rewritten); assertThat(rewritten, instanceOf(FiltersAggregationBuilder.class)); assertEquals("my-agg", ((FiltersAggregationBuilder) rewritten).getName()); @@ -180,7 +180,7 @@ public void testRewrite() throws IOException { // test sub-agg filter that does rewrite original = new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.BOOLEAN) .subAggregation(new FiltersAggregationBuilder("my-agg", new KeyedFilter("my-filter", new BoolQueryBuilder()))); - rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + rewritten = original.rewrite(new BaseQueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); assertNotSame(original, rewritten); assertNotEquals(original, rewritten); assertThat(rewritten, instanceOf(TermsAggregationBuilder.class)); @@ -189,7 +189,7 @@ public void testRewrite() throws IOException { assertThat(subAgg, instanceOf(FiltersAggregationBuilder.class)); assertNotSame(original.getSubAggregations().iterator().next(), subAgg); assertEquals("my-agg", subAgg.getName()); - assertSame(rewritten, rewritten.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L))); + assertSame(rewritten, rewritten.rewrite(new BaseQueryRewriteContext(xContentRegistry(), null, null, () -> 0L))); } public void testRewritePreservesOtherBucket() throws IOException { @@ -197,7 +197,7 @@ public void testRewritePreservesOtherBucket() throws IOException { originalFilters.otherBucket(randomBoolean()); originalFilters.otherBucketKey(randomAlphaOfLength(10)); - AggregationBuilder rewritten = originalFilters.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + AggregationBuilder rewritten = originalFilters.rewrite(new BaseQueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); assertThat(rewritten, instanceOf(FiltersAggregationBuilder.class)); FiltersAggregationBuilder rewrittenFilters = (FiltersAggregationBuilder) rewritten; diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java new file mode 100644 index 0000000000000..824e2b6c36d81 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.startree; + +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedSetStarTreeValuesIterator; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.search.startree.filter.DimensionFilter.MatchType; +import org.opensearch.search.startree.filter.provider.DimensionFilterMapper; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.Optional; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DimensionFilterAndMapperTests extends OpenSearchTestCase { + + public void testKeywordOrdinalMapping() throws IOException { + DimensionFilterMapper dimensionFilterMapper = DimensionFilterMapper.Factory.fromMappedFieldType( + new KeywordFieldMapper.KeywordFieldType("keyword") + ); + StarTreeValues starTreeValues = mock(StarTreeValues.class); + SortedSetStarTreeValuesIterator sortedSetStarTreeValuesIterator = mock(SortedSetStarTreeValuesIterator.class); + TermsEnum termsEnum = mock(TermsEnum.class); + when(sortedSetStarTreeValuesIterator.termsEnum()).thenReturn(termsEnum); + when(starTreeValues.getDimensionValuesIterator("field")).thenReturn(sortedSetStarTreeValuesIterator); + Optional matchingOrdinal; + + // Case Exact Match and found + BytesRef bytesRef = new BytesRef(new byte[] { 17, 29 }); + when(sortedSetStarTreeValuesIterator.lookupTerm(bytesRef)).thenReturn(1L); + matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, MatchType.EXACT); + assertTrue(matchingOrdinal.isPresent()); + assertEquals(1, (long) matchingOrdinal.get()); + + // Case Exact Match and not found + when(sortedSetStarTreeValuesIterator.lookupTerm(bytesRef)).thenReturn(-10L); + matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, MatchType.EXACT); + assertFalse(matchingOrdinal.isPresent()); + + // Case GTE -> FOUND and NOT_FOUND + for (TermsEnum.SeekStatus seekStatus : new TermsEnum.SeekStatus[] { TermsEnum.SeekStatus.FOUND, TermsEnum.SeekStatus.NOT_FOUND }) { + when(termsEnum.seekCeil(bytesRef)).thenReturn(seekStatus); + when(termsEnum.ord()).thenReturn(10L); + matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, MatchType.GTE); + assertTrue(matchingOrdinal.isPresent()); + assertEquals(10L, (long) matchingOrdinal.get()); + } + + // Seek Status END is same for GTE, GT + for (MatchType matchType : new MatchType[] { MatchType.GT, MatchType.GTE }) { + when(termsEnum.seekCeil(bytesRef)).thenReturn(TermsEnum.SeekStatus.END); + when(termsEnum.ord()).thenReturn(10L); + matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, matchType); + assertFalse(matchingOrdinal.isPresent()); + } + + // Case GT -> FOUND and matched + when(termsEnum.seekCeil(bytesRef)).thenReturn(TermsEnum.SeekStatus.FOUND); + when(sortedSetStarTreeValuesIterator.getValueCount()).thenReturn(2L); + when(termsEnum.ord()).thenReturn(0L); + matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, MatchType.GT); + assertTrue(matchingOrdinal.isPresent()); + assertEquals(1L, (long) matchingOrdinal.get()); + // Case GT -> FOUND and unmatched + when(termsEnum.ord()).thenReturn(3L); + matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, MatchType.GT); + assertFalse(matchingOrdinal.isPresent()); + + // Case GT -> NOT_FOUND + when(termsEnum.seekCeil(bytesRef)).thenReturn(TermsEnum.SeekStatus.NOT_FOUND); + when(termsEnum.ord()).thenReturn(10L); + matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, MatchType.GT); + assertTrue(matchingOrdinal.isPresent()); + assertEquals(10L, (long) matchingOrdinal.get()); + + // Seek Status END is same for LTE, LT + for (MatchType matchType : new MatchType[] { MatchType.LT, MatchType.LTE }) { + when(termsEnum.seekCeil(bytesRef)).thenReturn(TermsEnum.SeekStatus.END); + when(termsEnum.ord()).thenReturn(10L); + matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, matchType); + assertTrue(matchingOrdinal.isPresent()); + assertEquals(10L, (long) matchingOrdinal.get()); + } + + // Seek Status NOT_FOUND is same for LTE, LT + for (MatchType matchType : new MatchType[] { MatchType.LT, MatchType.LTE }) { + when(termsEnum.seekCeil(bytesRef)).thenReturn(TermsEnum.SeekStatus.NOT_FOUND); + when(sortedSetStarTreeValuesIterator.getValueCount()).thenReturn(2L); + when(termsEnum.ord()).thenReturn(1L); + matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, matchType); + assertTrue(matchingOrdinal.isPresent()); + assertEquals(0L, (long) matchingOrdinal.get()); + // Case unmatched + when(termsEnum.ord()).thenReturn(0L); + matchingOrdinal = dimensionFilterMapper.getMatchingOrdinal("field", bytesRef, starTreeValues, matchType); + assertFalse(matchingOrdinal.isPresent()); + } + + } + +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java index 21a4155a53ee5..20f682899f136 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java @@ -15,13 +15,20 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene101.Lucene101Codec; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleField; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.KeywordField; +import org.apache.lucene.document.LongField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.sandbox.document.HalfFloatPoint; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.RandomIndexWriter; @@ -40,12 +47,16 @@ import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.OrdinalDimension; +import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.RangeQueryBuilder; import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.TermsQueryBuilder; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; @@ -69,11 +80,16 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Random; import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.opensearch.search.aggregations.AggregationBuilders.avg; import static org.opensearch.search.aggregations.AggregationBuilders.count; @@ -100,11 +116,17 @@ public void teardown() throws IOException { FeatureFlags.initializeFeatureFlags(Settings.EMPTY); } - protected Codec getCodec() { + protected Codec getCodec( + Supplier maxLeafDocsSupplier, + LinkedHashMap dimensionAndType, + Map metricFieldAndType + ) { final Logger testLogger = LogManager.getLogger(MetricAggregatorTests.class); MapperService mapperService; try { - mapperService = StarTreeDocValuesFormatTests.createMapperService(StarTreeFilterTests.getExpandedMapping(1, false)); + mapperService = StarTreeDocValuesFormatTests.createMapperService( + StarTreeQueryTests.getExpandedMapping(maxLeafDocsSupplier.get(), false, dimensionAndType, metricFieldAndType) + ); } catch (IOException e) { throw new RuntimeException(e); } @@ -112,36 +134,65 @@ protected Codec getCodec() { } public void testStarTreeDocValues() throws IOException { + final List> MAX_LEAF_DOC_VARIATIONS = List.of( + () -> 1, + () -> randomIntBetween(2, 100), + () -> randomIntBetween(101, 10_000) + ); + final List dimensionFieldDatum = List.of( + new DimensionFieldData("sndv", () -> random().nextInt(10) - 5, DimensionTypes.INTEGER), + new DimensionFieldData("dv", () -> random().nextInt(20) - 10, DimensionTypes.INTEGER), + new DimensionFieldData("keyword_field", () -> random().nextInt(50), DimensionTypes.KEYWORD), + new DimensionFieldData("long_field", () -> random().nextInt(50), DimensionTypes.LONG), + new DimensionFieldData("half_float_field", () -> random().nextFloat(50), DimensionTypes.HALF_FLOAT), + new DimensionFieldData("float_field", () -> random().nextFloat(50), DimensionTypes.FLOAT), + new DimensionFieldData("double_field", () -> random().nextDouble(50), DimensionTypes.DOUBLE) + ); + for (Supplier maxLeafDocsSupplier : MAX_LEAF_DOC_VARIATIONS) { + testStarTreeDocValuesInternal( + getCodec( + maxLeafDocsSupplier, + dimensionFieldDatum.stream() + .collect( + Collectors.toMap( + df -> df.getDimension().getField(), + DimensionFieldData::getFieldType, + (v1, v2) -> v1, + LinkedHashMap::new + ) + ), + StarTreeQueryTests.METRIC_TYPE_MAP + ), + dimensionFieldDatum + ); + } + } + + private void testStarTreeDocValuesInternal(Codec codec, List dimensionFieldData) throws IOException { Directory directory = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(null); - conf.setCodec(getCodec()); + conf.setCodec(codec); conf.setMergePolicy(newLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(random(), directory, conf); Random random = RandomizedTest.getRandom(); int totalDocs = 100; - final String SNDV = "sndv"; - final String DV = "dv"; int val; - List docs = new ArrayList<>(); // Index 100 random documents for (int i = 0; i < totalDocs; i++) { Document doc = new Document(); - if (random.nextBoolean()) { - val = random.nextInt(10) - 5; // Random long between -5 and 4 - doc.add(new SortedNumericDocValuesField(SNDV, val)); - } - if (random.nextBoolean()) { - val = random.nextInt(20) - 10; // Random long between -10 and 9 - doc.add(new SortedNumericDocValuesField(DV, val)); + for (DimensionFieldData fieldData : dimensionFieldData) { + // FIXME: Reduce the frequency of nulls to be with at least some non-null docs like after every 1-2 ? + if (random.nextBoolean()) { + doc.add(fieldData.getField()); + } } if (random.nextBoolean()) { val = random.nextInt(50); // Random long between 0 and 49 doc.add(new SortedNumericDocValuesField(FIELD_NAME, val)); } iw.addDocument(doc); - docs.add(doc); } if (randomBoolean()) { @@ -157,6 +208,23 @@ public void testStarTreeDocValues() throws IOException { IndexSearcher indexSearcher = newSearcher(reader, false, false); CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + MapperService mapperService = mapperServiceMock(); + CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); + for (DimensionFieldData fieldData : dimensionFieldData) { + when(mapperService.fieldType(fieldData.fieldName)).thenReturn(fieldData.getMappedField()); + } + QueryShardContext queryShardContext = queryShardContextMock( + indexSearcher, + mapperService, + createIndexSettings(), + circuitBreakerService, + new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), circuitBreakerService).withCircuitBreaking() + ); + for (DimensionFieldData fieldData : dimensionFieldData) { + when(mapperService.fieldType(fieldData.fieldName)).thenReturn(fieldData.getMappedField()); + when(queryShardContext.fieldMapper(fieldData.fieldName)).thenReturn(fieldData.getMappedField()); + } + List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); CompositeIndexFieldInfo starTree = compositeIndexFields.get(0); @@ -166,131 +234,71 @@ public void testStarTreeDocValues() throws IOException { ValueCountAggregationBuilder valueCountAggregationBuilder = count("_name").field(FIELD_NAME); AvgAggregationBuilder avgAggregationBuilder = avg("_name").field(FIELD_NAME); - List supportedDimensions = new LinkedList<>(); - supportedDimensions.add(new NumericDimension(SNDV)); - supportedDimensions.add(new NumericDimension(DV)); - - Query query = new MatchAllDocsQuery(); - // match-all query - QueryBuilder queryBuilder = null; // no predicates - testCase( - indexSearcher, - query, - queryBuilder, - sumAggregationBuilder, - starTree, - supportedDimensions, - verifyAggregation(InternalSum::getValue) - ); - testCase( - indexSearcher, - query, - queryBuilder, - maxAggregationBuilder, - starTree, - supportedDimensions, - verifyAggregation(InternalMax::getValue) - ); - testCase( - indexSearcher, - query, - queryBuilder, - minAggregationBuilder, - starTree, - supportedDimensions, - verifyAggregation(InternalMin::getValue) - ); - testCase( - indexSearcher, - query, - queryBuilder, - valueCountAggregationBuilder, - starTree, - supportedDimensions, - verifyAggregation(InternalValueCount::getValue) - ); - testCase( - indexSearcher, - query, - queryBuilder, - avgAggregationBuilder, - starTree, - supportedDimensions, - verifyAggregation(InternalAvg::getValue) - ); + LinkedHashMap supportedDimensions = dimensionFieldData.stream() + .collect( + Collectors.toMap(DimensionFieldData::getDimension, DimensionFieldData::getMappedField, (v1, v2) -> v1, LinkedHashMap::new) + ); - // Numeric-terms query - for (int cases = 0; cases < 100; cases++) { - String queryField; - long queryValue; - if (randomBoolean()) { - queryField = SNDV; - queryValue = random.nextInt(10); - } else { - queryField = DV; - queryValue = random.nextInt(20) - 15; + Query query = null; + QueryBuilder queryBuilder = null; + + for (int cases = 0; cases < 15; cases++) { + // Get all types of queries (Term/Terms/Range) for all the given dimensions. + List allFieldQueries = dimensionFieldData.stream() + .flatMap(x -> Stream.of(x.getTermQueryBuilder(), x.getTermsQueryBuilder(), x.getRangeQueryBuilder())) + .toList(); + + for (QueryBuilder qb : allFieldQueries) { + query = qb.toQuery(queryShardContext); + queryBuilder = qb; + testCase( + indexSearcher, + query, + qb, + sumAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalSum::getValue) + ); + testCase( + indexSearcher, + query, + qb, + maxAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalMax::getValue) + ); + testCase( + indexSearcher, + query, + qb, + minAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalMin::getValue) + ); + testCase( + indexSearcher, + query, + qb, + valueCountAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalValueCount::getValue) + ); + testCase( + indexSearcher, + query, + qb, + avgAggregationBuilder, + starTree, + supportedDimensions, + verifyAggregation(InternalAvg::getValue) + ); } - - query = SortedNumericDocValuesField.newSlowExactQuery(queryField, queryValue); - queryBuilder = new TermQueryBuilder(queryField, queryValue); - - testCase( - indexSearcher, - query, - queryBuilder, - sumAggregationBuilder, - starTree, - supportedDimensions, - verifyAggregation(InternalSum::getValue) - ); - testCase( - indexSearcher, - query, - queryBuilder, - maxAggregationBuilder, - starTree, - supportedDimensions, - verifyAggregation(InternalMax::getValue) - ); - testCase( - indexSearcher, - query, - queryBuilder, - minAggregationBuilder, - starTree, - supportedDimensions, - verifyAggregation(InternalMin::getValue) - ); - testCase( - indexSearcher, - query, - queryBuilder, - valueCountAggregationBuilder, - starTree, - supportedDimensions, - verifyAggregation(InternalValueCount::getValue) - ); - testCase( - indexSearcher, - query, - queryBuilder, - avgAggregationBuilder, - starTree, - supportedDimensions, - verifyAggregation(InternalAvg::getValue) - ); } - CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); - - QueryShardContext queryShardContext = queryShardContextMock( - indexSearcher, - mapperServiceMock(), - createIndexSettings(), - circuitBreakerService, - new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), circuitBreakerService).withCircuitBreaking() - ); - MetricAggregatorFactory aggregatorFactory = mock(MetricAggregatorFactory.class); when(aggregatorFactory.getSubFactories()).thenReturn(AggregatorFactories.EMPTY); when(aggregatorFactory.getField()).thenReturn(FIELD_NAME); @@ -385,6 +393,22 @@ public void testStarTreeDocValues() throws IOException { false ); + // Keyword Range query with missing Low Ordinal + RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder("keyword_field"); + rangeQueryBuilder.from(Long.MAX_VALUE).includeLower(random().nextBoolean()); + testCase( + indexSearcher, + rangeQueryBuilder.toQuery(queryShardContext), + rangeQueryBuilder, + sumAggregationBuilder, + starTree, + supportedDimensions, + List.of(new Metric(FIELD_NAME, List.of(MetricStat.SUM, MetricStat.MAX, MetricStat.MIN, MetricStat.AVG))), + verifyAggregation(InternalSum::getValue), + null, + true + ); + ir.close(); directory.close(); } @@ -403,7 +427,7 @@ private void testC QueryBuilder queryBuilder, T aggBuilder, CompositeIndexFieldInfo starTree, - List supportedDimensions, + LinkedHashMap supportedDimensions, BiConsumer verify ) throws IOException { testCase(searcher, query, queryBuilder, aggBuilder, starTree, supportedDimensions, Collections.emptyList(), verify, null, true); @@ -415,7 +439,8 @@ private void testC QueryBuilder queryBuilder, T aggBuilder, CompositeIndexFieldInfo starTree, - List supportedDimensions, + LinkedHashMap supportedDimensions, // FIXME : Merge with the same input that goes to generating the + // codec. List supportedMetrics, BiConsumer verify, AggregatorFactory aggregatorFactory, @@ -453,4 +478,164 @@ private void testC ); verify.accept(expectedAggregation, starTreeAggregation); } + + private interface DimensionFieldDataSupplier { + IndexableField getField(String fieldName, Supplier valueSupplier); + + MappedFieldType getMappedField(String fieldName); + + Dimension getDimension(String fieldName); + } + + private abstract static class NumericDimensionFieldDataSupplier implements DimensionFieldDataSupplier { + + @Override + public Dimension getDimension(String fieldName) { + return new NumericDimension(fieldName); + } + + @Override + public MappedFieldType getMappedField(String fieldName) { + return new NumberFieldMapper.NumberFieldType(fieldName, numberType()); + } + + abstract NumberFieldMapper.NumberType numberType(); + } + + private static class DimensionFieldData { + private final String fieldName; + private final Supplier valueSupplier; + private final DimensionFieldDataSupplier dimensionFieldDataSupplier; + private final String fieldType; + + DimensionFieldData(String fieldName, Supplier valueSupplier, DimensionTypes dimensionType) { + this.fieldName = fieldName; + this.valueSupplier = valueSupplier; + this.dimensionFieldDataSupplier = dimensionType.getFieldDataSupplier(); + this.fieldType = dimensionType.name().toLowerCase(Locale.ROOT); + } + + public Dimension getDimension() { + return dimensionFieldDataSupplier.getDimension(fieldName); + } + + public MappedFieldType getMappedField() { + return dimensionFieldDataSupplier.getMappedField(fieldName); + } + + public IndexableField getField() { + return dimensionFieldDataSupplier.getField(fieldName, valueSupplier); + } + + public QueryBuilder getTermQueryBuilder() { + return new TermQueryBuilder(fieldName, valueSupplier.get()); + } + + public QueryBuilder getTermsQueryBuilder() { + int limit = randomIntBetween(1, 20); + List values = new ArrayList<>(limit); + for (int i = 0; i < limit; i++) { + values.add(valueSupplier.get()); + } + return new TermsQueryBuilder(fieldName, values); + } + + public QueryBuilder getRangeQueryBuilder() { + return new RangeQueryBuilder(fieldName).from(valueSupplier.get()) + .to(valueSupplier.get()) + .includeLower(randomBoolean()) + .includeUpper(randomBoolean()); + } + + public String getFieldType() { + return fieldType; + } + } + + private enum DimensionTypes { + + INTEGER(new NumericDimensionFieldDataSupplier() { + @Override + NumberFieldMapper.NumberType numberType() { + return NumberFieldMapper.NumberType.INTEGER; + } + + @Override + public IndexableField getField(String fieldName, Supplier valueSupplier) { + return new IntField(fieldName, (Integer) valueSupplier.get(), Field.Store.YES); + } + }), + LONG(new NumericDimensionFieldDataSupplier() { + @Override + NumberFieldMapper.NumberType numberType() { + return NumberFieldMapper.NumberType.LONG; + } + + @Override + public IndexableField getField(String fieldName, Supplier valueSupplier) { + return new LongField(fieldName, (Integer) valueSupplier.get(), Field.Store.YES); + } + }), + HALF_FLOAT(new NumericDimensionFieldDataSupplier() { + @Override + public IndexableField getField(String fieldName, Supplier valueSupplier) { + return new SortedNumericDocValuesField(fieldName, HalfFloatPoint.halfFloatToSortableShort((Float) valueSupplier.get())); + } + + @Override + NumberFieldMapper.NumberType numberType() { + return NumberFieldMapper.NumberType.HALF_FLOAT; + } + }), + FLOAT(new NumericDimensionFieldDataSupplier() { + @Override + public IndexableField getField(String fieldName, Supplier valueSupplier) { + return new FloatField(fieldName, (Float) valueSupplier.get(), Field.Store.YES); + } + + @Override + NumberFieldMapper.NumberType numberType() { + return NumberFieldMapper.NumberType.FLOAT; + } + }), + DOUBLE(new NumericDimensionFieldDataSupplier() { + @Override + public IndexableField getField(String fieldName, Supplier valueSupplier) { + return new DoubleField(fieldName, (Double) valueSupplier.get(), Field.Store.YES); + } + + @Override + NumberFieldMapper.NumberType numberType() { + return NumberFieldMapper.NumberType.DOUBLE; + } + }), + KEYWORD(new DimensionFieldDataSupplier() { + @Override + public IndexableField getField(String fieldName, Supplier valueSupplier) { + return new KeywordField(fieldName, String.valueOf(valueSupplier.get()), Field.Store.YES); + } + + @Override + public MappedFieldType getMappedField(String fieldName) { + return new KeywordFieldMapper.KeywordFieldType(fieldName, Lucene.STANDARD_ANALYZER); + } + + @Override + public Dimension getDimension(String fieldName) { + return new OrdinalDimension(fieldName); + } + }); + + private final DimensionFieldDataSupplier dimensionFieldDataSupplier; + + DimensionTypes(DimensionFieldDataSupplier dimensionFieldDataSupplier) { + this.dimensionFieldDataSupplier = dimensionFieldDataSupplier; + } + + public DimensionFieldDataSupplier getFieldDataSupplier() { + return dimensionFieldDataSupplier; + } + + } + } diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeQueryTests.java similarity index 65% rename from server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java rename to server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeQueryTests.java index eb891318dd087..cfa619e8f4592 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeQueryTests.java @@ -33,29 +33,47 @@ import org.opensearch.index.codec.composite912.datacube.startree.StarTreeDocValuesFormatTests; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.search.aggregations.AggregatorTestCase; -import org.opensearch.search.startree.StarTreeFilter; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeQueryHelper; +import org.opensearch.search.startree.StarTreeTraversalUtil; +import org.opensearch.search.startree.filter.ExactMatchDimFilter; +import org.opensearch.search.startree.filter.StarTreeFilter; import org.junit.After; import org.junit.Before; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.mockito.Mockito; + import static org.opensearch.index.codec.composite912.datacube.startree.AbstractStarTreeDVFormatTests.topMapping; -public class StarTreeFilterTests extends AggregatorTestCase { +public class StarTreeQueryTests extends AggregatorTestCase { private static final String FIELD_NAME = "field"; private static final String SNDV = "sndv"; private static final String SDV = "sdv"; private static final String DV = "dv"; + public static final LinkedHashMap DIMENSION_TYPE_MAP = new LinkedHashMap<>(); + public static final Map METRIC_TYPE_MAP = Map.of(FIELD_NAME, "integer"); + + static { + // Ordered dimensions + DIMENSION_TYPE_MAP.put(SNDV, "integer"); + DIMENSION_TYPE_MAP.put(SDV, "integer"); + DIMENSION_TYPE_MAP.put(DV, "integer"); + } + @Before public void setup() { FeatureFlags.initializeFeatureFlags(Settings.builder().put(FeatureFlags.STAR_TREE_INDEX, true).build()); @@ -67,11 +85,11 @@ public void teardown() throws IOException { } protected Codec getCodec(int maxLeafDoc, boolean skipStarNodeCreationForSDVDimension) { - final Logger testLogger = LogManager.getLogger(StarTreeFilterTests.class); + final Logger testLogger = LogManager.getLogger(StarTreeQueryTests.class); MapperService mapperService; try { mapperService = StarTreeDocValuesFormatTests.createMapperService( - getExpandedMapping(maxLeafDoc, skipStarNodeCreationForSDVDimension) + getExpandedMapping(maxLeafDoc, skipStarNodeCreationForSDVDimension, DIMENSION_TYPE_MAP, StarTreeQueryTests.METRIC_TYPE_MAP) ); } catch (IOException e) { throw new RuntimeException(e); @@ -128,69 +146,141 @@ private void testStarTreeFilter(int maxLeafDoc, boolean skipStarNodeCreationForS long starTreeDocCount, docCount; + MapperService mapperService = Mockito.mock(MapperService.class); + SearchContext searchContext = Mockito.mock(SearchContext.class); + + Mockito.when(searchContext.mapperService()).thenReturn(mapperService); + Mockito.when(mapperService.fieldType(SNDV)) + .thenReturn(new NumberFieldMapper.NumberFieldType(SNDV, NumberFieldMapper.NumberType.INTEGER)); + Mockito.when(mapperService.fieldType(DV)) + .thenReturn(new NumberFieldMapper.NumberFieldType(DV, NumberFieldMapper.NumberType.INTEGER)); + Mockito.when(mapperService.fieldType(SDV)) + .thenReturn(new NumberFieldMapper.NumberFieldType(SDV, NumberFieldMapper.NumberType.INTEGER)); + // assert that all documents are included if no filters are given - starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(), context); + starTreeDocCount = getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter(Collections.emptyMap()), + context, + searchContext + ); docCount = getDocCount(docs, Map.of()); assertEquals(totalDocs, starTreeDocCount); assertEquals(docCount, starTreeDocCount); // single filter - matches docs - starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SNDV, 0L), context); + starTreeDocCount = getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter(Map.of(SNDV, List.of(new ExactMatchDimFilter(SNDV, List.of(0L))))), + context, + searchContext + ); docCount = getDocCount(docs, Map.of(SNDV, 0L)); assertEquals(1, docCount); assertEquals(docCount, starTreeDocCount); // single filter on 3rd field in ordered dimension - matches docs - starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(DV, 0L), context); + starTreeDocCount = getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter(Map.of(DV, List.of(new ExactMatchDimFilter(DV, List.of(0L))))), + context, + searchContext + ); docCount = getDocCount(docs, Map.of(DV, 0L)); assertEquals(1, docCount); assertEquals(docCount, starTreeDocCount); // single filter - does not match docs - starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SNDV, 101L), context); + starTreeDocCount = getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter(Map.of(SNDV, List.of(new ExactMatchDimFilter(SNDV, List.of(101L))))), + context, + searchContext + ); docCount = getDocCount(docs, Map.of(SNDV, 101L)); assertEquals(0, docCount); assertEquals(docCount, starTreeDocCount); // single filter on 3rd field in ordered dimension - does not match docs - starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(DV, -101L), context); + starTreeDocCount = getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter(Map.of(SNDV, List.of(new ExactMatchDimFilter(SNDV, List.of(-101L))))), + context, + searchContext + ); docCount = getDocCount(docs, Map.of(SNDV, -101L)); assertEquals(0, docCount); assertEquals(docCount, starTreeDocCount); // multiple filters - matches docs - starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SNDV, 0L, DV, 0L), context); + starTreeDocCount = getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter( + Map.of(SNDV, List.of(new ExactMatchDimFilter(SNDV, List.of(0L))), DV, List.of(new ExactMatchDimFilter(DV, List.of(0L)))) + ), + context, + searchContext + ); docCount = getDocCount(docs, Map.of(SNDV, 0L, DV, 0L)); assertEquals(1, docCount); assertEquals(docCount, starTreeDocCount); // no document should match the filter - starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SNDV, 0L, DV, -11L), context); + starTreeDocCount = getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter( + Map.of(SNDV, List.of(new ExactMatchDimFilter(SNDV, List.of(0L))), DV, List.of(new ExactMatchDimFilter(DV, List.of(-11L)))) + ), + context, + searchContext + ); docCount = getDocCount(docs, Map.of(SNDV, 0L, DV, -11L)); assertEquals(0, docCount); assertEquals(docCount, starTreeDocCount); // Only the first filter should match some documents, second filter matches none - starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SNDV, 0L, DV, -100L), context); + starTreeDocCount = getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter( + Map.of(SNDV, List.of(new ExactMatchDimFilter(SNDV, List.of(0L))), DV, List.of(new ExactMatchDimFilter(DV, List.of(-100L)))) + ), + context, + searchContext + ); docCount = getDocCount(docs, Map.of(SNDV, 0L, DV, -100L)); assertEquals(0, docCount); assertEquals(docCount, starTreeDocCount); // non-dimension fields in filter - should throw IllegalArgumentException expectThrows( - IllegalArgumentException.class, - () -> getDocCountFromStarTree(starTreeDocValuesReader, Map.of(FIELD_NAME, 0L), context) + IllegalStateException.class, + () -> getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter(Map.of(FIELD_NAME, List.of(new ExactMatchDimFilter(FIELD_NAME, List.of(0L))))), + context, + searchContext + ) ); if (skipStarNodeCreationForSDVDimension == true) { // Documents are not indexed - starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SDV, 4L), context); + starTreeDocCount = getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter(Map.of(SDV, List.of(new ExactMatchDimFilter(SDV, List.of(4L))))), + context, + searchContext + ); docCount = getDocCount(docs, Map.of(SDV, 4L)); assertEquals(1, docCount); assertEquals(docCount, starTreeDocCount); } else { // Documents are indexed - starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, Map.of(SDV, 4L), context); + starTreeDocCount = getDocCountFromStarTree( + starTreeDocValuesReader, + new StarTreeFilter(Map.of(SDV, List.of(new ExactMatchDimFilter(SDV, List.of(4L))))), + context, + searchContext + ); docCount = getDocCount(docs, Map.of(SDV, 4L)); assertEquals(0, docCount); assertEquals(docCount, starTreeDocCount); @@ -224,12 +314,17 @@ private long getDocCount(List documents, Map filters) { } // Returns count of documents in the star tree having field SNDV & applied filters - private long getDocCountFromStarTree(CompositeIndexReader starTreeDocValuesReader, Map filters, LeafReaderContext context) - throws IOException { + private long getDocCountFromStarTree( + CompositeIndexReader starTreeDocValuesReader, + StarTreeFilter starTreeFilter, + LeafReaderContext context, + SearchContext searchContext + ) throws IOException { List compositeIndexFields = starTreeDocValuesReader.getCompositeIndexFields(); CompositeIndexFieldInfo starTree = compositeIndexFields.get(0); StarTreeValues starTreeValues = StarTreeQueryHelper.getStarTreeValues(context, starTree); - FixedBitSet filteredValues = StarTreeFilter.getStarTreeResult(starTreeValues, filters); + + FixedBitSet filteredValues = StarTreeTraversalUtil.getStarTreeResult(starTreeValues, starTreeFilter, searchContext); SortedNumericStarTreeValuesIterator valuesIterator = (SortedNumericStarTreeValuesIterator) starTreeValues.getMetricValuesIterator( StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -261,7 +356,12 @@ private long getDocCountFromStarTree(CompositeIndexReader starTreeDocValuesReade return docCount; } - public static XContentBuilder getExpandedMapping(int maxLeafDocs, boolean skipStarNodeCreationForSDVDimension) throws IOException { + public static XContentBuilder getExpandedMapping( + int maxLeafDocs, + boolean skipStarNodeCreationForSDVDimension, + LinkedHashMap dimensionNameAndType, + Map metricFieldNameAndType + ) throws IOException { return topMapping(b -> { b.startObject("composite"); b.startObject("startree"); @@ -273,16 +373,14 @@ public static XContentBuilder getExpandedMapping(int maxLeafDocs, boolean skipSt b.value("sdv"); b.endArray(); } + // FIXME : Change to take dimension order and other inputs as method params. + // FIXME : Create default constants for the existing so other can call easily. b.startArray("ordered_dimensions"); - b.startObject(); - b.field("name", "sndv"); - b.endObject(); - b.startObject(); - b.field("name", "sdv"); - b.endObject(); - b.startObject(); - b.field("name", "dv"); - b.endObject(); + for (String dimension : dimensionNameAndType.keySet()) { + b.startObject(); + b.field("name", dimension); + b.endObject(); + } b.endArray(); b.startArray("metrics"); b.startObject(); @@ -310,18 +408,16 @@ public static XContentBuilder getExpandedMapping(int maxLeafDocs, boolean skipSt b.endObject(); b.endObject(); b.startObject("properties"); - b.startObject("sndv"); - b.field("type", "integer"); - b.endObject(); - b.startObject("sdv"); - b.field("type", "integer"); - b.endObject(); - b.startObject("dv"); - b.field("type", "integer"); - b.endObject(); - b.startObject("field"); - b.field("type", "integer"); - b.endObject(); + for (String dimension : dimensionNameAndType.keySet()) { + b.startObject(dimension); + b.field("type", dimensionNameAndType.get(dimension)); + b.endObject(); + } + for (String metricField : metricFieldNameAndType.keySet()) { + b.startObject(metricField); + b.field("type", metricFieldNameAndType.get(metricField)); + b.endObject(); + } b.endObject(); }); } diff --git a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java index 90962a5c613f1..4ee1ee61d9586 100644 --- a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java @@ -47,10 +47,10 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.query.BaseQueryRewriteContext; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchNoneQueryBuilder; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.RandomQueryBuilder; import org.opensearch.index.query.Rewriteable; import org.opensearch.script.Script; @@ -737,7 +737,7 @@ private void assertIndicesBoostParseErrorMessage(String restContent, String expe private SearchSourceBuilder rewrite(SearchSourceBuilder searchSourceBuilder) throws IOException { return Rewriteable.rewrite( searchSourceBuilder, - new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, Long.valueOf(1)::longValue) + new BaseQueryRewriteContext(xContentRegistry(), writableRegistry(), null, Long.valueOf(1)::longValue) ); } } diff --git a/server/src/test/java/org/opensearch/search/fetch/FetchPhaseTests.java b/server/src/test/java/org/opensearch/search/fetch/FetchPhaseTests.java index a4820c6cff003..4f03dfd47c512 100644 --- a/server/src/test/java/org/opensearch/search/fetch/FetchPhaseTests.java +++ b/server/src/test/java/org/opensearch/search/fetch/FetchPhaseTests.java @@ -32,8 +32,22 @@ package org.opensearch.search.fetch; +import org.opensearch.index.fieldvisitor.CustomFieldsVisitor; +import org.opensearch.index.fieldvisitor.FieldsVisitor; +import org.opensearch.search.fetch.subphase.FetchSourceContext; +import org.opensearch.search.internal.SearchContext; import org.opensearch.test.OpenSearchTestCase; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class FetchPhaseTests extends OpenSearchTestCase { public void testSequentialDocs() { FetchPhase.DocIdToIndex[] docs = new FetchPhase.DocIdToIndex[10]; @@ -52,4 +66,54 @@ public void testSequentialDocs() { } assertFalse(FetchPhase.hasSequentialDocs(docs)); } + + public void testFieldsVisitorsInFetchPhase() { + + FetchPhase fetchPhase = new FetchPhase(new ArrayList<>()); + SearchContext mockSearchContext = mock(SearchContext.class); + when(mockSearchContext.docIdsToLoadSize()).thenReturn(1); + when(mockSearchContext.docIdsToLoad()).thenReturn(new int[] { 1 }); + String[] includes = new String[] { "field1", "field2" }; + String[] excludes = new String[] { "field7", "field8" }; + + FetchSourceContext mockFetchSourceContext = new FetchSourceContext(true, includes, excludes); + when(mockSearchContext.hasFetchSourceContext()).thenReturn(true); + when(mockSearchContext.fetchSourceContext()).thenReturn(mockFetchSourceContext); + + // Case 1 + // if storedFieldsContext is null + FieldsVisitor fieldsVisitor = fetchPhase.createStoredFieldsVisitor(mockSearchContext, null); + assertArrayEquals(fieldsVisitor.excludes(), excludes); + assertArrayEquals(fieldsVisitor.includes(), includes); + + // Case 2 + // if storedFieldsContext is not null + StoredFieldsContext storedFieldsContext = mock(StoredFieldsContext.class); + when(mockSearchContext.storedFieldsContext()).thenReturn(storedFieldsContext); + + fieldsVisitor = fetchPhase.createStoredFieldsVisitor(mockSearchContext, null); + assertNull(fieldsVisitor); + + // Case 3 + // if storedFieldsContext is true but fieldNames are empty + when(storedFieldsContext.fetchFields()).thenReturn(true); + when(storedFieldsContext.fieldNames()).thenReturn(List.of()); + fieldsVisitor = fetchPhase.createStoredFieldsVisitor(mockSearchContext, Collections.emptyMap()); + assertArrayEquals(fieldsVisitor.excludes(), excludes); + assertArrayEquals(fieldsVisitor.includes(), includes); + + // Case 4 + // if storedToRequested Fields is not empty + // creates an instance of CustomFieldsVisitor + Map> storedToRequestedFields = new HashMap<>(); + storedToRequestedFields.put("test_field_key", Set.of("test_field_value")); + + fieldsVisitor = fetchPhase.createStoredFieldsVisitor(mockSearchContext, storedToRequestedFields); + + assertTrue(fieldsVisitor instanceof CustomFieldsVisitor); + assertArrayEquals(fieldsVisitor.excludes(), excludes); + assertArrayEquals(fieldsVisitor.includes(), includes); + + } + } diff --git a/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java b/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java index bfda27c97ca4c..f733154c643da 100644 --- a/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java +++ b/server/src/test/java/org/opensearch/snapshots/RestoreServiceIntegTests.java @@ -31,7 +31,7 @@ import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.settings.Settings; import org.opensearch.repositories.fs.FsRepository; diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 697ec511b54f2..493a3ed431e00 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -106,9 +106,9 @@ import org.opensearch.action.support.PlainActionFuture; import org.opensearch.action.support.TransportAction; import org.opensearch.action.support.WriteRequest; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.action.support.clustermanager.term.GetTermVersionAction; import org.opensearch.action.support.clustermanager.term.TransportGetTermVersionAction; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.action.update.UpdateHelper; import org.opensearch.client.AdminClient; import org.opensearch.client.node.NodeClient; diff --git a/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java index 933385dedcf49..76c7ce0628aac 100644 --- a/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java +++ b/test/framework/src/main/java/org/opensearch/bootstrap/BootstrapForTesting.java @@ -39,6 +39,7 @@ import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.Booleans; import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.bootstrap.JarHell; import org.opensearch.common.io.PathUtils; import org.opensearch.common.network.IfConfig; import org.opensearch.common.network.NetworkAddress; diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java index ca7646c072e70..0626caaaff66c 100644 --- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java @@ -95,10 +95,8 @@ import org.opensearch.index.codec.composite.CompositeIndexFieldInfo; import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.compositeindex.datacube.Metric; -import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeQueryHelper; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexFieldDataCache; -import org.opensearch.index.fielddata.IndexFieldDataService; import org.opensearch.index.mapper.BinaryFieldMapper; import org.opensearch.index.mapper.CompletionFieldMapper; import org.opensearch.index.mapper.CompositeDataCubeFieldType; @@ -129,7 +127,6 @@ import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.SearchOperationListener; import org.opensearch.indices.IndicesModule; -import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache; import org.opensearch.indices.mapper.MapperRegistry; import org.opensearch.plugins.SearchPlugin; import org.opensearch.script.ScriptService; @@ -143,7 +140,6 @@ import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; import org.opensearch.search.aggregations.support.ValuesSourceType; -import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.fetch.subphase.FetchDocValuesPhase; import org.opensearch.search.fetch.subphase.FetchSourcePhase; @@ -162,6 +158,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -349,7 +346,7 @@ protected CountingAggregator createCountingAggregator( IndexSearcher indexSearcher, IndexSettings indexSettings, CompositeIndexFieldInfo starTree, - List supportedDimensions, + LinkedHashMap supportedDimensions, List supportedMetrics, MultiBucketConsumer bucketConsumer, AggregatorFactory aggregatorFactory, @@ -394,7 +391,7 @@ protected SearchContext createSearchContextWithStarTreeContext( Query query, QueryBuilder queryBuilder, CompositeIndexFieldInfo starTree, - List supportedDimensions, + LinkedHashMap supportedDimensions, List supportedMetrics, MultiBucketConsumer bucketConsumer, AggregatorFactory aggregatorFactory, @@ -426,16 +423,23 @@ protected SearchContext createSearchContextWithStarTreeContext( when(compositeMappedFieldType.getCompositeIndexType()).thenReturn(starTree.getType()); Set compositeFieldTypes = Set.of(compositeMappedFieldType); - when((compositeMappedFieldType).getDimensions()).thenReturn(supportedDimensions); + when((compositeMappedFieldType).getDimensions()).thenReturn(new ArrayList<>(supportedDimensions.keySet())); when((compositeMappedFieldType).getMetrics()).thenReturn(supportedMetrics); MapperService mapperService = mock(MapperService.class); when(mapperService.getCompositeFieldTypes()).thenReturn(compositeFieldTypes); when(searchContext.mapperService()).thenReturn(mapperService); - SearchSourceBuilder sb = new SearchSourceBuilder().query(queryBuilder); - StarTreeQueryContext starTreeQueryContext = StarTreeQueryHelper.getStarTreeQueryContext(searchContext, sb); + for (Dimension dimension : supportedDimensions.keySet()) { + when(mapperService.fieldType(dimension.getField())).thenReturn(supportedDimensions.get(dimension)); + when(searchContext.getQueryShardContext().fieldMapper(dimension.getField())).thenReturn(supportedDimensions.get(dimension)); + } + + StarTreeQueryContext starTreeQueryContext = new StarTreeQueryContext(searchContext, queryBuilder); + boolean consolidated = starTreeQueryContext.consolidateAllFilters(searchContext); + if (consolidated) { + searchContext.getQueryShardContext().setStarTreeQueryContext(starTreeQueryContext); + } - when(searchContext.getStarTreeQueryContext()).thenReturn(starTreeQueryContext); return searchContext; } @@ -495,13 +499,6 @@ public boolean shouldCache(Query query) { when(mapperService.getIndexSettings()).thenReturn(indexSettings); when(mapperService.hasNested()).thenReturn(false); when(searchContext.mapperService()).thenReturn(mapperService); - IndexFieldDataService ifds = new IndexFieldDataService( - indexSettings, - new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() { - }), - circuitBreakerService, - mapperService - ); QueryShardContext queryShardContext = queryShardContextMock( contextIndexSearcher, mapperService, @@ -755,7 +752,7 @@ protected A searchAndReduc QueryBuilder queryBuilder, AggregationBuilder builder, CompositeIndexFieldInfo compositeIndexFieldInfo, - List supportedDimensions, + LinkedHashMap supportedDimensions, List supportedMetrics, int maxBucket, boolean hasNested, diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java index 7db9875387500..d153e8d6aef53 100644 --- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java @@ -38,7 +38,7 @@ import org.opensearch.action.index.IndexRequestBuilder; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.support.PlainActionFuture; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateObserver; import org.opensearch.cluster.ClusterStateUpdateTask; diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index cb93622ab8728..e1eafb8570022 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -71,7 +71,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.AdminClient; import org.opensearch.client.Client; import org.opensearch.client.ClusterAdminClient; diff --git a/test/framework/src/main/java/org/opensearch/test/TestCluster.java b/test/framework/src/main/java/org/opensearch/test/TestCluster.java index 8c41e6e5d5b38..f5a439e2ffd02 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/TestCluster.java @@ -38,7 +38,7 @@ import org.opensearch.action.admin.indices.datastream.DeleteDataStreamAction; import org.opensearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.opensearch.action.support.IndicesOptions; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexTemplateMetadata; diff --git a/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java b/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java index 3c145c2bd4e74..cd7669e52cdf8 100644 --- a/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java +++ b/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java @@ -49,8 +49,8 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.action.search.ShardSearchFailure; import org.opensearch.action.support.broadcast.BroadcastResponse; -import org.opensearch.action.support.master.AcknowledgedRequestBuilder; -import org.opensearch.action.support.master.AcknowledgedResponse; +import org.opensearch.action.support.clustermanager.AcknowledgedRequestBuilder; +import org.opensearch.action.support.clustermanager.AcknowledgedResponse; import org.opensearch.cluster.block.ClusterBlock; import org.opensearch.cluster.block.ClusterBlockException; import org.opensearch.cluster.metadata.IndexMetadata;