Skip to content

Commit d4ba4de

Browse files
ruai0511ruai0511
authored
Merge branch 'main' into backport/backport-12768-to-2.x
Signed-off-by: Ruirui Zhang <mariazrr@amazon.com>
2 parents 5839e47 + 8ad0dc0 commit d4ba4de

File tree

395 files changed

+8605
-883
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

395 files changed

+8605
-883
lines changed

‎.ci/bwcVersions

+1
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,4 @@ BWC_VERSION:
3030
- "2.12.0"
3131
- "2.12.1"
3232
- "2.13.0"
33+
- "2.14.0"

‎.github/workflows/version.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ jobs:
6262

6363
- name: Create PR for BASE
6464
id: base_pr
65-
uses: peter-evans/create-pull-request@v5
65+
uses: peter-evans/create-pull-request@v6
6666
with:
6767
base: ${{ env.BASE }}
6868
branch: 'create-pull-request/patch-${{ env.BASE }}'
@@ -88,7 +88,7 @@ jobs:
8888

8989
- name: Create PR for BASE_X
9090
id: base_x_pr
91-
uses: peter-evans/create-pull-request@v5
91+
uses: peter-evans/create-pull-request@v6
9292
with:
9393
base: ${{ env.BASE_X }}
9494
branch: 'create-pull-request/patch-${{ env.BASE_X }}'
@@ -114,7 +114,7 @@ jobs:
114114

115115
- name: Create PR for main
116116
id: main_pr
117-
uses: peter-evans/create-pull-request@v5
117+
uses: peter-evans/create-pull-request@v6
118118
with:
119119
base: main
120120
branch: 'create-pull-request/patch-main'

‎CHANGELOG.md

+12-54
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
1919
- Remote reindex: Add support for configurable retry mechanism ([#12561](https://github.com/opensearch-project/OpenSearch/pull/12561))
2020
- [Admission Control] Integrate IO Usage Tracker to the Resource Usage Collector Service and Emit IO Usage Stats ([#11880](https://github.com/opensearch-project/OpenSearch/pull/11880))
2121
- Tracing for deep search path ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12103))
22+
- Add explicit dependency to validatePom and generatePom tasks ([#12807](https://github.com/opensearch-project/OpenSearch/pull/12807))
23+
- Replace configureEach with all for publication iteration ([#12876](https://github.com/opensearch-project/OpenSearch/pull/12876))
2224

2325
### Dependencies
2426
- Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -101,72 +103,28 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
101103

102104
## [Unreleased 2.x]
103105
### Added
104-
- [Tiered caching] Introducing cache plugins and exposing Ehcache as one of the pluggable disk cache option ([#11874](https://github.com/opensearch-project/OpenSearch/pull/11874))
105-
- Add support for dependencies in plugin descriptor properties with semver range ([#11441](https://github.com/opensearch-project/OpenSearch/pull/11441))
106-
- Add community_id ingest processor ([#12121](https://github.com/opensearch-project/OpenSearch/pull/12121))
107-
- Introduce query level setting `index.query.max_nested_depth` limiting nested queries ([#3268](https://github.com/opensearch-project/OpenSearch/issues/3268)
108-
- Add toString methods to MultiSearchRequest, MultiGetRequest and CreateIndexRequest ([#12163](https://github.com/opensearch-project/OpenSearch/pull/12163))
109-
- Support for returning scores in matched queries ([#11626](https://github.com/opensearch-project/OpenSearch/pull/11626))
110-
- Add shard id property to SearchLookup for use in field types provided by plugins ([#1063](https://github.com/opensearch-project/OpenSearch/pull/1063))
111-
- Force merge API supports performing on primary shards only ([#11269](https://github.com/opensearch-project/OpenSearch/pull/11269))
112-
- [Tiered caching] Make IndicesRequestCache implementation configurable [EXPERIMENTAL] ([#12533](https://github.com/opensearch-project/OpenSearch/pull/12533))
113-
- Add kuromoji_completion analyzer and filter ([#4835](https://github.com/opensearch-project/OpenSearch/issues/4835))
114-
- The org.opensearch.bootstrap.Security should support codebase for JAR files with classifiers ([#12586](https://github.com/opensearch-project/OpenSearch/issues/12586))
115-
- [Metrics Framework] Adds support for asynchronous gauge metric type. ([#12642](https://github.com/opensearch-project/OpenSearch/issues/12642))
116-
- Make search query counters dynamic to support all query types ([#12601](https://github.com/opensearch-project/OpenSearch/pull/12601))
117-
- [Tiered caching] Add policies controlling which values can enter pluggable caches [EXPERIMENTAL] ([#12542](https://github.com/opensearch-project/OpenSearch/pull/12542))
118-
- [Tiered caching] Add Stale keys Management and CacheCleaner to IndicesRequestCache ([#12625](https://github.com/opensearch-project/OpenSearch/pull/12625))
119-
- [Tiered caching] Add serializer integration to allow ehcache disk cache to use non-primitive values ([#12709](https://github.com/opensearch-project/OpenSearch/pull/12709))
120-
- [Admission Control] Integrated IO Based AdmissionController to AdmissionControl Framework ([#12583](https://github.com/opensearch-project/OpenSearch/pull/12583))
106+
- Convert ingest processor supports ip type ([#12818](https://github.com/opensearch-project/OpenSearch/pull/12818))
121107
- Add a counter to node stat api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768))
108+
- Allow setting KEYSTORE_PASSWORD through env variable ([#12865](https://github.com/opensearch-project/OpenSearch/pull/12865))
109+
- [Concurrent Segment Search] Perform buildAggregation concurrently and support Composite Aggregations ([#12697](https://github.com/opensearch-project/OpenSearch/pull/12697))
122110

123111
### Dependencies
124-
- Bump `peter-evans/find-comment` from 2 to 3 ([#12288](https://github.com/opensearch-project/OpenSearch/pull/12288))
125-
- Bump `com.google.api.grpc:proto-google-common-protos` from 2.25.1 to 2.33.0 ([#12289](https://github.com/opensearch-project/OpenSearch/pull/12289))
126-
- Bump `com.squareup.okio:okio` from 3.7.0 to 3.8.0 ([#12290](https://github.com/opensearch-project/OpenSearch/pull/12290))
127-
- Bump `gradle/wrapper-validation-action` from 1 to 2 ([#12367](https://github.com/opensearch-project/OpenSearch/pull/12367))
128-
- Bump `netty` from 4.1.106.Final to 4.1.107.Final ([#12372](https://github.com/opensearch-project/OpenSearch/pull/12372))
129-
- Bump `opentelemetry` from 1.34.1 to 1.36.0 ([#12388](https://github.com/opensearch-project/OpenSearch/pull/12388), [#12618](https://github.com/opensearch-project/OpenSearch/pull/12618))
130-
- Bump Apache Lucene from 9.9.2 to 9.10.0 ([#12392](https://github.com/opensearch-project/OpenSearch/pull/12392))
131-
- Bump `org.apache.logging.log4j:log4j-core` from 2.22.1 to 2.23.1 ([#12464](https://github.com/opensearch-project/OpenSearch/pull/12464), [#12587](https://github.com/opensearch-project/OpenSearch/pull/12587))
132-
- Bump `antlr4` from 4.11.1 to 4.13.1 ([#12445](https://github.com/opensearch-project/OpenSearch/pull/12445))
133-
- Bump `com.netflix.nebula.ospackage-base` from 11.8.0 to 11.8.1 ([#12461](https://github.com/opensearch-project/OpenSearch/pull/12461))
134-
- Bump `peter-evans/create-or-update-comment` from 3 to 4 ([#12462](https://github.com/opensearch-project/OpenSearch/pull/12462))
135-
- Bump `lycheeverse/lychee-action` from 1.9.1 to 1.9.3 ([#12521](https://github.com/opensearch-project/OpenSearch/pull/12521))
136-
- Bump `com.azure:azure-core` from 1.39.0 to 1.47.0 ([#12520](https://github.com/opensearch-project/OpenSearch/pull/12520))
137-
- Bump `ch.qos.logback:logback-core` from 1.2.13 to 1.5.3 ([#12519](https://github.com/opensearch-project/OpenSearch/pull/12519))
138-
- Bump `codecov/codecov-action` from 3 to 4 ([#12585](https://github.com/opensearch-project/OpenSearch/pull/12585))
139-
- Bump `org.apache.zookeeper:zookeeper` from 3.9.1 to 3.9.2 ([#12580](https://github.com/opensearch-project/OpenSearch/pull/12580))
140-
- Bump `org.codehaus.woodstox:stax2-api` from 4.2.1 to 4.2.2 ([#12579](https://github.com/opensearch-project/OpenSearch/pull/12579))
141-
- Bump Jackson version from 2.16.1 to 2.17.0 ([#12611](https://github.com/opensearch-project/OpenSearch/pull/12611), [#12662](https://github.com/opensearch-project/OpenSearch/pull/12662))
142-
- Bump `aws-sdk-java` from 2.20.55 to 2.20.86 ([#12251](https://github.com/opensearch-project/OpenSearch/pull/12251))
143-
- Bump `reactor-netty` from 1.1.15 to 1.1.17 ([#12633](https://github.com/opensearch-project/OpenSearch/pull/12633))
144-
- Bump `reactor` from 3.5.14 to 3.5.15 ([#12633](https://github.com/opensearch-project/OpenSearch/pull/12633))
112+
- Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))
113+
- Bump `asm` from 9.6 to 9.7 ([#12908](https://github.com/opensearch-project/OpenSearch/pull/12908))
114+
- Bump `net.minidev:json-smart` from 2.5.0 to 2.5.1 ([#12893](https://github.com/opensearch-project/OpenSearch/pull/12893))
115+
- Bump `netty` from 4.1.107.Final to 4.1.108.Final ([#12924](https://github.com/opensearch-project/OpenSearch/pull/12924))
145116

146117
### Changed
147-
- Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499))
148-
- Quickly compute terms aggregations when the top-level query is functionally match-all for a segment ([#11643](https://github.com/opensearch-project/OpenSearch/pull/11643))
149-
- Mark fuzzy filter GA and remove experimental setting ([12631](https://github.com/opensearch-project/OpenSearch/pull/12631))
150-
- Keep the election scheduler open until cluster state has been applied ([#11699](https://github.com/opensearch-project/OpenSearch/pull/11699))
118+
- [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
151119

152120
### Deprecated
153121

154122
### Removed
155123

156124
### Fixed
157-
- Fix for deserilization bug in weighted round-robin metadata ([#11679](https://github.com/opensearch-project/OpenSearch/pull/11679))
158-
- [Revert] [Bug] Check phase name before SearchRequestOperationsListener onPhaseStart ([#12035](https://github.com/opensearch-project/OpenSearch/pull/12035))
159-
- Add support of special WrappingSearchAsyncActionPhase so the onPhaseStart() will always be followed by onPhaseEnd() within AbstractSearchAsyncAction ([#12293](https://github.com/opensearch-project/OpenSearch/pull/12293))
160-
- Add a system property to configure YamlParser codepoint limits ([#12298](https://github.com/opensearch-project/OpenSearch/pull/12298))
161-
- Prevent read beyond slice boundary in ByteArrayIndexInput ([#10481](https://github.com/opensearch-project/OpenSearch/issues/10481))
162-
- Fix the "highlight.max_analyzer_offset" request parameter with "plain" highlighter ([#10919](https://github.com/opensearch-project/OpenSearch/pull/10919))
163-
- Prevent unnecessary fetch sub phase processor initialization during fetch phase execution ([#12503](https://github.com/opensearch-project/OpenSearch/pull/12503))
164-
- Warn about deprecated and ignored index.mapper.dynamic index setting ([#11193](https://github.com/opensearch-project/OpenSearch/pull/11193))
165-
- Fix `terms` query on `float` field when `doc_values` are turned off by reverting back to `FloatPoint` from `FloatField` ([#12499](https://github.com/opensearch-project/OpenSearch/pull/12499))
166-
- Fix get task API does not refresh resource stats ([#11531](https://github.com/opensearch-project/OpenSearch/pull/11531))
167-
- onShardResult and onShardFailure are executed on one shard causes opensearch jvm crashed ([#12158](https://github.com/opensearch-project/OpenSearch/pull/12158))
125+
- Fix issue with feature flags where default value may not be honored ([#12849](https://github.com/opensearch-project/OpenSearch/pull/12849))
168126

169127
### Security
170128

171129
[Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
172-
[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
130+
[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.13...2.x

‎README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
[![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
88
[![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
99
[![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
10-
[![2.10 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.10.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.10.0")
11-
[![3.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
10+
[![2.14.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.14.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.14.0")
11+
[![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
1212
[![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
1313
[![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)
1414
[![GHA precommit](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml)

‎buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java

+5-4
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,6 @@ public void apply(Project project) {
6565
addLocalMavenRepo(project);
6666
addZipArtifact(project);
6767
Task validatePluginZipPom = project.getTasks().findByName("validatePluginZipPom");
68-
if (validatePluginZipPom != null) {
69-
validatePluginZipPom.dependsOn("generatePomFileForNebulaPublication");
70-
}
7168

7269
// There are number of tasks prefixed by 'publishPluginZipPublication', f.e.:
7370
// publishPluginZipPublicationToZipStagingRepository, publishPluginZipPublicationToMavenLocal
@@ -76,7 +73,11 @@ public void apply(Project project) {
7673
.filter(t -> t.getName().startsWith("publishPluginZipPublicationTo"))
7774
.collect(Collectors.toSet());
7875
if (!publishPluginZipPublicationToTasks.isEmpty()) {
79-
publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn("generatePomFileForNebulaPublication"));
76+
if (validatePluginZipPom != null) {
77+
publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn(validatePluginZipPom));
78+
} else {
79+
publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn("generatePomFileForNebulaPublication"));
80+
}
8081
}
8182
} else {
8283
project.getLogger()

‎buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationPrecommitPlugin.java

+11-4
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,19 @@ public TaskProvider<? extends Task> createTask(Project project) {
5353
TaskProvider<PomValidationTask> validateTask = project.getTasks()
5454
.register("validate" + publicationName + "Pom", PomValidationTask.class);
5555
validatePom.configure(t -> t.dependsOn(validateTask));
56+
TaskProvider<GenerateMavenPom> generateMavenPom = project.getTasks()
57+
.withType(GenerateMavenPom.class)
58+
.named("generatePomFileFor" + publicationName + "Publication");
5659
validateTask.configure(task -> {
57-
GenerateMavenPom generateMavenPom = project.getTasks()
58-
.withType(GenerateMavenPom.class)
59-
.getByName("generatePomFileFor" + publicationName + "Publication");
6060
task.dependsOn(generateMavenPom);
61-
task.getPomFile().fileValue(generateMavenPom.getDestination());
61+
task.getPomFile().fileProvider(generateMavenPom.map(GenerateMavenPom::getDestination));
62+
publishing.getPublications().all(publicationForPomGen -> {
63+
task.mustRunAfter(
64+
project.getTasks()
65+
.withType(GenerateMavenPom.class)
66+
.getByName("generatePomFileFor" + Util.capitalize(publicationForPomGen.getName()) + "Publication")
67+
);
68+
});
6269
});
6370
});
6471

‎buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationTask.java

+1
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ private <T> void validateNonNull(String element, T value, Runnable validator) {
106106

107107
private void validateString(String element, String value) {
108108
validateNonNull(element, value, () -> validateNonEmpty(element, value, s -> s.trim().isEmpty()));
109+
getLogger().info(element + " with value " + value + " is validated.");
109110
}
110111

111112
private <T> void validateCollection(String element, Collection<T> value, Consumer<T> validator) {

‎buildSrc/src/testFixtures/java/org/opensearch/gradle/test/GradleThreadsFilter.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ public class GradleThreadsFilter implements ThreadFilter {
4545
public boolean reject(Thread t) {
4646
return t.getName().startsWith("Exec process")
4747
|| t.getName().startsWith("Memory manager")
48-
|| t.getName().startsWith("File watcher consumer");
48+
|| t.getName().startsWith("File watcher consumer")
49+
|| t.getName().startsWith("sshd-SshClient") /* Started by SshClient (sshd-core), part of SftpFileSystemProvider */
50+
|| t.getName().startsWith("Thread-"); /* Started by AbstractFactoryManager (sshd-core), part of SftpFileSystemProvider */
4951
}
5052
}

‎buildSrc/version.properties

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ icu4j = 70.1
1414
supercsv = 2.4.0
1515
log4j = 2.21.0
1616
slf4j = 1.7.36
17-
asm = 9.6
17+
asm = 9.7
1818
jettison = 1.5.4
1919
woodstox = 6.4.0
2020
kotlin = 1.7.10
@@ -26,7 +26,7 @@ jakarta_annotation = 1.3.5
2626
# when updating the JNA version, also update the version in buildSrc/build.gradle
2727
jna = 5.13.0
2828

29-
netty = 4.1.107.Final
29+
netty = 4.1.108.Final
3030
joda = 2.12.2
3131

3232
# project reactor

‎distribution/src/bin/opensearch

+7-5
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,16 @@ fi
3636

3737
# get keystore password before setting java options to avoid
3838
# conflicting GC configurations for the keystore tools
39-
unset KEYSTORE_PASSWORD
40-
KEYSTORE_PASSWORD=
4139
if [[ $CHECK_KEYSTORE = true ]] \
4240
&& bin/opensearch-keystore has-passwd --silent
4341
then
44-
if ! read -s -r -p "OpenSearch keystore password: " KEYSTORE_PASSWORD ; then
45-
echo "Failed to read keystore password on console" 1>&2
46-
exit 1
42+
if [[ ! -z "${KEYSTORE_PASSWORD}" ]]; then
43+
echo "Using value of KEYSTORE_PASSWORD from the environment"
44+
else
45+
if ! read -s -r -p "OpenSearch keystore password: " KEYSTORE_PASSWORD ; then
46+
echo "Failed to read keystore password on console" 1>&2
47+
exit 1
48+
fi
4749
fi
4850
fi
4951

‎distribution/src/bin/opensearch.bat

+8-5
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,17 @@ if not exist "%SERVICE_LOG_DIR%" (
6262
mkdir "%SERVICE_LOG_DIR%"
6363
)
6464

65-
SET KEYSTORE_PASSWORD=
6665
IF "%checkpassword%"=="Y" (
6766
CALL "%~dp0opensearch-keystore.bat" has-passwd --silent
6867
IF !ERRORLEVEL! EQU 0 (
69-
SET /P KEYSTORE_PASSWORD=OpenSearch keystore password:
70-
IF !ERRORLEVEL! NEQ 0 (
71-
ECHO Failed to read keystore password on standard input
72-
EXIT /B !ERRORLEVEL!
68+
if defined KEYSTORE_PASSWORD (
69+
ECHO Using value of KEYSTORE_PASSWORD from the environment
70+
) else (
71+
SET /P KEYSTORE_PASSWORD=OpenSearch keystore password:
72+
IF !ERRORLEVEL! NEQ 0 (
73+
ECHO Failed to read keystore password on standard input
74+
EXIT /B !ERRORLEVEL!
75+
)
7376
)
7477
)
7578
)

‎gradle/wrapper/gradle-wrapper.properties

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
distributionBase=GRADLE_USER_HOME
1313
distributionPath=wrapper/dists
14-
distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-all.zip
14+
distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
1515
zipStoreBase=GRADLE_USER_HOME
1616
zipStorePath=wrapper/dists
17-
distributionSha256Sum=85719317abd2112f021d4f41f09ec370534ba288432065f4b477b6a3b652910d
17+
distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0

0 commit comments

Comments
 (0)