Skip to content

Commit 2d9bb3d

Browse files
ruai0511ruai0511
authored
Merge branch 'main' into search-idle
Signed-off-by: Ruirui Zhang <mariazrr@amazon.com>
2 parents ef646c6 + 7bd3715 commit 2d9bb3d

File tree

126 files changed

+5939
-333
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

126 files changed

+5939
-333
lines changed

‎.ci/bwcVersions

+1
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,4 @@ BWC_VERSION:
3030
- "2.12.0"
3131
- "2.12.1"
3232
- "2.13.0"
33+
- "2.14.0"

‎.github/workflows/version.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ jobs:
6262

6363
- name: Create PR for BASE
6464
id: base_pr
65-
uses: peter-evans/create-pull-request@v5
65+
uses: peter-evans/create-pull-request@v6
6666
with:
6767
base: ${{ env.BASE }}
6868
branch: 'create-pull-request/patch-${{ env.BASE }}'
@@ -88,7 +88,7 @@ jobs:
8888

8989
- name: Create PR for BASE_X
9090
id: base_x_pr
91-
uses: peter-evans/create-pull-request@v5
91+
uses: peter-evans/create-pull-request@v6
9292
with:
9393
base: ${{ env.BASE_X }}
9494
branch: 'create-pull-request/patch-${{ env.BASE_X }}'
@@ -114,7 +114,7 @@ jobs:
114114

115115
- name: Create PR for main
116116
id: main_pr
117-
uses: peter-evans/create-pull-request@v5
117+
uses: peter-evans/create-pull-request@v6
118118
with:
119119
base: main
120120
branch: 'create-pull-request/patch-main'

‎CHANGELOG.md

+8-1
Original file line numberDiff line numberDiff line change
@@ -119,10 +119,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
119119
- [Tiered caching] Add serializer integration to allow ehcache disk cache to use non-primitive values ([#12709](https://github.com/opensearch-project/OpenSearch/pull/12709))
120120
- [Admission Control] Integrated IO Based AdmissionController to AdmissionControl Framework ([#12583](https://github.com/opensearch-project/OpenSearch/pull/12583))
121121
- Add a counter to node stat api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768))
122+
- Introduce a new setting `index.check_pending_flush.enabled` to expose the ability to disable the check for pending flushes by write threads ([#12710](https://github.com/opensearch-project/OpenSearch/pull/12710))
123+
- Built-in secure transports support ([#12435](https://github.com/opensearch-project/OpenSearch/pull/12435))
124+
- Lightweight Transport action to verify local term before fetching cluster-state from remote ([#12252](https://github.com/opensearch-project/OpenSearch/pull/12252/))
125+
- Integrate with admission controller for cluster-manager Read API. ([#12496](https://github.com/opensearch-project/OpenSearch/pull/12496))
122126

123127
### Dependencies
124128
- Bump `peter-evans/find-comment` from 2 to 3 ([#12288](https://github.com/opensearch-project/OpenSearch/pull/12288))
125-
- Bump `com.google.api.grpc:proto-google-common-protos` from 2.25.1 to 2.33.0 ([#12289](https://github.com/opensearch-project/OpenSearch/pull/12289))
129+
- Bump `com.google.api.grpc:proto-google-common-protos` from 2.25.1 to 2.37.1 ([#12289](https://github.com/opensearch-project/OpenSearch/pull/12289), [#12365](https://github.com/opensearch-project/OpenSearch/pull/12365))
126130
- Bump `com.squareup.okio:okio` from 3.7.0 to 3.8.0 ([#12290](https://github.com/opensearch-project/OpenSearch/pull/12290))
127131
- Bump `gradle/wrapper-validation-action` from 1 to 2 ([#12367](https://github.com/opensearch-project/OpenSearch/pull/12367))
128132
- Bump `netty` from 4.1.106.Final to 4.1.107.Final ([#12372](https://github.com/opensearch-project/OpenSearch/pull/12372))
@@ -142,6 +146,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
142146
- Bump `aws-sdk-java` from 2.20.55 to 2.20.86 ([#12251](https://github.com/opensearch-project/OpenSearch/pull/12251))
143147
- Bump `reactor-netty` from 1.1.15 to 1.1.17 ([#12633](https://github.com/opensearch-project/OpenSearch/pull/12633))
144148
- Bump `reactor` from 3.5.14 to 3.5.15 ([#12633](https://github.com/opensearch-project/OpenSearch/pull/12633))
149+
- Bump `peter-evans/create-pull-request` from 5 to 6 ([#12724](https://github.com/opensearch-project/OpenSearch/pull/12724))
150+
- Bump `org.apache.commons:commons-configuration2` from 2.9.0 to 2.10.0 ([#12721](https://github.com/opensearch-project/OpenSearch/pull/12721))
151+
- Bump `com.azure:azure-json` from 1.0.1 to 1.1.0 ([#12723](https://github.com/opensearch-project/OpenSearch/pull/12723))
145152

146153
### Changed
147154
- Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499))

‎libs/core/src/main/java/org/opensearch/Version.java

+1
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
101101
public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_2);
102102
public static final Version V_2_12_1 = new Version(2120199, org.apache.lucene.util.Version.LUCENE_9_9_2);
103103
public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0);
104+
public static final Version V_2_14_0 = new Version(2140099, org.apache.lucene.util.Version.LUCENE_9_10_0);
104105
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0);
105106
public static final Version CURRENT = V_3_0_0;
106107

‎modules/transport-netty4/src/main/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransport.java

+169
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
/*
2+
* Copyright 2015-2017 floragunn GmbH
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
*/
17+
18+
/*
19+
* SPDX-License-Identifier: Apache-2.0
20+
*
21+
* The OpenSearch Contributors require contributions made to
22+
* this file be licensed under the Apache-2.0 license or a
23+
* compatible open source license.
24+
*
25+
* Modifications Copyright OpenSearch Contributors. See
26+
* GitHub history for details.
27+
*/
28+
29+
package org.opensearch.http.netty4.ssl;
30+
31+
import org.apache.logging.log4j.LogManager;
32+
import org.apache.logging.log4j.Logger;
33+
import org.opensearch.common.network.NetworkService;
34+
import org.opensearch.common.settings.ClusterSettings;
35+
import org.opensearch.common.settings.Settings;
36+
import org.opensearch.common.util.BigArrays;
37+
import org.opensearch.core.xcontent.NamedXContentRegistry;
38+
import org.opensearch.http.HttpChannel;
39+
import org.opensearch.http.HttpHandlingSettings;
40+
import org.opensearch.http.netty4.Netty4HttpChannel;
41+
import org.opensearch.http.netty4.Netty4HttpServerTransport;
42+
import org.opensearch.plugins.SecureTransportSettingsProvider;
43+
import org.opensearch.telemetry.tracing.Tracer;
44+
import org.opensearch.threadpool.ThreadPool;
45+
import org.opensearch.transport.SharedGroupFactory;
46+
import org.opensearch.transport.netty4.ssl.SslUtils;
47+
48+
import javax.net.ssl.SSLEngine;
49+
50+
import io.netty.channel.Channel;
51+
import io.netty.channel.ChannelHandler;
52+
import io.netty.channel.ChannelHandlerContext;
53+
import io.netty.handler.codec.DecoderException;
54+
import io.netty.handler.ssl.ApplicationProtocolNames;
55+
import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler;
56+
import io.netty.handler.ssl.SslHandler;
57+
58+
/**
59+
* @see <a href="https://github.com/opensearch-project/security/blob/d526c9f6c2a438c14db8b413148204510b9fe2e2/src/main/java/org/opensearch/security/ssl/http/netty/SecuritySSLNettyHttpServerTransport.java">SecuritySSLNettyHttpServerTransport</a>
60+
*/
61+
public class SecureNetty4HttpServerTransport extends Netty4HttpServerTransport {
62+
private static final Logger logger = LogManager.getLogger(SecureNetty4HttpServerTransport.class);
63+
private final SecureTransportSettingsProvider secureTransportSettingsProvider;
64+
private final SecureTransportSettingsProvider.ServerExceptionHandler exceptionHandler;
65+
66+
public SecureNetty4HttpServerTransport(
67+
final Settings settings,
68+
final NetworkService networkService,
69+
final BigArrays bigArrays,
70+
final ThreadPool threadPool,
71+
final NamedXContentRegistry namedXContentRegistry,
72+
final Dispatcher dispatcher,
73+
final ClusterSettings clusterSettings,
74+
final SharedGroupFactory sharedGroupFactory,
75+
final SecureTransportSettingsProvider secureTransportSettingsProvider,
76+
final Tracer tracer
77+
) {
78+
super(
79+
settings,
80+
networkService,
81+
bigArrays,
82+
threadPool,
83+
namedXContentRegistry,
84+
dispatcher,
85+
clusterSettings,
86+
sharedGroupFactory,
87+
tracer
88+
);
89+
this.secureTransportSettingsProvider = secureTransportSettingsProvider;
90+
this.exceptionHandler = secureTransportSettingsProvider.buildHttpServerExceptionHandler(settings, this)
91+
.orElse(SecureTransportSettingsProvider.ServerExceptionHandler.NOOP);
92+
}
93+
94+
@Override
95+
public ChannelHandler configureServerChannelHandler() {
96+
return new SslHttpChannelHandler(this, handlingSettings);
97+
}
98+
99+
@Override
100+
public void onException(HttpChannel channel, Exception cause0) {
101+
Throwable cause = cause0;
102+
103+
if (cause0 instanceof DecoderException && cause0 != null) {
104+
cause = cause0.getCause();
105+
}
106+
107+
exceptionHandler.onError(cause);
108+
logger.error("Exception during establishing a SSL connection: " + cause, cause);
109+
super.onException(channel, cause0);
110+
}
111+
112+
protected class SslHttpChannelHandler extends Netty4HttpServerTransport.HttpChannelHandler {
113+
/**
114+
* Application negotiation handler to select either HTTP 1.1 or HTTP 2 protocol, based
115+
* on client/server ALPN negotiations.
116+
*/
117+
private class Http2OrHttpHandler extends ApplicationProtocolNegotiationHandler {
118+
protected Http2OrHttpHandler() {
119+
super(ApplicationProtocolNames.HTTP_1_1);
120+
}
121+
122+
@Override
123+
protected void configurePipeline(ChannelHandlerContext ctx, String protocol) throws Exception {
124+
if (ApplicationProtocolNames.HTTP_2.equals(protocol)) {
125+
configureDefaultHttp2Pipeline(ctx.pipeline());
126+
} else if (ApplicationProtocolNames.HTTP_1_1.equals(protocol)) {
127+
configureDefaultHttpPipeline(ctx.pipeline());
128+
} else {
129+
throw new IllegalStateException("Unknown application protocol: " + protocol);
130+
}
131+
}
132+
133+
@Override
134+
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
135+
super.exceptionCaught(ctx, cause);
136+
Netty4HttpChannel channel = ctx.channel().attr(HTTP_CHANNEL_KEY).get();
137+
if (channel != null) {
138+
if (cause instanceof Error) {
139+
onException(channel, new Exception(cause));
140+
} else {
141+
onException(channel, (Exception) cause);
142+
}
143+
}
144+
}
145+
}
146+
147+
protected SslHttpChannelHandler(final Netty4HttpServerTransport transport, final HttpHandlingSettings handlingSettings) {
148+
super(transport, handlingSettings);
149+
}
150+
151+
@Override
152+
protected void initChannel(Channel ch) throws Exception {
153+
super.initChannel(ch);
154+
155+
final SSLEngine sslEngine = secureTransportSettingsProvider.buildSecureHttpServerEngine(
156+
settings,
157+
SecureNetty4HttpServerTransport.this
158+
).orElseGet(SslUtils::createDefaultServerSSLEngine);
159+
160+
final SslHandler sslHandler = new SslHandler(sslEngine);
161+
ch.pipeline().addFirst("ssl_http", sslHandler);
162+
}
163+
164+
@Override
165+
protected void configurePipeline(Channel ch) {
166+
ch.pipeline().addLast(new Http2OrHttpHandler());
167+
}
168+
}
169+
}

‎modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java

+64
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,14 @@
4646
import org.opensearch.core.xcontent.NamedXContentRegistry;
4747
import org.opensearch.http.HttpServerTransport;
4848
import org.opensearch.http.netty4.Netty4HttpServerTransport;
49+
import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport;
4950
import org.opensearch.plugins.NetworkPlugin;
5051
import org.opensearch.plugins.Plugin;
52+
import org.opensearch.plugins.SecureTransportSettingsProvider;
5153
import org.opensearch.telemetry.tracing.Tracer;
5254
import org.opensearch.threadpool.ThreadPool;
5355
import org.opensearch.transport.netty4.Netty4Transport;
56+
import org.opensearch.transport.netty4.ssl.SecureNetty4Transport;
5457

5558
import java.util.Arrays;
5659
import java.util.Collections;
@@ -61,7 +64,9 @@
6164
public class Netty4ModulePlugin extends Plugin implements NetworkPlugin {
6265

6366
public static final String NETTY_TRANSPORT_NAME = "netty4";
67+
public static final String NETTY_SECURE_TRANSPORT_NAME = "netty4-secure";
6468
public static final String NETTY_HTTP_TRANSPORT_NAME = "netty4";
69+
public static final String NETTY_SECURE_HTTP_TRANSPORT_NAME = "netty4-secure";
6570

6671
private final SetOnce<SharedGroupFactory> groupFactory = new SetOnce<>();
6772

@@ -144,6 +149,65 @@ public Map<String, Supplier<HttpServerTransport>> getHttpTransports(
144149
);
145150
}
146151

152+
@Override
153+
public Map<String, Supplier<HttpServerTransport>> getSecureHttpTransports(
154+
Settings settings,
155+
ThreadPool threadPool,
156+
BigArrays bigArrays,
157+
PageCacheRecycler pageCacheRecycler,
158+
CircuitBreakerService circuitBreakerService,
159+
NamedXContentRegistry xContentRegistry,
160+
NetworkService networkService,
161+
HttpServerTransport.Dispatcher dispatcher,
162+
ClusterSettings clusterSettings,
163+
SecureTransportSettingsProvider secureTransportSettingsProvider,
164+
Tracer tracer
165+
) {
166+
return Collections.singletonMap(
167+
NETTY_SECURE_HTTP_TRANSPORT_NAME,
168+
() -> new SecureNetty4HttpServerTransport(
169+
settings,
170+
networkService,
171+
bigArrays,
172+
threadPool,
173+
xContentRegistry,
174+
dispatcher,
175+
clusterSettings,
176+
getSharedGroupFactory(settings),
177+
secureTransportSettingsProvider,
178+
tracer
179+
)
180+
);
181+
}
182+
183+
@Override
184+
public Map<String, Supplier<Transport>> getSecureTransports(
185+
Settings settings,
186+
ThreadPool threadPool,
187+
PageCacheRecycler pageCacheRecycler,
188+
CircuitBreakerService circuitBreakerService,
189+
NamedWriteableRegistry namedWriteableRegistry,
190+
NetworkService networkService,
191+
SecureTransportSettingsProvider secureTransportSettingsProvider,
192+
Tracer tracer
193+
) {
194+
return Collections.singletonMap(
195+
NETTY_SECURE_TRANSPORT_NAME,
196+
() -> new SecureNetty4Transport(
197+
settings,
198+
Version.CURRENT,
199+
threadPool,
200+
networkService,
201+
pageCacheRecycler,
202+
namedWriteableRegistry,
203+
circuitBreakerService,
204+
getSharedGroupFactory(settings),
205+
secureTransportSettingsProvider,
206+
tracer
207+
)
208+
);
209+
}
210+
147211
SharedGroupFactory getSharedGroupFactory(Settings settings) {
148212
SharedGroupFactory groupFactory = this.groupFactory.get();
149213
if (groupFactory != null) {

0 commit comments

Comments
 (0)