Skip to content

Commit bf11aad

Browse files
kiranprakash154kiranprakash154
committed
Merge branch 'kp/irc-stalekeymgmt-thresh-dyn' into kp/integration-test-stalekeys
# Conflicts: # CHANGELOG.md # server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java # server/src/main/java/org/opensearch/common/settings/ClusterSettings.java # server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
2 parents b04632b + bc00536 commit bf11aad

File tree

53 files changed

+1888
-340
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+1888
-340
lines changed

‎CHANGELOG.md

+5
Original file line numberDiff line numberDiff line change
@@ -103,9 +103,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
103103

104104
## [Unreleased 2.x]
105105
### Added
106+
- Convert ingest processor supports ip type ([#12818](https://github.com/opensearch-project/OpenSearch/pull/12818))
106107
- Add a counter to node stat api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768))
108+
- Allow setting KEYSTORE_PASSWORD through env variable ([#12865](https://github.com/opensearch-project/OpenSearch/pull/12865))
107109
- [Concurrent Segment Search] Perform buildAggregation concurrently and support Composite Aggregations ([#12697](https://github.com/opensearch-project/OpenSearch/pull/12697))
108110
- [Tiered caching] Add Integration Tests for Indices Request Cache Stale Key Management ([#12941](https://github.com/opensearch-project/OpenSearch/pull/12941))
111+
- [Concurrent Segment Search] Disable concurrent segment search for system indices and throttled requests ([#12954](https://github.com/opensearch-project/OpenSearch/pull/12954))
112+
- [Tiered Caching] Make Indices Request Cache Stale Key Mgmt Threshold setting dynamic ([#12975](https://github.com/opensearch-project/OpenSearch/pull/12975))
109113

110114
### Dependencies
111115
- Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))
@@ -115,6 +119,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
115119

116120
### Changed
117121
- [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
122+
- Improve built-in secure transports support ([#12907](https://github.com/opensearch-project/OpenSearch/pull/12907))
118123

119124
### Deprecated
120125

‎distribution/src/bin/opensearch

+7-5
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,16 @@ fi
3636

3737
# get keystore password before setting java options to avoid
3838
# conflicting GC configurations for the keystore tools
39-
unset KEYSTORE_PASSWORD
40-
KEYSTORE_PASSWORD=
4139
if [[ $CHECK_KEYSTORE = true ]] \
4240
&& bin/opensearch-keystore has-passwd --silent
4341
then
44-
if ! read -s -r -p "OpenSearch keystore password: " KEYSTORE_PASSWORD ; then
45-
echo "Failed to read keystore password on console" 1>&2
46-
exit 1
42+
if [[ ! -z "${KEYSTORE_PASSWORD}" ]]; then
43+
echo "Using value of KEYSTORE_PASSWORD from the environment"
44+
else
45+
if ! read -s -r -p "OpenSearch keystore password: " KEYSTORE_PASSWORD ; then
46+
echo "Failed to read keystore password on console" 1>&2
47+
exit 1
48+
fi
4749
fi
4850
fi
4951

‎distribution/src/bin/opensearch.bat

+8-5
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,17 @@ if not exist "%SERVICE_LOG_DIR%" (
6262
mkdir "%SERVICE_LOG_DIR%"
6363
)
6464

65-
SET KEYSTORE_PASSWORD=
6665
IF "%checkpassword%"=="Y" (
6766
CALL "%~dp0opensearch-keystore.bat" has-passwd --silent
6867
IF !ERRORLEVEL! EQU 0 (
69-
SET /P KEYSTORE_PASSWORD=OpenSearch keystore password:
70-
IF !ERRORLEVEL! NEQ 0 (
71-
ECHO Failed to read keystore password on standard input
72-
EXIT /B !ERRORLEVEL!
68+
if defined KEYSTORE_PASSWORD (
69+
ECHO Using value of KEYSTORE_PASSWORD from the environment
70+
) else (
71+
SET /P KEYSTORE_PASSWORD=OpenSearch keystore password:
72+
IF !ERRORLEVEL! NEQ 0 (
73+
ECHO Failed to read keystore password on standard input
74+
EXIT /B !ERRORLEVEL!
75+
)
7376
)
7477
)
7578
)

‎modules/ingest-common/src/main/java/org/opensearch/ingest/common/ConvertProcessor.java

+14
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
package org.opensearch.ingest.common;
3434

35+
import org.opensearch.common.network.InetAddresses;
3536
import org.opensearch.ingest.AbstractProcessor;
3637
import org.opensearch.ingest.ConfigurationUtils;
3738
import org.opensearch.ingest.IngestDocument;
@@ -118,6 +119,19 @@ public Object convert(Object value) {
118119
return value.toString();
119120
}
120121
},
122+
IP {
123+
@Override
124+
public Object convert(Object value) {
125+
// If the value is a valid ipv4/ipv6 address, we return the original value directly because IpFieldType
126+
// can accept string value, this is simpler than we return an InetAddress object which needs to do more
127+
// work such as serialization
128+
if (value instanceof String && InetAddresses.isInetAddress(value.toString())) {
129+
return value;
130+
} else {
131+
throw new IllegalArgumentException("[" + value + "] is not a valid ipv4/ipv6 address");
132+
}
133+
}
134+
},
121135
AUTO {
122136
@Override
123137
public Object convert(Object value) {

‎modules/ingest-common/src/test/java/org/opensearch/ingest/common/ConvertProcessorTests.java

+25
Original file line numberDiff line numberDiff line change
@@ -550,4 +550,29 @@ public void testTargetField() throws Exception {
550550
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(String.valueOf(randomInt)));
551551
assertThat(ingestDocument.getFieldValue(targetField, Integer.class), equalTo(randomInt));
552552
}
553+
554+
public void testConvertIP() throws Exception {
555+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
556+
String validIPString;
557+
if (randomBoolean()) {
558+
validIPString = "1.2.3.4";
559+
} else {
560+
validIPString = "::1";
561+
}
562+
String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, validIPString);
563+
564+
Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.IP, false);
565+
processor.execute(ingestDocument);
566+
assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(validIPString));
567+
568+
String invalidIPString = randomAlphaOfLength(10);
569+
fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, invalidIPString);
570+
Processor processorWithInvalidIP = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.IP, false);
571+
try {
572+
processorWithInvalidIP.execute(ingestDocument);
573+
fail("processor execute should have failed");
574+
} catch (IllegalArgumentException e) {
575+
assertThat(e.getMessage(), equalTo("[" + invalidIPString + "] is not a valid ipv4/ipv6 address"));
576+
}
577+
}
553578
}

‎modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_convert_processor.yml

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
---
2+
teardown:
3+
- do:
4+
ingest.delete_pipeline:
5+
id: "1"
6+
ignore: 404
7+
8+
---
9+
"Test convert processor with ip type":
10+
- skip:
11+
version: " - 2.13.99"
12+
reason: "introduced in 2.14.0"
13+
- do:
14+
ingest.put_pipeline:
15+
id: "1"
16+
body: >
17+
{
18+
"processors": [
19+
{
20+
"convert" : {
21+
"field" : "raw_ip",
22+
"type": "ip"
23+
}
24+
}
25+
]
26+
}
27+
- match: { acknowledged: true }
28+
29+
- do:
30+
catch: /\[1.1.1.\] is not a valid ipv4\/ipv6 address/
31+
index:
32+
index: test
33+
id: 1
34+
pipeline: "1"
35+
body: {
36+
raw_ip: "1.1.1."
37+
}
38+
39+
- do:
40+
ingest.put_pipeline:
41+
id: "1"
42+
body: >
43+
{
44+
"processors": [
45+
{
46+
"convert" : {
47+
"field" : "raw_ip",
48+
"target_field" : "ip_field",
49+
"type" : "ip",
50+
"ignore_failure" : true
51+
}
52+
}
53+
]
54+
}
55+
- match: { acknowledged: true }
56+
57+
- do:
58+
index:
59+
index: test
60+
id: 1
61+
pipeline: "1"
62+
body: {
63+
raw_ip: "1.1.1."
64+
}
65+
- do:
66+
get:
67+
index: test
68+
id: 1
69+
- match: { _source: { raw_ip: "1.1.1."} }
70+
71+
- do:
72+
index:
73+
index: test
74+
id: 1
75+
pipeline: "1"
76+
body: {
77+
raw_ip: "1.1.1.1"
78+
}
79+
- do:
80+
get:
81+
index: test
82+
id: 1
83+
- match: { _source: { raw_ip: "1.1.1.1", ip_field: "1.1.1.1"} }

‎modules/transport-netty4/src/main/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransport.java

+70-8
Original file line numberDiff line numberDiff line change
@@ -37,19 +37,27 @@
3737
import org.opensearch.core.xcontent.NamedXContentRegistry;
3838
import org.opensearch.http.HttpChannel;
3939
import org.opensearch.http.HttpHandlingSettings;
40+
import org.opensearch.http.HttpServerTransport;
4041
import org.opensearch.http.netty4.Netty4HttpChannel;
4142
import org.opensearch.http.netty4.Netty4HttpServerTransport;
42-
import org.opensearch.plugins.SecureTransportSettingsProvider;
43+
import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
44+
import org.opensearch.plugins.TransportExceptionHandler;
4345
import org.opensearch.telemetry.tracing.Tracer;
4446
import org.opensearch.threadpool.ThreadPool;
4547
import org.opensearch.transport.SharedGroupFactory;
48+
import org.opensearch.transport.TransportAdapterProvider;
4649
import org.opensearch.transport.netty4.ssl.SslUtils;
4750

4851
import javax.net.ssl.SSLEngine;
4952

53+
import java.util.List;
54+
import java.util.Optional;
55+
import java.util.stream.Collectors;
56+
5057
import io.netty.channel.Channel;
5158
import io.netty.channel.ChannelHandler;
5259
import io.netty.channel.ChannelHandlerContext;
60+
import io.netty.channel.ChannelInboundHandlerAdapter;
5361
import io.netty.handler.codec.DecoderException;
5462
import io.netty.handler.ssl.ApplicationProtocolNames;
5563
import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler;
@@ -59,9 +67,14 @@
5967
* @see <a href="https://github.com/opensearch-project/security/blob/d526c9f6c2a438c14db8b413148204510b9fe2e2/src/main/java/org/opensearch/security/ssl/http/netty/SecuritySSLNettyHttpServerTransport.java">SecuritySSLNettyHttpServerTransport</a>
6068
*/
6169
public class SecureNetty4HttpServerTransport extends Netty4HttpServerTransport {
70+
public static final String REQUEST_HEADER_VERIFIER = "HeaderVerifier";
71+
public static final String REQUEST_DECOMPRESSOR = "RequestDecompressor";
72+
6273
private static final Logger logger = LogManager.getLogger(SecureNetty4HttpServerTransport.class);
63-
private final SecureTransportSettingsProvider secureTransportSettingsProvider;
64-
private final SecureTransportSettingsProvider.ServerExceptionHandler exceptionHandler;
74+
private final SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider;
75+
private final TransportExceptionHandler exceptionHandler;
76+
private final ChannelInboundHandlerAdapter headerVerifier;
77+
private final TransportAdapterProvider<HttpServerTransport> decompressorProvider;
6578

6679
public SecureNetty4HttpServerTransport(
6780
final Settings settings,
@@ -72,7 +85,7 @@ public SecureNetty4HttpServerTransport(
7285
final Dispatcher dispatcher,
7386
final ClusterSettings clusterSettings,
7487
final SharedGroupFactory sharedGroupFactory,
75-
final SecureTransportSettingsProvider secureTransportSettingsProvider,
88+
final SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider,
7689
final Tracer tracer
7790
) {
7891
super(
@@ -86,9 +99,45 @@ public SecureNetty4HttpServerTransport(
8699
sharedGroupFactory,
87100
tracer
88101
);
89-
this.secureTransportSettingsProvider = secureTransportSettingsProvider;
90-
this.exceptionHandler = secureTransportSettingsProvider.buildHttpServerExceptionHandler(settings, this)
91-
.orElse(SecureTransportSettingsProvider.ServerExceptionHandler.NOOP);
102+
103+
this.secureHttpTransportSettingsProvider = secureHttpTransportSettingsProvider;
104+
this.exceptionHandler = secureHttpTransportSettingsProvider.buildHttpServerExceptionHandler(settings, this)
105+
.orElse(TransportExceptionHandler.NOOP);
106+
107+
final List<ChannelInboundHandlerAdapter> headerVerifiers = secureHttpTransportSettingsProvider.getHttpTransportAdapterProviders(
108+
settings
109+
)
110+
.stream()
111+
.filter(p -> REQUEST_HEADER_VERIFIER.equalsIgnoreCase(p.name()))
112+
.map(p -> p.create(settings, this, ChannelInboundHandlerAdapter.class))
113+
.filter(Optional::isPresent)
114+
.map(Optional::get)
115+
.collect(Collectors.toList());
116+
117+
if (headerVerifiers.size() > 1) {
118+
throw new IllegalArgumentException("Cannot have more than one header verifier configured, supplied " + headerVerifiers.size());
119+
}
120+
121+
final Optional<TransportAdapterProvider<HttpServerTransport>> decompressorProviderOpt = secureHttpTransportSettingsProvider
122+
.getHttpTransportAdapterProviders(settings)
123+
.stream()
124+
.filter(p -> REQUEST_DECOMPRESSOR.equalsIgnoreCase(p.name()))
125+
.findFirst();
126+
// There could be multiple request decompressor providers configured, using the first one
127+
decompressorProviderOpt.ifPresent(p -> logger.debug("Using request decompressor provider: {}", p));
128+
129+
this.headerVerifier = headerVerifiers.isEmpty() ? null : headerVerifiers.get(0);
130+
this.decompressorProvider = decompressorProviderOpt.orElseGet(() -> new TransportAdapterProvider<HttpServerTransport>() {
131+
@Override
132+
public String name() {
133+
return REQUEST_DECOMPRESSOR;
134+
}
135+
136+
@Override
137+
public <C> Optional<C> create(Settings settings, HttpServerTransport transport, Class<C> adapterClass) {
138+
return Optional.empty();
139+
}
140+
});
92141
}
93142

94143
@Override
@@ -152,7 +201,7 @@ protected SslHttpChannelHandler(final Netty4HttpServerTransport transport, final
152201
protected void initChannel(Channel ch) throws Exception {
153202
super.initChannel(ch);
154203

155-
final SSLEngine sslEngine = secureTransportSettingsProvider.buildSecureHttpServerEngine(
204+
final SSLEngine sslEngine = secureHttpTransportSettingsProvider.buildSecureHttpServerEngine(
156205
settings,
157206
SecureNetty4HttpServerTransport.this
158207
).orElseGet(SslUtils::createDefaultServerSSLEngine);
@@ -166,4 +215,17 @@ protected void configurePipeline(Channel ch) {
166215
ch.pipeline().addLast(new Http2OrHttpHandler());
167216
}
168217
}
218+
219+
protected ChannelInboundHandlerAdapter createHeaderVerifier() {
220+
if (headerVerifier != null) {
221+
return headerVerifier;
222+
} else {
223+
return super.createHeaderVerifier();
224+
}
225+
}
226+
227+
@Override
228+
protected ChannelInboundHandlerAdapter createDecompressor() {
229+
return decompressorProvider.create(settings, this, ChannelInboundHandlerAdapter.class).orElseGet(super::createDecompressor);
230+
}
169231
}

‎modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport;
5050
import org.opensearch.plugins.NetworkPlugin;
5151
import org.opensearch.plugins.Plugin;
52+
import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
5253
import org.opensearch.plugins.SecureTransportSettingsProvider;
5354
import org.opensearch.telemetry.tracing.Tracer;
5455
import org.opensearch.threadpool.ThreadPool;
@@ -160,7 +161,7 @@ public Map<String, Supplier<HttpServerTransport>> getSecureHttpTransports(
160161
NetworkService networkService,
161162
HttpServerTransport.Dispatcher dispatcher,
162163
ClusterSettings clusterSettings,
163-
SecureTransportSettingsProvider secureTransportSettingsProvider,
164+
SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider,
164165
Tracer tracer
165166
) {
166167
return Collections.singletonMap(
@@ -174,7 +175,7 @@ public Map<String, Supplier<HttpServerTransport>> getSecureHttpTransports(
174175
dispatcher,
175176
clusterSettings,
176177
getSharedGroupFactory(settings),
177-
secureTransportSettingsProvider,
178+
secureHttpTransportSettingsProvider,
178179
tracer
179180
)
180181
);

‎modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
4343
import org.opensearch.core.indices.breaker.CircuitBreakerService;
4444
import org.opensearch.plugins.SecureTransportSettingsProvider;
45+
import org.opensearch.plugins.TransportExceptionHandler;
4546
import org.opensearch.telemetry.tracing.Tracer;
4647
import org.opensearch.threadpool.ThreadPool;
4748
import org.opensearch.transport.SharedGroupFactory;
@@ -72,7 +73,7 @@ public class SecureNetty4Transport extends Netty4Transport {
7273

7374
private static final Logger logger = LogManager.getLogger(SecureNetty4Transport.class);
7475
private final SecureTransportSettingsProvider secureTransportSettingsProvider;
75-
private final SecureTransportSettingsProvider.ServerExceptionHandler exceptionHandler;
76+
private final TransportExceptionHandler exceptionHandler;
7677

7778
public SecureNetty4Transport(
7879
final Settings settings,
@@ -100,7 +101,7 @@ public SecureNetty4Transport(
100101

101102
this.secureTransportSettingsProvider = secureTransportSettingsProvider;
102103
this.exceptionHandler = secureTransportSettingsProvider.buildServerTransportExceptionHandler(settings, this)
103-
.orElse(SecureTransportSettingsProvider.ServerExceptionHandler.NOOP);
104+
.orElse(TransportExceptionHandler.NOOP);
104105
}
105106

106107
@Override

0 commit comments

Comments
 (0)