Skip to content

Commit 7a0e8fb

Browse files
finnegancarrollfinnegancarroll
authored
Add gRPC server as transport-grpc plugin (#16534)
Introduce auxiliary transport to NetworkPlugin and add gRPC plugin. Auxiliary transports are optional lifecycle components provided by network plugins which run in parallel to the http server/native transport. They are distinct from the existing NetworkPlugin interfaces of 'getTransports' and 'getHttpTransports' as auxiliary transports are optional. Each AuxTransport implements it's own 'aux.transport.type' and 'aux.transport.<type>.ports' setting. Since Security.java initializes previous to Node.java during bootstrap socket binding permissions are granted based on 'aux.transport.<type>.ports' for each enabled 'aux.transport.type', falling back to a default if no ports are specified. Signed-off-by: Finn Carroll <carrofin@amazon.com>
1 parent 7ae66d0 commit 7a0e8fb

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1958
-124
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2525
- Add search replica stats to segment replication stats API ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678))
2626
- Introduce a setting to disable download of full cluster state from remote on term mismatch([#16798](https://github.com/opensearch-project/OpenSearch/pull/16798/))
2727
- Added ability to retrieve value from DocValues in a flat_object filed([#16802](https://github.com/opensearch-project/OpenSearch/pull/16802))
28+
- Introduce framework for auxiliary transports and an experimental gRPC transport plugin ([#16534](https://github.com/opensearch-project/OpenSearch/pull/16534))
2829

2930
### Dependencies
3031
- Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))

‎gradle/libs.versions.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ google_http_client = "1.44.1"
2727
google_auth = "1.29.0"
2828
tdigest = "3.3"
2929
hdrhistogram = "2.2.2"
30-
grpc = "1.68.0"
30+
grpc = "1.68.2"
3131

3232
# when updating the JNA version, also update the version in buildSrc/build.gradle
3333
jna = "5.13.0"

‎plugins/discovery-gce/licenses/grpc-api-1.68.0.jar.sha1

-1
This file was deleted.

‎plugins/discovery-gce/licenses/grpc-api-1.68.2.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
a257a5dd25dda1c97a99b56d5b9c1e56c12ae554

‎plugins/repository-gcs/licenses/grpc-api-1.68.0.jar.sha1

-1
This file was deleted.

‎plugins/repository-gcs/licenses/grpc-api-1.68.2.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
a257a5dd25dda1c97a99b56d5b9c1e56c12ae554

‎plugins/transport-grpc/build.gradle

+168
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
import org.gradle.api.attributes.java.TargetJvmEnvironment
2+
3+
/*
4+
* SPDX-License-Identifier: Apache-2.0
5+
*
6+
* The OpenSearch Contributors require contributions made to
7+
* this file be licensed under the Apache-2.0 license or a
8+
* compatible open source license.
9+
*/
10+
11+
opensearchplugin {
12+
description 'gRPC based transport implementation'
13+
classname 'org.opensearch.transport.grpc.GrpcPlugin'
14+
}
15+
16+
dependencies {
17+
compileOnly "com.google.code.findbugs:jsr305:3.0.2"
18+
runtimeOnly "com.google.guava:guava:${versions.guava}"
19+
implementation "com.google.errorprone:error_prone_annotations:2.24.1"
20+
implementation "com.google.guava:failureaccess:1.0.1"
21+
implementation "io.grpc:grpc-api:${versions.grpc}"
22+
implementation "io.grpc:grpc-core:${versions.grpc}"
23+
implementation "io.grpc:grpc-netty-shaded:${versions.grpc}"
24+
implementation "io.grpc:grpc-protobuf-lite:${versions.grpc}"
25+
implementation "io.grpc:grpc-protobuf:${versions.grpc}"
26+
implementation "io.grpc:grpc-services:${versions.grpc}"
27+
implementation "io.grpc:grpc-stub:${versions.grpc}"
28+
implementation "io.grpc:grpc-util:${versions.grpc}"
29+
implementation "io.perfmark:perfmark-api:0.26.0"
30+
}
31+
32+
tasks.named("dependencyLicenses").configure {
33+
mapping from: /grpc-.*/, to: 'grpc'
34+
}
35+
36+
thirdPartyAudit {
37+
ignoreMissingClasses(
38+
'com.aayushatharva.brotli4j.Brotli4jLoader',
39+
'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
40+
'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
41+
'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
42+
'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
43+
'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
44+
// classes are missing
45+
46+
// from io.netty.logging.CommonsLoggerFactory (netty)
47+
'org.apache.commons.logging.Log',
48+
'org.apache.commons.logging.LogFactory',
49+
50+
// from Log4j (deliberate, Netty will fallback to Log4j 2)
51+
'org.apache.log4j.Level',
52+
'org.apache.log4j.Logger',
53+
54+
// from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty)
55+
'org.bouncycastle.cert.X509v3CertificateBuilder',
56+
'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
57+
'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
58+
'org.bouncycastle.openssl.PEMEncryptedKeyPair',
59+
'org.bouncycastle.openssl.PEMParser',
60+
'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
61+
'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
62+
'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
63+
'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
64+
65+
// from io.netty.handler.ssl.JettyNpnSslEngine (netty)
66+
'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
67+
'org.eclipse.jetty.npn.NextProtoNego$ServerProvider',
68+
'org.eclipse.jetty.npn.NextProtoNego',
69+
70+
// from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty)
71+
'org.jboss.marshalling.ByteInput',
72+
73+
// from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty)
74+
'org.jboss.marshalling.ByteOutput',
75+
76+
// from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty)
77+
'org.jboss.marshalling.Marshaller',
78+
79+
// from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty)
80+
'org.jboss.marshalling.MarshallerFactory',
81+
'org.jboss.marshalling.MarshallingConfiguration',
82+
'org.jboss.marshalling.Unmarshaller',
83+
84+
// from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional
85+
'org.slf4j.helpers.FormattingTuple',
86+
'org.slf4j.helpers.MessageFormatter',
87+
'org.slf4j.Logger',
88+
'org.slf4j.LoggerFactory',
89+
'org.slf4j.spi.LocationAwareLogger',
90+
91+
'com.google.gson.stream.JsonReader',
92+
'com.google.gson.stream.JsonToken',
93+
'com.google.protobuf.util.Durations',
94+
'com.google.protobuf.util.Timestamps',
95+
'com.google.protobuf.nano.CodedOutputByteBufferNano',
96+
'com.google.protobuf.nano.MessageNano',
97+
'com.google.rpc.Status',
98+
'com.google.rpc.Status$Builder',
99+
'com.ning.compress.BufferRecycler',
100+
'com.ning.compress.lzf.ChunkDecoder',
101+
'com.ning.compress.lzf.ChunkEncoder',
102+
'com.ning.compress.lzf.LZFChunk',
103+
'com.ning.compress.lzf.LZFEncoder',
104+
'com.ning.compress.lzf.util.ChunkDecoderFactory',
105+
'com.ning.compress.lzf.util.ChunkEncoderFactory',
106+
'lzma.sdk.lzma.Encoder',
107+
'net.jpountz.lz4.LZ4Compressor',
108+
'net.jpountz.lz4.LZ4Factory',
109+
'net.jpountz.lz4.LZ4FastDecompressor',
110+
'net.jpountz.xxhash.XXHash32',
111+
'net.jpountz.xxhash.XXHashFactory',
112+
'org.eclipse.jetty.alpn.ALPN$ClientProvider',
113+
'org.eclipse.jetty.alpn.ALPN$ServerProvider',
114+
'org.eclipse.jetty.alpn.ALPN',
115+
116+
'org.conscrypt.AllocatedBuffer',
117+
'org.conscrypt.BufferAllocator',
118+
'org.conscrypt.Conscrypt',
119+
'org.conscrypt.HandshakeListener',
120+
121+
'reactor.blockhound.BlockHound$Builder',
122+
'reactor.blockhound.integration.BlockHoundIntegration'
123+
)
124+
125+
ignoreViolations(
126+
// uses internal java api: sun.misc.Unsafe
127+
'com.google.common.cache.Striped64',
128+
'com.google.common.cache.Striped64$1',
129+
'com.google.common.cache.Striped64$Cell',
130+
'com.google.common.hash.Striped64',
131+
'com.google.common.hash.Striped64$1',
132+
'com.google.common.hash.Striped64$Cell',
133+
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
134+
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
135+
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
136+
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
137+
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
138+
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
139+
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
140+
141+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
142+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1',
143+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2',
144+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$3',
145+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$4',
146+
'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$5',
147+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0',
148+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$1',
149+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$2',
150+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$3',
151+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$4',
152+
'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$6',
153+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef',
154+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef',
155+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields',
156+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields',
157+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields',
158+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode',
159+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField',
160+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField',
161+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField',
162+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField',
163+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField',
164+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
165+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess',
166+
'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess'
167+
)
168+
}

‎plugins/transport-grpc/licenses/error_prone_annotations-2.24.1.jar.sha1

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
32b299e45105aa9b0df8279c74dc1edfcf313ff0

0 commit comments

Comments
 (0)