Add AWS v2 SDK and netty dependencies in repository-s3 plugin

Signed-off-by: Raghuvansh Raj <raghraaj@amazon.com>
opensearch-project · Jun 14, 2023 · b50eab3 · b50eab3
1 parent aa21585
commit b50eab3
Show file tree

Hide file tree

Showing 14 changed files with 529 additions and 50 deletions.
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
@@ -46,6 +46,7 @@ opensearchplugin {
 }
 
 dependencies {
+  // aws sdk v2 stack
   api "software.amazon.awssdk:sdk-core:${versions.aws}"
   api "software.amazon.awssdk:annotations:${versions.aws}"
   api "software.amazon.awssdk:aws-core:${versions.aws}"
@@ -67,6 +68,8 @@ dependencies {
   api "software.amazon.awssdk:aws-json-protocol:${versions.aws}"
   api "software.amazon.awssdk:aws-query-protocol:${versions.aws}"
   api "software.amazon.awssdk:sts:${versions.aws}"
+  api "software.amazon.awssdk:netty-nio-client:${versions.aws}"
+
   api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
@@ -78,8 +81,21 @@ dependencies {
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
   api "joda-time:joda-time:${versions.joda}"
+  api "org.reactivestreams:reactive-streams:${versions.reactivestreams}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
 
+  // network stack
+  api "io.netty:netty-buffer:${versions.netty}"
+  api "io.netty:netty-codec:${versions.netty}"
+  api "io.netty:netty-codec-http:${versions.netty}"
+  api "io.netty:netty-codec-http2:${versions.netty}"
+  api "io.netty:netty-common:${versions.netty}"
+  api "io.netty:netty-handler:${versions.netty}"
+  api "io.netty:netty-resolver:${versions.netty}"
+  api "io.netty:netty-transport:${versions.netty}"
+  api "io.netty:netty-transport-native-unix-common:${versions.netty}"
+  api "io.netty:netty-transport-classes-epoll:${versions.netty}"
+
   // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
   // and allowlist this hack in JarHell
   api 'javax.xml.bind:jaxb-api:2.3.1'
@@ -96,6 +112,7 @@ restResources {
 tasks.named("dependencyLicenses").configure {
   mapping from: /jackson-.*/, to: 'jackson'
   mapping from: /jaxb-.*/, to: 'jaxb'
+  mapping from: /netty-.*/, to: 'netty'
 }
 
 bundlePlugin {
@@ -375,56 +392,189 @@ TaskProvider s3ThirdPartyTest = tasks.register("s3ThirdPartyTest", Test) {
 }
 tasks.named("check").configure { dependsOn(s3ThirdPartyTest) }
 
-thirdPartyAudit.ignoreMissingClasses(
-  // classes are missing
-  'javax.jms.Message',
-  'javax.servlet.ServletContextEvent',
-  'javax.servlet.ServletContextListener',
-  'org.apache.avalon.framework.logger.Logger',
-  'org.apache.log.Hierarchy',
-  'org.apache.log.Logger',
-  'org.slf4j.impl.StaticLoggerBinder',
-  'org.slf4j.impl.StaticMDCBinder',
-  'org.slf4j.impl.StaticMarkerBinder',
-  'software.amazon.awssdk.arns.Arn',
-  'software.amazon.awssdk.arns.ArnResource',
-  'software.amazon.awssdk.crt.CRT',
-  'software.amazon.awssdk.crt.auth.credentials.Credentials',
-  'software.amazon.awssdk.crt.auth.credentials.CredentialsProvider',
-  'software.amazon.awssdk.crt.auth.credentials.DelegateCredentialsProvider$DelegateCredentialsProviderBuilder',
-  'software.amazon.awssdk.crt.http.HttpHeader',
-  'software.amazon.awssdk.crt.http.HttpMonitoringOptions',
-  'software.amazon.awssdk.crt.http.HttpProxyOptions',
-  'software.amazon.awssdk.crt.http.HttpRequest',
-  'software.amazon.awssdk.crt.http.HttpRequestBodyStream',
-  'software.amazon.awssdk.crt.io.ClientBootstrap',
-  'software.amazon.awssdk.crt.io.ExponentialBackoffRetryOptions',
-  'software.amazon.awssdk.crt.io.StandardRetryOptions',
-  'software.amazon.awssdk.crt.io.TlsCipherPreference',
-  'software.amazon.awssdk.crt.io.TlsContext',
-  'software.amazon.awssdk.crt.io.TlsContextOptions',
-  'software.amazon.awssdk.crt.s3.ChecksumAlgorithm',
-  'software.amazon.awssdk.crt.s3.ChecksumConfig',
-  'software.amazon.awssdk.crt.s3.ChecksumConfig$ChecksumLocation',
-  'software.amazon.awssdk.crt.s3.ResumeToken',
-  'software.amazon.awssdk.crt.s3.S3Client',
-  'software.amazon.awssdk.crt.s3.S3ClientOptions',
-  'software.amazon.awssdk.crt.s3.S3FinishedResponseContext',
-  'software.amazon.awssdk.crt.s3.S3MetaRequest',
-  'software.amazon.awssdk.crt.s3.S3MetaRequestOptions',
-  'software.amazon.awssdk.crt.s3.S3MetaRequestOptions$MetaRequestType',
-  'software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandler',
-  'software.amazon.awssdk.crtcore.CrtConfigurationUtils',
-  'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration',
-  'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$Builder',
-  'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$DefaultBuilder',
-  'software.amazon.awssdk.crtcore.CrtProxyConfiguration',
-  'software.amazon.awssdk.crtcore.CrtProxyConfiguration$Builder',
-  'software.amazon.awssdk.crtcore.CrtProxyConfiguration$DefaultBuilder',
-  'software.amazon.eventstream.HeaderValue',
-  'software.amazon.eventstream.Message',
-  'software.amazon.eventstream.MessageDecoder'
-)
+thirdPartyAudit {
+  ignoreMissingClasses(
+    // classes are missing
+    'javax.jms.Message',
+    'javax.servlet.ServletContextEvent',
+    'javax.servlet.ServletContextListener',
+    'org.apache.avalon.framework.logger.Logger',
+    'org.apache.log.Hierarchy',
+    'org.apache.log.Logger',
+
+    // from io.netty.handler.ssl.OpenSslEngine (netty)
+    'io.netty.internal.tcnative.AsyncSSLPrivateKeyMethod',
+    'io.netty.internal.tcnative.AsyncTask',
+    'io.netty.internal.tcnative.Buffer',
+    'io.netty.internal.tcnative.CertificateCompressionAlgo',
+    'io.netty.internal.tcnative.Library',
+    'io.netty.internal.tcnative.ResultCallback',
+    'io.netty.internal.tcnative.SSL',
+    'io.netty.internal.tcnative.SSLContext',
+    'io.netty.internal.tcnative.SSLPrivateKeyMethod',
+    'io.netty.internal.tcnative.SSLSession',
+    'io.netty.internal.tcnative.SSLSessionCache',
+    'io.netty.internal.tcnative.CertificateCallback',
+    'io.netty.internal.tcnative.CertificateVerifier',
+    'io.netty.internal.tcnative.SessionTicketKey',
+    'io.netty.internal.tcnative.SniHostNameMatcher',
+
+    'com.aayushatharva.brotli4j.Brotli4jLoader',
+    'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
+    'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
+    'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
+    'com.aayushatharva.brotli4j.encoder.Encoder',
+    'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
+    'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
+
+    'com.google.protobuf.nano.CodedOutputByteBufferNano',
+    'com.google.protobuf.nano.MessageNano',
+
+    'com.ning.compress.BufferRecycler',
+    'com.ning.compress.lzf.ChunkDecoder',
+    'com.ning.compress.lzf.ChunkEncoder',
+    'com.ning.compress.lzf.LZFChunk',
+    'com.ning.compress.lzf.LZFEncoder',
+    'com.ning.compress.lzf.util.ChunkDecoderFactory',
+    'com.ning.compress.lzf.util.ChunkEncoderFactory',
+
+    'io.netty.internal.tcnative.AsyncSSLPrivateKeyMethod',
+    'io.netty.internal.tcnative.AsyncTask',
+
+    'io.netty.internal.tcnative.CertificateCallback',
+    'io.netty.internal.tcnative.CertificateVerifier',
+    'io.netty.internal.tcnative.ResultCallback',
+    'io.netty.internal.tcnative.SessionTicketKey',
+    'io.netty.internal.tcnative.SniHostNameMatcher',
+    'io.netty.internal.tcnative.SSL',
+    'io.netty.internal.tcnative.SSLSession',
+    'io.netty.internal.tcnative.SSLSessionCache',
+
+    'lzma.sdk.lzma.Encoder',
+    'net.jpountz.lz4.LZ4Compressor',
+    'net.jpountz.lz4.LZ4Factory',
+    'net.jpountz.lz4.LZ4FastDecompressor',
+    'net.jpountz.xxhash.XXHash32',
+    'net.jpountz.xxhash.XXHashFactory',
+
+    // from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty)
+    'org.bouncycastle.cert.X509v3CertificateBuilder',
+    'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
+    'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+    'org.bouncycastle.openssl.PEMEncryptedKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
+    'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
+    'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
+
+    'org.conscrypt.AllocatedBuffer',
+    'org.conscrypt.BufferAllocator',
+    'org.conscrypt.Conscrypt',
+    'org.conscrypt.HandshakeListener',
+
+    'org.eclipse.jetty.alpn.ALPN$ClientProvider',
+    'org.eclipse.jetty.alpn.ALPN$ServerProvider',
+    'org.eclipse.jetty.alpn.ALPN',
+
+    // from io.netty.handler.ssl.JettyNpnSslEngine (netty)
+    'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
+    'org.eclipse.jetty.npn.NextProtoNego$ServerProvider',
+    'org.eclipse.jetty.npn.NextProtoNego',
+
+    // from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty)
+    'org.jboss.marshalling.ByteInput',
+
+    // from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty)
+    'org.jboss.marshalling.ByteOutput',
+
+    // from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty)
+    'org.jboss.marshalling.Marshaller',
+
+    // from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty)
+    'org.jboss.marshalling.MarshallerFactory',
+    'org.jboss.marshalling.MarshallingConfiguration',
+    'org.jboss.marshalling.Unmarshaller',
+
+    'org.slf4j.impl.StaticLoggerBinder',
+    'org.slf4j.impl.StaticMDCBinder',
+    'org.slf4j.impl.StaticMarkerBinder',
+    'reactor.blockhound.BlockHound$Builder',
+    'reactor.blockhound.integration.BlockHoundIntegration',
+
+    'software.amazon.awssdk.arns.Arn',
+    'software.amazon.awssdk.arns.ArnResource',
+    'software.amazon.awssdk.crt.CRT',
+    'software.amazon.awssdk.crt.auth.credentials.Credentials',
+    'software.amazon.awssdk.crt.auth.credentials.CredentialsProvider',
+    'software.amazon.awssdk.crt.auth.credentials.DelegateCredentialsProvider$DelegateCredentialsProviderBuilder',
+    'software.amazon.awssdk.crt.http.HttpHeader',
+    'software.amazon.awssdk.crt.http.HttpMonitoringOptions',
+    'software.amazon.awssdk.crt.http.HttpProxyOptions',
+    'software.amazon.awssdk.crt.http.HttpRequest',
+    'software.amazon.awssdk.crt.http.HttpRequestBodyStream',
+    'software.amazon.awssdk.crt.io.ClientBootstrap',
+    'software.amazon.awssdk.crt.io.ExponentialBackoffRetryOptions',
+    'software.amazon.awssdk.crt.io.StandardRetryOptions',
+    'software.amazon.awssdk.crt.io.TlsCipherPreference',
+    'software.amazon.awssdk.crt.io.TlsContext',
+    'software.amazon.awssdk.crt.io.TlsContextOptions',
+    'software.amazon.awssdk.crt.s3.ChecksumAlgorithm',
+    'software.amazon.awssdk.crt.s3.ChecksumConfig',
+    'software.amazon.awssdk.crt.s3.ChecksumConfig$ChecksumLocation',
+    'software.amazon.awssdk.crt.s3.ResumeToken',
+    'software.amazon.awssdk.crt.s3.S3Client',
+    'software.amazon.awssdk.crt.s3.S3ClientOptions',
+    'software.amazon.awssdk.crt.s3.S3FinishedResponseContext',
+    'software.amazon.awssdk.crt.s3.S3MetaRequest',
+    'software.amazon.awssdk.crt.s3.S3MetaRequestOptions',
+    'software.amazon.awssdk.crt.s3.S3MetaRequestOptions$MetaRequestType',
+    'software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandler',
+    'software.amazon.awssdk.crtcore.CrtConfigurationUtils',
+    'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration',
+    'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$Builder',
+    'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$DefaultBuilder',
+    'software.amazon.awssdk.crtcore.CrtProxyConfiguration',
+    'software.amazon.awssdk.crtcore.CrtProxyConfiguration$Builder',
+    'software.amazon.awssdk.crtcore.CrtProxyConfiguration$DefaultBuilder',
+    'software.amazon.eventstream.HeaderValue',
+    'software.amazon.eventstream.Message',
+    'software.amazon.eventstream.MessageDecoder',
+
+    'com.jcraft.jzlib.Deflater',
+    'com.jcraft.jzlib.Inflater',
+    'com.jcraft.jzlib.JZlib',
+    'com.jcraft.jzlib.JZlib$WrapperType'
+  )
+
+  ignoreViolations (
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$3',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$4',
+    'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$5',
+
+    'io.netty.util.internal.PlatformDependent0',
+    'io.netty.util.internal.PlatformDependent0$1',
+    'io.netty.util.internal.PlatformDependent0$2',
+    'io.netty.util.internal.PlatformDependent0$3',
+    'io.netty.util.internal.PlatformDependent0$4',
+    'io.netty.util.internal.PlatformDependent0$6',
+
+    'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef',
+    'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef',
+    'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields',
+    'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields',
+    'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields',
+    'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField',
+    'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField',
+    'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
+    'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess',
+  )
+}
 
 // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
 if (BuildParams.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {