[Extensions] Moving ExtensionsRequest to use Protobuf

CHANGELOG.md

@@ -80,6 +80,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- [Extensions] Moving Extensions APIs to protobuf serialization. ([#6960](https://github.com/opensearch-project/OpenSearch/pull/6960))
 
 ### Dependencies
 

buildSrc/version.properties

@@ -22,6 +22,7 @@ woodstox          = 6.4.0
 kotlin            = 1.7.10
 antlr4            = 4.11.1
 guava             = 31.1-jre
+protobuf          = 3.22.2
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0

modules/transport-netty4/build.gradle

@@ -131,12 +131,6 @@ thirdPartyAudit {
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
     // classes are missing
 
-    // from io.netty.handler.codec.protobuf.ProtobufDecoder (netty)
-    'com.google.protobuf.ExtensionRegistry',
-    'com.google.protobuf.MessageLite$Builder',
-    'com.google.protobuf.MessageLite',
-    'com.google.protobuf.Parser',
-
     // from io.netty.logging.CommonsLoggerFactory (netty)
     'org.apache.commons.logging.Log',
     'org.apache.commons.logging.LogFactory',
@@ -191,8 +185,6 @@ thirdPartyAudit {
     'org.slf4j.spi.LocationAwareLogger',
 
     'com.github.luben.zstd.Zstd',
-    'com.google.protobuf.ExtensionRegistryLite',
-    'com.google.protobuf.MessageLiteOrBuilder',
     'com.google.protobuf.nano.CodedOutputByteBufferNano',
     'com.google.protobuf.nano.MessageNano',
     'com.jcraft.jzlib.Deflater',

plugins/repository-gcs/build.gradle

@@ -65,8 +65,6 @@ dependencies {
   api 'com.google.api:api-common:1.8.1'
   api 'com.google.api:gax:2.17.0'
   api 'org.threeten:threetenbp:1.4.4'
-  api 'com.google.protobuf:protobuf-java-util:3.22.2'
-  api 'com.google.protobuf:protobuf-java:3.22.2'
   api 'com.google.code.gson:gson:2.9.0'
   api 'com.google.api.grpc:proto-google-common-protos:2.10.0'
   api 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
@@ -105,13 +103,6 @@ tasks.named("dependencyLicenses").configure {
 thirdPartyAudit {
   ignoreViolations(
     // uses internal java api: sun.misc.Unsafe
-    'com.google.protobuf.UnsafeUtil',
-    'com.google.protobuf.UnsafeUtil$1',
-    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
-    'com.google.protobuf.UnsafeUtil$MemoryAccessor',
-    'com.google.protobuf.MessageSchema',
-    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
-    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
     'com.google.common.cache.Striped64',
     'com.google.common.cache.Striped64$1',
     'com.google.common.cache.Striped64$Cell',
@@ -151,6 +142,10 @@ thirdPartyAudit {
     'com.google.appengine.api.urlfetch.URLFetchService',
     'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
     'com.oracle.svm.core.configure.ResourcesRegistry',
+    'com.google.protobuf.util.JsonFormat',
+    'com.google.protobuf.util.JsonFormat$Parser',
+    'com.google.protobuf.util.JsonFormat$Printer',
+    'com.google.protobuf.util.Timestamps',
     // commons-logging optional dependencies
     'org.apache.avalon.framework.logger.Logger',
     'org.apache.log.Hierarchy',

plugins/repository-hdfs/build.gradle

@@ -69,7 +69,6 @@ dependencies {
   api 'org.apache.avro:avro:1.11.1'
   api 'com.google.code.gson:gson:2.10.1'
   runtimeOnly "com.google.guava:guava:${versions.guava}"
-  api 'com.google.protobuf:protobuf-java:3.22.3'
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api 'commons-cli:commons-cli:1.5.0'
   api "commons-codec:commons-codec:${versions.commonscodec}"
@@ -116,14 +115,7 @@ tasks.named("dependencyLicenses").configure {
 
 thirdPartyAudit {
   ignoreViolations(
-    // uses internal java api: sun.misc.Unsafe
-    'com.google.protobuf.MessageSchema',
-    'com.google.protobuf.UnsafeUtil',
-    'com.google.protobuf.UnsafeUtil$1',
-    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
-    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
-    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
-    'com.google.protobuf.UnsafeUtil$MemoryAccessor'
+
   )
 }
 

plugins/transport-nio/build.gradle

@@ -66,12 +66,6 @@ thirdPartyAudit {
     'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
 
-    // from io.netty.handler.codec.protobuf.ProtobufDecoder (netty)
-    'com.google.protobuf.ExtensionRegistry',
-    'com.google.protobuf.MessageLite$Builder',
-    'com.google.protobuf.MessageLite',
-    'com.google.protobuf.Parser',
-
     // from io.netty.logging.CommonsLoggerFactory (netty)
     'org.apache.commons.logging.Log',
     'org.apache.commons.logging.LogFactory',
@@ -118,8 +112,6 @@ thirdPartyAudit {
     'org.slf4j.spi.LocationAwareLogger',
 
     'com.github.luben.zstd.Zstd',
-    'com.google.protobuf.ExtensionRegistryLite',
-    'com.google.protobuf.MessageLiteOrBuilder',
     'com.google.protobuf.nano.CodedOutputByteBufferNano',
     'com.google.protobuf.nano.MessageNano',
     'com.jcraft.jzlib.Deflater',

server/build.gradle

@@ -30,10 +30,13 @@
 
 import org.opensearch.gradle.info.BuildParams
 
-apply plugin: 'opensearch.build'
-apply plugin: 'com.netflix.nebula.optional-base'
-apply plugin: 'opensearch.publish'
-apply plugin: 'opensearch.internal-cluster-test'
+plugins {
+  id('com.google.protobuf') version 'latest.release'
+  id('opensearch.build')
+  id('com.netflix.nebula.optional-base')
+  id('opensearch.publish')
+  id('opensearch.internal-cluster-test')
+}
 
 publishing {
   publications {
@@ -45,6 +48,13 @@ publishing {
 
 archivesBaseName = 'opensearch'
 
+sourceSets {
+  main {
+    java {
+      srcDir "${buildDir}/generated/source/proto/main/java"
+    }
+  }
+}
 // we want to keep the JDKs in our IDEs set to JDK 8 until minimum JDK is bumped to 11 so we do not include this source set in our IDEs
 if (!isEclipse) {
   sourceSets {
@@ -136,6 +146,9 @@ dependencies {
   // jna
   api "net.java.dev.jna:jna:${versions.jna}"
 
+  // protobuf
+  api "com.google.protobuf:protobuf-java:${versions.protobuf}"
+
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server
     exclude group: 'org.opensearch', module: 'server'
@@ -157,6 +170,7 @@ tasks.named("internalClusterTest").configure {
 }
 
 tasks.named("forbiddenPatterns").configure {
+    dependsOn("generateProto")
     exclude '**/*.json'
     exclude '**/*.jmx'
     exclude '**/*.dic'
@@ -203,6 +217,12 @@ def generatePluginsList = tasks.register("generatePluginsList") {
     }
 }
 
+protobuf {
+  protoc {
+    artifact = "com.google.protobuf:protoc:${versions.protobuf}"
+  }
+}
+
 tasks.named("processResources").configure {
     dependsOn generateModulesList, generatePluginsList
 }
@@ -303,6 +323,15 @@ tasks.named("thirdPartyAudit").configure {
             'com.google.common.geometry.S2$Metric',
             'com.google.common.geometry.S2LatLng'
     )
+    ignoreViolations(
+            'com.google.protobuf.MessageSchema',
+            'com.google.protobuf.UnsafeUtil',
+            'com.google.protobuf.UnsafeUtil$1',
+            'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+            'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
+            'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
+            'com.google.protobuf.UnsafeUtil$MemoryAccessor'
+    )
 }
 
 tasks.named("dependencyLicenses").configure {
@@ -315,13 +344,34 @@ tasks.named("dependencyLicenses").configure {
     }
 }
 
+tasks.named("missingJavadoc").configure {
+  /*
+   * Generated code doesn't follow javadocs formats.
+   * Unfortunately the missingJavadoc task doesnt take *.
+   * TODO: Add support to missingJavadoc task to ignore all generated source code
+   */
+  dependsOn("generateProto")
+  javadocMissingIgnore = [
+    "org.opensearch.extensions.proto.ExtensionRequestProto",
+    "org.opensearch.extensions.proto.ExtensionRequestProto.ExtensionRequestOrBuilder",
+    "org.opensearch.extensions.proto"
+  ]
+}
+
+tasks.named("filepermissions").configure {
+  mustRunAfter("generateProto")
+}
+
 tasks.named("licenseHeaders").configure {
+    dependsOn("generateProto")
     // Ignore our vendored version of Google Guice
     excludes << 'org/opensearch/common/inject/**/*'
     // Ignore temporary copies of impending 8.7 Lucene classes
     excludes << 'org/apache/lucene/search/RegExp87*'
     excludes << 'org/apache/lucene/search/RegexpQuery87*'
     excludes << 'org/opensearch/client/documentation/placeholder.txt'
+    // Ignore for generated code
+    excludes << 'org/opensearch/extensions/proto/*'
 }
 
 tasks.test {

server/src/main/java/org/opensearch/extensions/ExtensionRequest.java

@@ -13,11 +13,11 @@
 import org.opensearch.common.Nullable;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.extensions.proto.ExtensionRequestProto;
 import org.opensearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.Objects;
-import java.util.Optional;
 
 /**
  * CLusterService Request for Extensibility
@@ -26,54 +26,54 @@
  */
 public class ExtensionRequest extends TransportRequest {
     private static final Logger logger = LogManager.getLogger(ExtensionRequest.class);
-    private final ExtensionsManager.RequestType requestType;
-    private final Optional<String> uniqueId;
+    private final ExtensionRequestProto.ExtensionRequest request;
 
-    public ExtensionRequest(ExtensionsManager.RequestType requestType) {
+    public ExtensionRequest(ExtensionRequestProto.RequestType requestType) {
         this(requestType, null);
     }
 
-    public ExtensionRequest(ExtensionsManager.RequestType requestType, @Nullable String uniqueId) {
-        this.requestType = requestType;
-        this.uniqueId = uniqueId == null ? Optional.empty() : Optional.of(uniqueId);
+    public ExtensionRequest(ExtensionRequestProto.RequestType requestType, @Nullable String uniqueId) {
+        ExtensionRequestProto.ExtensionRequest.Builder builder = ExtensionRequestProto.ExtensionRequest.newBuilder();
+        if (uniqueId != null) {
+            builder.setUniqueId(uniqueId);
+        }
+        this.request = builder.setRequestType(requestType).build();
     }
 
     public ExtensionRequest(StreamInput in) throws IOException {
         super(in);
-        this.requestType = in.readEnum(ExtensionsManager.RequestType.class);
-        String id = in.readOptionalString();
-        this.uniqueId = id == null ? Optional.empty() : Optional.of(id);
+        this.request = ExtensionRequestProto.ExtensionRequest.parseFrom(in.readByteArray());
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeEnum(requestType);
-        out.writeOptionalString(uniqueId.orElse(null));
+        out.writeByteArray(request.toByteArray());
     }
 
-    public ExtensionsManager.RequestType getRequestType() {
-        return this.requestType;
+    public ExtensionRequestProto.RequestType getRequestType() {
+        return this.request.getRequestType();
     }
 
-    public Optional<String> getUniqueId() {
-        return uniqueId;
+    public String getUniqueId() {
+        return request.getUniqueId();
     }
 
     public String toString() {
-        return "ExtensionRequest{" + "requestType=" + requestType + "uniqueId=" + uniqueId + '}';
+        return "ExtensionRequest{" + request.toString() + '}';
     }
 
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         ExtensionRequest that = (ExtensionRequest) o;
-        return Objects.equals(requestType, that.requestType) && Objects.equals(uniqueId, that.uniqueId);
+        return Objects.equals(request.getRequestType(), that.request.getRequestType())
+            && Objects.equals(request.getUniqueId(), that.request.getUniqueId());
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(requestType, uniqueId);
+        return Objects.hash(request.getRequestType(), request.getUniqueId());
     }
 }

server/src/main/java/org/opensearch/extensions/ExtensionsManager.java

@@ -89,32 +89,13 @@ public class ExtensionsManager {
     public static final String REQUEST_EXTENSION_REGISTER_CUSTOM_SETTINGS = "internal:discovery/registercustomsettings";
     public static final String REQUEST_EXTENSION_REGISTER_REST_ACTIONS = "internal:discovery/registerrestactions";
     public static final String REQUEST_EXTENSION_REGISTER_TRANSPORT_ACTIONS = "internal:discovery/registertransportactions";
-    public static final String REQUEST_OPENSEARCH_PARSE_NAMED_WRITEABLE = "internal:discovery/parsenamedwriteable";
     public static final String REQUEST_REST_EXECUTE_ON_EXTENSION_ACTION = "internal:extensions/restexecuteonextensiontaction";
     public static final String REQUEST_EXTENSION_HANDLE_TRANSPORT_ACTION = "internal:extensions/handle-transportaction";
     public static final String REQUEST_EXTENSION_HANDLE_REMOTE_TRANSPORT_ACTION = "internal:extensions/handle-remote-transportaction";
     public static final String TRANSPORT_ACTION_REQUEST_FROM_EXTENSION = "internal:extensions/request-transportaction-from-extension";
     public static final int EXTENSION_REQUEST_WAIT_TIMEOUT = 10;
-
     private static final Logger logger = LogManager.getLogger(ExtensionsManager.class);
 
-    /**
-     * Enum for Extension Requests
-     *
-     * @opensearch.internal
-     */
-    public static enum RequestType {
-        REQUEST_EXTENSION_CLUSTER_STATE,
-        REQUEST_EXTENSION_CLUSTER_SETTINGS,
-        REQUEST_EXTENSION_REGISTER_REST_ACTIONS,
-        REQUEST_EXTENSION_REGISTER_SETTINGS,
-        REQUEST_EXTENSION_ENVIRONMENT_SETTINGS,
-        REQUEST_EXTENSION_DEPENDENCY_INFORMATION,
-        CREATE_COMPONENT,
-        ON_INDEX_MODULE,
-        GET_SETTINGS
-    };
-
     /**
      * Enum for OpenSearch Requests
      *
@@ -430,7 +411,7 @@ public String executor() {
     /**
      * Handles an {@link ExtensionRequest}.
      *
-     * @param extensionRequest  The request to handle, of a type defined in the {@link RequestType} enum.
+     * @param extensionRequest  The request to handle, of a type defined in the {@link org.opensearch.extensions.proto.ExtensionRequestProto.RequestType} enum.
      * @return  an Response matching the request.
      * @throws Exception if the request is not handled properly.
      */
@@ -443,7 +424,7 @@ TransportResponse handleExtensionRequest(ExtensionRequest extensionRequest) thro
             case REQUEST_EXTENSION_ENVIRONMENT_SETTINGS:
                 return new EnvironmentSettingsResponse(this.environmentSettings);
             case REQUEST_EXTENSION_DEPENDENCY_INFORMATION:
-                String uniqueId = extensionRequest.getUniqueId().orElse(null);
+                String uniqueId = extensionRequest.getUniqueId();
                 if (uniqueId == null) {
                     return new ExtensionDependencyResponse(extensions);
                 } else {
@@ -686,10 +667,6 @@ public static String getRequestExtensionRegisterRestActions() {
         return REQUEST_EXTENSION_REGISTER_REST_ACTIONS;
     }
 
-    public static String getRequestOpensearchParseNamedWriteable() {
-        return REQUEST_OPENSEARCH_PARSE_NAMED_WRITEABLE;
-    }
-
     public static String getRequestRestExecuteOnExtensionAction() {
         return REQUEST_REST_EXECUTE_ON_EXTENSION_ACTION;
     }