diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f5c036fe95..5bc27edbcc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,5 +4,5 @@ # For syntax help see: # https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax -* @googleapis/actools-python @googleapis/yoshi-python @lukesneeringer -*.yaml @googleapis/actools @googleapis/yoshi-python @googleapis/actools-python @lukesneeringer +* @googleapis/actools-python @googleapis/yoshi-python +*.yaml @googleapis/actools @googleapis/yoshi-python @googleapis/actools-python diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index dae6d0bd54..74fb502ec4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -310,4 +310,4 @@ jobs: python -m pip install autopep8 - name: Check diff run: | - find gapic tests -name "*.py" | xargs autopep8 --in-place --exit-code + find gapic tests -name "*.py" | xargs autopep8 --diff --exit-code diff --git a/gapic/ads-templates/%namespace/%name/%version/gapic_metadata.json.j2 b/gapic/ads-templates/%namespace/%name/%version/gapic_metadata.json.j2 new file mode 100644 index 0000000000..edd79fda8a --- /dev/null +++ b/gapic/ads-templates/%namespace/%name/%version/gapic_metadata.json.j2 @@ -0,0 +1 @@ +{# {{ api.gapic_metadata_json(opts) }} #} {# TODO(dovs): This is temporarily commented out pending the addition of a flag #} diff --git a/gapic/cli/generate.py b/gapic/cli/generate.py index b0e50ccf2e..414170a213 100644 --- a/gapic/cli/generate.py +++ b/gapic/cli/generate.py @@ -47,10 +47,11 @@ def generate( # This generator uses a slightly different mechanism for determining # which files to generate; it tracks at package level rather than file # level. - package = os.path.commonprefix([i.package for i in filter( - lambda p: p.name in req.file_to_generate, - req.proto_file, - )]).rstrip('.') + package = os.path.commonprefix([ + p.package + for p in req.proto_file + if p.name in req.file_to_generate + ]).rstrip('.') # Build the API model object. # This object is a frozen representation of the whole API, and is sent diff --git a/gapic/schema/api.py b/gapic/schema/api.py index 35a59dcee0..39f0c49045 100644 --- a/gapic/schema/api.py +++ b/gapic/schema/api.py @@ -28,8 +28,10 @@ from google.api_core import exceptions # type: ignore from google.api import resource_pb2 # type: ignore +from google.gapic.metadata import gapic_metadata_pb2 # type: ignore from google.longrunning import operations_pb2 # type: ignore from google.protobuf import descriptor_pb2 +from google.protobuf.json_format import MessageToJson import grpc # type: ignore @@ -392,6 +394,45 @@ def subpackages(self) -> Mapping[str, 'API']: ) return answer + def gapic_metadata(self, options: Options) -> gapic_metadata_pb2.GapicMetadata: + gm = gapic_metadata_pb2.GapicMetadata( + schema="1.0", + comment="This file maps proto services/RPCs to the corresponding library clients/methods", + language="python", + proto_package=self.naming.proto_package, + library_package=".".join( + self.naming.module_namespace + + (self.naming.versioned_module_name,) + ), + ) + + for service in sorted(self.services.values(), key=lambda s: s.name): + service_desc = gm.services.get_or_create(service.name) + + # At least one of "grpc" or "rest" is guaranteed to be present because + # of the way that Options instances are created. + # This assumes the options are generated by the class method factory. + transports = [] + if "grpc" in options.transport: + transports.append(("grpc", service.client_name)) + transports.append(("grpcAsync", service.async_client_name)) + + if "rest" in options.transport: + transports.append(("rest", service.client_name)) + + methods = sorted(service.methods.values(), key=lambda m: m.name) + for tprt, client_name in transports: + transport = service_desc.clients.get_or_create(tprt) + transport.library_client = client_name + for method in methods: + method_desc = transport.rpcs.get_or_create(method.name) + method_desc.methods.append(to_snake_case(method.name)) + + return gm + + def gapic_metadata_json(self, options: Options) -> str: + return MessageToJson(self.gapic_metadata(options), sort_keys=True) + def requires_package(self, pkg: Tuple[str, ...]) -> bool: return any( message.ident.package == pkg diff --git a/gapic/templates/%namespace/%name_%version/gapic_metadata.json.j2 b/gapic/templates/%namespace/%name_%version/gapic_metadata.json.j2 new file mode 100644 index 0000000000..edd79fda8a --- /dev/null +++ b/gapic/templates/%namespace/%name_%version/gapic_metadata.json.j2 @@ -0,0 +1 @@ +{# {{ api.gapic_metadata_json(opts) }} #} {# TODO(dovs): This is temporarily commented out pending the addition of a flag #} diff --git a/setup.py b/setup.py index 409dea8b65..69bb01d55f 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ install_requires=( "click >= 6.7", "google-api-core >= 1.17.0", - "googleapis-common-protos >= 1.6.0", + "googleapis-common-protos >= 1.53.0", "grpcio >= 1.24.3", "jinja2 >= 2.10", "protobuf >= 3.12.0", diff --git a/tests/unit/schema/test_api.py b/tests/unit/schema/test_api.py index e91a310ee0..1eaa8a57ae 100644 --- a/tests/unit/schema/test_api.py +++ b/tests/unit/schema/test_api.py @@ -22,8 +22,10 @@ from google.api import client_pb2 from google.api import resource_pb2 from google.api_core import exceptions +from google.gapic.metadata import gapic_metadata_pb2 from google.longrunning import operations_pb2 from google.protobuf import descriptor_pb2 +from google.protobuf.json_format import MessageToJson from gapic.schema import api from gapic.schema import imp @@ -260,8 +262,8 @@ def test_proto_oneof(): name='Bar', fields=( make_field_pb2(name='imported_message', number=1, - type_name='.google.dep.ImportedMessage', - oneof_index=0), + type_name='.google.dep.ImportedMessage', + oneof_index=0), make_field_pb2( name='primitive', number=2, type=1, oneof_index=0), ), @@ -1287,3 +1289,224 @@ def test_map_field_name_disambiguation(): # The same module used in the same place should have the same import alias. # Because there's a "mollusc" name used, the import should be disambiguated. assert mollusc_ident == mollusc_map_ident == "am_mollusc.Mollusc" + + +def test_gapic_metadata(): + api_schema = api.API.build( + file_descriptors=[ + descriptor_pb2.FileDescriptorProto( + name="cephalopod.proto", + package="animalia.mollusca.v1", + message_type=[ + descriptor_pb2.DescriptorProto( + name="MolluscRequest", + ), + descriptor_pb2.DescriptorProto( + name="Mollusc", + ), + ], + service=[ + descriptor_pb2.ServiceDescriptorProto( + name="Squid", + method=[ + descriptor_pb2.MethodDescriptorProto( + name="Ramshorn", + input_type="animalia.mollusca.v1.MolluscRequest", + output_type="animalia.mollusca.v1.Mollusc", + ), + descriptor_pb2.MethodDescriptorProto( + name="Humboldt", + input_type="animalia.mollusca.v1.MolluscRequest", + output_type="animalia.mollusca.v1.Mollusc", + ), + descriptor_pb2.MethodDescriptorProto( + name="Giant", + input_type="animalia.mollusca.v1.MolluscRequest", + output_type="animalia.mollusca.v1.Mollusc", + ), + ], + ), + descriptor_pb2.ServiceDescriptorProto( + name="Octopus", + method=[ + descriptor_pb2.MethodDescriptorProto( + name="GiantPacific", + input_type="animalia.mollusca.v1.MolluscRequest", + output_type="animalia.mollusca.v1.Mollusc", + ), + descriptor_pb2.MethodDescriptorProto( + name="BlueSpot", + input_type="animalia.mollusca.v1.MolluscRequest", + output_type="animalia.mollusca.v1.Mollusc", + ), + ] + ), + ], + ) + ] + ) + + opts = Options.build("transport=grpc") + expected = gapic_metadata_pb2.GapicMetadata( + schema="1.0", + comment="This file maps proto services/RPCs to the corresponding library clients/methods", + language="python", + proto_package="animalia.mollusca.v1", + library_package="animalia.mollusca_v1", + services={ + "Octopus": gapic_metadata_pb2.GapicMetadata.ServiceForTransport( + clients={ + "grpc": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="OctopusClient", + rpcs={ + "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]), + "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]), + }, + ), + "grpcAsync": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="OctopusAsyncClient", + rpcs={ + "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]), + "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]), + }, + ), + } + ), + "Squid": gapic_metadata_pb2.GapicMetadata.ServiceForTransport( + clients={ + "grpc": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="SquidClient", + rpcs={ + "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]), + "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]), + "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]), + }, + ), + "grpcAsync": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="SquidAsyncClient", + rpcs={ + "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]), + "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]), + "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]), + }, + ), + } + ), + } + ) + actual = api_schema.gapic_metadata(opts) + assert expected == actual + expected = MessageToJson(expected, sort_keys=True) + actual = api_schema.gapic_metadata_json(opts) + assert expected == actual + + opts = Options.build("transport=rest") + expected = gapic_metadata_pb2.GapicMetadata( + schema="1.0", + comment="This file maps proto services/RPCs to the corresponding library clients/methods", + language="python", + proto_package="animalia.mollusca.v1", + library_package="animalia.mollusca_v1", + services={ + "Octopus": gapic_metadata_pb2.GapicMetadata.ServiceForTransport( + clients={ + "rest": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="OctopusClient", + rpcs={ + "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]), + "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]), + }, + ) + } + ), + "Squid": gapic_metadata_pb2.GapicMetadata.ServiceForTransport( + clients={ + "rest": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="SquidClient", + rpcs={ + "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]), + "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]), + "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]), + }, + ), + + } + ), + } + ) + actual = api_schema.gapic_metadata(opts) + assert expected == actual + expected = MessageToJson(expected, sort_keys=True) + actual = api_schema.gapic_metadata_json(opts) + assert expected == actual + + opts = Options.build("transport=rest+grpc") + expected = gapic_metadata_pb2.GapicMetadata( + schema="1.0", + comment="This file maps proto services/RPCs to the corresponding library clients/methods", + language="python", + proto_package="animalia.mollusca.v1", + library_package="animalia.mollusca_v1", + services={ + "Octopus": gapic_metadata_pb2.GapicMetadata.ServiceForTransport( + clients={ + "grpc": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="OctopusClient", + rpcs={ + "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]), + "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]), + }, + ), + "grpcAsync": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="OctopusAsyncClient", + rpcs={ + "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]), + "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]), + }, + ), + "rest": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="OctopusClient", + rpcs={ + "BlueSpot": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["blue_spot"]), + "GiantPacific": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant_pacific"]), + }, + ) + } + ), + "Squid": gapic_metadata_pb2.GapicMetadata.ServiceForTransport( + clients={ + "grpc": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="SquidClient", + rpcs={ + "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]), + "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]), + "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]), + }, + ), + "grpcAsync": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="SquidAsyncClient", + rpcs={ + "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]), + "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]), + "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]), + }, + ), + "rest": gapic_metadata_pb2.GapicMetadata.ServiceAsClient( + library_client="SquidClient", + rpcs={ + "Giant": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["giant"]), + "Humboldt": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["humboldt"]), + "Ramshorn": gapic_metadata_pb2.GapicMetadata.MethodList(methods=["ramshorn"]), + }, + ), + + } + ), + } + ) + + actual = api_schema.gapic_metadata(opts) + assert expected == actual + expected = MessageToJson(expected, sort_keys=True) + actual = api_schema.gapic_metadata_json(opts) + assert expected == actual