From d71cda79604a229adfd0ea23fe1e95ef934bdefc Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 28 Mar 2024 16:57:55 -0400 Subject: [PATCH 1/7] feat: Expose ChannelPoolSettings Getter for gRPC --- .../api/gax/grpc/InstantiatingGrpcChannelProvider.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java b/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java index bf8ffc81da..f62015ffbb 100644 --- a/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java +++ b/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java @@ -473,6 +473,13 @@ public Integer getMaxInboundMetadataSize() { return maxInboundMetadataSize; } + /** + * The configured channel pool settings for gRPC channels + */ + public ChannelPoolSettings getChannelPoolSettings() { + return channelPoolSettings; + } + @Override public boolean shouldAutoClose() { return true; From e2dc2d8ac497656d240546bc67305b5f6640a4fd Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 28 Mar 2024 17:44:34 -0400 Subject: [PATCH 2/7] chore: Fix lint issues --- .../google/api/gax/grpc/InstantiatingGrpcChannelProvider.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java b/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java index f62015ffbb..9699b26019 100644 --- a/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java +++ b/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java @@ -473,9 +473,7 @@ public Integer getMaxInboundMetadataSize() { return maxInboundMetadataSize; } - /** - * The configured channel pool settings for gRPC channels - */ + /** The configured channel pool settings used for gRPC's ChannelProvider */ public ChannelPoolSettings getChannelPoolSettings() { return channelPoolSettings; } From f8ad3c4c9892c984e6ca96c46dd6db6fffed82bf Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:22:24 +0000 Subject: [PATCH 3/7] chore: sync renovate template from synthtool (#2601) In this PR: - Sync https://github.com/googleapis/synthtool/pull/1921 - Sync https://github.com/googleapis/synthtool/pull/1922 - Sync https://github.com/googleapis/synthtool/pull/1941 --- .../templates/java_library/renovate.json | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/library_generation/owlbot/templates/java_library/renovate.json b/library_generation/owlbot/templates/java_library/renovate.json index 16c68a2387..0dd74f9f2c 100644 --- a/library_generation/owlbot/templates/java_library/renovate.json +++ b/library_generation/owlbot/templates/java_library/renovate.json @@ -16,6 +16,26 @@ ".kokoro/requirements.txt", ".github/workflows/**" ], + "customManagers": [ + { + "customType": "regex", + "fileMatch": [ + "^.kokoro/presubmit/graalvm-native.*.cfg$" + ], + "matchStrings": ["value: \"gcr.io/cloud-devrel-public-resources/graalvm.*:(?.*?)\""], + "depNameTemplate": "com.google.cloud:sdk-platform-java-config", + "datasourceTemplate": "maven" + }, + { + "customType": "regex", + "fileMatch": [ + "^.github/workflows/unmanaged_dependency_check.yaml$" + ], + "matchStrings": ["uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v(?.+?)\\n"], + "depNameTemplate": "com.google.cloud:sdk-platform-java-config", + "datasourceTemplate": "maven" + } + ], "packageRules": [ { "packagePatterns": [ @@ -57,7 +77,8 @@ "^com.google.truth:truth", "^org.mockito:mockito-core", "^org.objenesis:objenesis", - "^com.google.cloud:google-cloud-conformance-tests" + "^com.google.cloud:google-cloud-conformance-tests", + "^org.graalvm.buildtools:junit-platform-native" ], "semanticCommitType": "test", "semanticCommitScope": "deps" From f06fa6b1bd75c64b5430d2abf954d064406f0a26 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:50:19 +0000 Subject: [PATCH 4/7] feat: generate selected libraries (#2598) In this PR: - Generate a selected list of libraries based on `library_name` This PR is a preparation for improve nightly generation. Note that we need to change the [new library generation workflow](https://github.com/googleapis/google-cloud-java/blob/main/.github/workflows/generate_new_client_hermetic_build.yaml) in google-cloud-java to accommodate this change because `library_generation/generate_repo.py` has replaced `--target-library-api-shortname` with `--target-library-names`. --- library_generation/generate_repo.py | 71 ++++++++++++++----- .../test/generate_repo_unit_tests.py | 63 ++++++++++++++++ library_generation/utilities.py | 6 +- 3 files changed, 117 insertions(+), 23 deletions(-) create mode 100644 library_generation/test/generate_repo_unit_tests.py diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index abe51c50f2..b33e8ba10a 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -13,11 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import click import library_generation.utilities as util +import click import os from library_generation.generate_composed_library import generate_composed_library +from library_generation.model.generation_config import GenerationConfig from library_generation.model.generation_config import from_yaml +from library_generation.model.library_config import LibraryConfig from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing @@ -39,13 +41,22 @@ def main(ctx): """, ) @click.option( - "--target-library-api-shortname", + "--target-library-names", required=False, + default=None, type=str, help=""" - If specified, only the `library` whose api_shortname equals to - target-library-api-shortname will be generated. + A list of libraries will be generated. + + If specified, only the `library` whose library_name is in + target-library-names will be generated. If not specified, all libraries in the configuration yaml will be generated. + + The input string will be parsed to a list of string with comma as the + separator. + + For example, apigeeconnect,alloydb-connectors will be parsed as a + list of two strings, apigeeconnect and alloydb-connectors. """, ) @click.option( @@ -61,39 +72,46 @@ def main(ctx): ) def generate( generation_config_yaml: str, - target_library_api_shortname: str, + target_library_names: str, repository_path: str, ): generate_from_yaml( generation_config_yaml=generation_config_yaml, repository_path=repository_path, - target_library_api_shortname=target_library_api_shortname, + target_library_names=target_library_names.split(",") + if target_library_names is not None + else target_library_names, ) def generate_from_yaml( generation_config_yaml: str, repository_path: str, - target_library_api_shortname: str = None, + target_library_names: list[str] = None, ) -> None: """ Parses a config yaml and generates libraries via generate_composed_library.py + :param generation_config_yaml: Path to generation_config.yaml that contains + the metadata about library generation + :param repository_path: If specified, the generated files will be sent to + this location. If not specified, the repository will be generated to the + current working directory. + :param target_library_names: a list of libraries to be generated. + If specified, only the library whose library_name is in + target-library-names will be generated. + If specified with an empty list, then no library will be generated. + If not specified, all libraries in the configuration yaml will be generated. """ - # convert paths to absolute paths so they can be correctly referenced in + # convert paths to absolute paths, so they can be correctly referenced in # downstream scripts generation_config_yaml = os.path.abspath(generation_config_yaml) repository_path = os.path.abspath(repository_path) config = from_yaml(generation_config_yaml) - target_libraries = config.libraries - if target_library_api_shortname is not None: - target_libraries = [ - library - for library in config.libraries - if library.api_shortname == target_library_api_shortname - ] - + target_libraries = get_target_libraries( + config=config, target_library_names=target_library_names + ) repo_config = util.prepare_repo( gen_config=config, library_config=target_libraries, repo_path=repository_path ) @@ -118,5 +136,22 @@ def generate_from_yaml( ) -if __name__ == "__main__": - main() +def get_target_libraries( + config: GenerationConfig, target_library_names: list[str] = None +) -> list[LibraryConfig]: + """ + Returns LibraryConfig objects whose library_name is in target_library_names. + + :param config: a GenerationConfig object. + :param target_library_names: library_name of target libraries. + If not specified, all libraries in the given config will be returned. + :return: LibraryConfig objects. + """ + if target_library_names is None: + return config.libraries + target_libraries = set(target_library_names) + return [ + library + for library in config.libraries + if library.get_library_name() in target_libraries + ] diff --git a/library_generation/test/generate_repo_unit_tests.py b/library_generation/test/generate_repo_unit_tests.py new file mode 100644 index 0000000000..db8cb4eb63 --- /dev/null +++ b/library_generation/test/generate_repo_unit_tests.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.generate_repo import get_target_libraries +from library_generation.model.generation_config import GenerationConfig +from library_generation.model.library_config import LibraryConfig + + +class GenerateRepoTest(unittest.TestCase): + def test_get_target_library_returns_selected_libraries(self): + one_library = GenerateRepoTest.__get_an_empty_library_config() + one_library.api_shortname = "one_library" + another_library = GenerateRepoTest.__get_an_empty_library_config() + another_library.api_shortname = "another_library" + config = GenerateRepoTest.__get_an_empty_generation_config() + config.libraries.extend([one_library, another_library]) + target_libraries = get_target_libraries(config, ["another_library"]) + self.assertEqual([another_library], target_libraries) + + def test_get_target_library_given_null_returns_all_libraries(self): + one_library = GenerateRepoTest.__get_an_empty_library_config() + one_library.api_shortname = "one_library" + another_library = GenerateRepoTest.__get_an_empty_library_config() + another_library.api_shortname = "another_library" + config = GenerateRepoTest.__get_an_empty_generation_config() + config.libraries.extend([one_library, another_library]) + target_libraries = get_target_libraries(config) + self.assertEqual([one_library, another_library], target_libraries) + + @staticmethod + def __get_an_empty_generation_config() -> GenerationConfig: + return GenerationConfig( + gapic_generator_version="", + googleapis_commitish="", + synthtool_commitish="", + owlbot_cli_image="", + template_excludes=[], + path_to_yaml="", + libraries=[], + ) + + @staticmethod + def __get_an_empty_library_config() -> LibraryConfig: + return LibraryConfig( + api_shortname="", + name_pretty="", + api_description="", + product_documentation="", + gapic_configs=[], + ) diff --git a/library_generation/utilities.py b/library_generation/utilities.py index de3dd6ce44..fb7e8938d2 100755 --- a/library_generation/utilities.py +++ b/library_generation/utilities.py @@ -131,11 +131,7 @@ def prepare_repo( os.makedirs(output_folder, exist_ok=True) libraries = {} for library in library_config: - library_name = ( - f"{language}-{library.library_name}" - if library.library_name - else f"{language}-{library.api_shortname}" - ) + library_name = f"{language}-{library.get_library_name()}" library_path = ( f"{repo_path}/{library_name}" if gen_config.is_monorepo else f"{repo_path}" ) From f1143cb958c3febbf3b76b0b3002d9f1eeb027d5 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Wed, 27 Mar 2024 21:04:40 +0000 Subject: [PATCH 5/7] fix: add main to `generate_repo.py` (#2607) In todays's nightly generation job ([log](https://github.com/googleapis/google-cloud-java/actions/runs/8457767424/job/23170489394)), no actual generation is done. I think this is because `main` method has been removed by #2598. --- library_generation/generate_repo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index b33e8ba10a..a4a4a826f2 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -155,3 +155,7 @@ def get_target_libraries( for library in config.libraries if library.get_library_name() in target_libraries ] + + +if __name__ == "__main__": + main() From a828d7ce72f028a94093381a1f4b8ecd0f6ab58d Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Thu, 28 Mar 2024 20:09:58 +0000 Subject: [PATCH 6/7] chore: update renovate to explicitly ignore workflow files (#2611) Port template change in https://github.com/googleapis/synthtool/pull/1945 --- .../owlbot/templates/java_library/renovate.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/library_generation/owlbot/templates/java_library/renovate.json b/library_generation/owlbot/templates/java_library/renovate.json index 0dd74f9f2c..976d43a1f8 100644 --- a/library_generation/owlbot/templates/java_library/renovate.json +++ b/library_generation/owlbot/templates/java_library/renovate.json @@ -14,7 +14,10 @@ ], "ignorePaths": [ ".kokoro/requirements.txt", - ".github/workflows/**" + ".github/workflows/approve-readme.yaml", + ".github/workflows/ci.yaml", + ".github/workflows/renovate_config_check.yaml", + ".github/workflows/samples.yaml" ], "customManagers": [ { From d6e80c4d7599d9675f3b22128914019ceaf03fa3 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 1 Apr 2024 16:03:56 -0400 Subject: [PATCH 7/7] chore: Add @BetaApi annotation --- .../google/api/gax/grpc/InstantiatingGrpcChannelProvider.java | 1 + 1 file changed, 1 insertion(+) diff --git a/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java b/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java index 9699b26019..5969ed4693 100644 --- a/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java +++ b/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java @@ -474,6 +474,7 @@ public Integer getMaxInboundMetadataSize() { } /** The configured channel pool settings used for gRPC's ChannelProvider */ + @BetaApi("Channel pool sizing api is not yet stable") public ChannelPoolSettings getChannelPoolSettings() { return channelPoolSettings; }