From 95b66a80aa3454f58fb4789819967cfc4e7f9e1b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Apr 2024 19:01:36 -0400 Subject: [PATCH 01/34] feat: add entry_point.py --- library_generation/cli/entry_point.py | 87 +++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 library_generation/cli/entry_point.py diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py new file mode 100644 index 0000000000..e493c86797 --- /dev/null +++ b/library_generation/cli/entry_point.py @@ -0,0 +1,87 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import click as click +from library_generation.generate_pr_description import generate_pr_descriptions +from library_generation.generate_repo import generate_from_yaml +from library_generation.model.generation_config import from_yaml +from library_generation.utils.generation_config_comparator import compare_config + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--baseline-generation-config", + required=True, + type=str, + help=""" + + """, +) +@click.option( + "--latest-generation-config", + required=True, + type=str, + help=""" + Path to generation_config.yaml that contains the metadata about + library generation. + The googleapis commit in the configuration is the latest commit, + inclusively, from which the commit message is considered. + """, +) +@click.option( + "--repository-path", + type=str, + default="https://github.com/googleapis/googleapis.git", + show_default=True, + help=""" + + """, +) +def generate( + baseline_generation_config: str, + latest_generation_config: str, + repository_path: str, +): + # convert paths to absolute paths, so they can be correctly referenced in + # downstream scripts + baseline_generation_config = os.path.abspath(baseline_generation_config) + latest_generation_config = os.path.abspath(latest_generation_config) + repository_path = os.path.abspath(repository_path) + config_change = compare_config( + baseline_config=from_yaml(baseline_generation_config), + latest_config=from_yaml(latest_generation_config), + ) + # generate libraries + generate_from_yaml( + config=config_change.latest_config, + repository_path=repository_path, + target_library_names=config_change.get_changed_libraries(), + ) + # generate pull request description + generate_pr_descriptions( + config=config_change.latest_config, + baseline_commit=config_change.baseline_config.googleapis_commitish, + ) + + +if __name__ == "__main__": + main() From 964d81a8e0d859b27e7012c816e10aca9b9677c0 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Apr 2024 19:01:51 -0400 Subject: [PATCH 02/34] remove cli in generate_repo.py --- library_generation/generate_repo.py | 80 +---------------------------- 1 file changed, 2 insertions(+), 78 deletions(-) diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index 9af3a08c56..f0b0e2a5c1 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -12,88 +12,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import library_generation.utils.utilities as util -import click -import os from library_generation.generate_composed_library import generate_composed_library from library_generation.model.generation_config import GenerationConfig -from library_generation.model.generation_config import from_yaml from library_generation.model.library_config import LibraryConfig from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--generation-config-yaml", - required=True, - type=str, - help=""" - Path to generation_config.yaml that contains the metadata about - library generation - """, -) -@click.option( - "--target-library-names", - required=False, - default=None, - type=str, - help=""" - A list of libraries will be generated. - - If specified, only the `library` whose library_name is in - target-library-names will be generated. - If not specified, all libraries in the configuration yaml will be generated. - - The input string will be parsed to a list of string with comma as the - separator. - - For example, apigeeconnect,alloydb-connectors will be parsed as a - list of two strings, apigeeconnect and alloydb-connectors. - """, -) -@click.option( - "--repository-path", - required=False, - default=".", - type=str, - help=""" - If specified, the generated files will be sent to this location. - If not specified, the repository will be generated to the current working - directory. - """, -) -def generate( - generation_config_yaml: str, - target_library_names: str, - repository_path: str, -): - generate_from_yaml( - generation_config_yaml=generation_config_yaml, - repository_path=repository_path, - target_library_names=target_library_names.split(",") - if target_library_names is not None - else target_library_names, - ) - - def generate_from_yaml( - generation_config_yaml: str, + config: GenerationConfig, repository_path: str, target_library_names: list[str] = None, ) -> None: """ Parses a config yaml and generates libraries via generate_composed_library.py - :param generation_config_yaml: Path to generation_config.yaml that contains - the metadata about library generation + :param config: :param repository_path: If specified, the generated files will be sent to this location. If not specified, the repository will be generated to the current working directory. @@ -103,12 +37,6 @@ def generate_from_yaml( If specified with an empty list, then no library will be generated. If not specified, all libraries in the configuration yaml will be generated. """ - # convert paths to absolute paths, so they can be correctly referenced in - # downstream scripts - generation_config_yaml = os.path.abspath(generation_config_yaml) - repository_path = os.path.abspath(repository_path) - - config = from_yaml(generation_config_yaml) target_libraries = get_target_libraries( config=config, target_library_names=target_library_names ) @@ -155,7 +83,3 @@ def get_target_libraries( for library in config.libraries if library.get_library_name() in target_libraries ] - - -if __name__ == "__main__": - main() From 5b303cb4937595e4d057dca80196d51b86ad2763 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Apr 2024 19:02:09 -0400 Subject: [PATCH 03/34] remove cli in generate_pr_description.py --- library_generation/generate_pr_description.py | 68 +------------------ 1 file changed, 3 insertions(+), 65 deletions(-) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index 2af636d2fe..68af4e393e 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -15,76 +15,18 @@ import os import shutil from typing import Dict - -import click from git import Commit, Repo -from library_generation.model.generation_config import from_yaml +from library_generation.model.generation_config import GenerationConfig from library_generation.utils.proto_path_utils import find_versioned_proto_path from library_generation.utils.commit_message_formatter import format_commit_message from library_generation.utils.commit_message_formatter import wrap_override_commit -@click.group(invoke_without_command=False) -@click.pass_context -@click.version_option(message="%(version)s") -def main(ctx): - pass - - -@main.command() -@click.option( - "--generation-config-yaml", - required=True, - type=str, - help=""" - Path to generation_config.yaml that contains the metadata about - library generation. - The googleapis commit in the configuration is the latest commit, - inclusively, from which the commit message is considered. - """, -) -@click.option( - "--baseline-commit", - required=True, - type=str, - help=""" - The baseline (oldest) commit, exclusively, from which the commit message is - considered. - This commit should be an ancestor of googleapis commit in configuration. - """, -) -@click.option( - "--repo-url", - type=str, - default="https://github.com/googleapis/googleapis.git", - show_default=True, - help=""" - GitHub repository URL. - """, -) -def generate( - generation_config_yaml: str, - repo_url: str, - baseline_commit: str, -) -> str: - description = generate_pr_descriptions( - generation_config_yaml=generation_config_yaml, - repo_url=repo_url, - baseline_commit=baseline_commit, - ) - idx = generation_config_yaml.rfind("/") - config_path = generation_config_yaml[:idx] - with open(f"{config_path}/pr_description.txt", "w+") as f: - f.write(description) - return description - - def generate_pr_descriptions( - generation_config_yaml: str, - repo_url: str, + config: GenerationConfig, baseline_commit: str, + repo_url: str = "https://github.com/googleapis/googleapis.git", ) -> str: - config = from_yaml(generation_config_yaml) paths = config.get_proto_path_to_library_name() return __get_commit_messages( repo_url=repo_url, @@ -181,7 +123,3 @@ def __combine_commit_messages( ) return "\n".join(messages) - - -if __name__ == "__main__": - main() From 4f6cedd6ab8c849342aea85a62412e7870575b07 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Apr 2024 19:22:39 -0400 Subject: [PATCH 04/34] add comment --- library_generation/cli/entry_point.py | 15 ++++++++---- library_generation/generate_pr_description.py | 23 +++++++++++++++++-- library_generation/generate_repo.py | 10 ++++---- 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index e493c86797..35e2d43429 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -33,7 +33,10 @@ def main(ctx): required=True, type=str, help=""" - + Path to generation_config.yaml that contains the metadata about + library generation. + The googleapis commit in the configuration is the baseline commit, + exclusively, from which the commit message is considered. """, ) @click.option( @@ -44,16 +47,19 @@ def main(ctx): Path to generation_config.yaml that contains the metadata about library generation. The googleapis commit in the configuration is the latest commit, - inclusively, from which the commit message is considered. + inclusively, to which the commit message is considered. """, ) @click.option( "--repository-path", type=str, - default="https://github.com/googleapis/googleapis.git", + default=".", show_default=True, help=""" - + The repository path to which the generated files + will be sent. + If not specified, the repository will be generated to the current working + directory. """, ) def generate( @@ -80,6 +86,7 @@ def generate( generate_pr_descriptions( config=config_change.latest_config, baseline_commit=config_change.baseline_config.googleapis_commitish, + description_path=repository_path, ) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index 68af4e393e..2a1008cca6 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -25,10 +25,26 @@ def generate_pr_descriptions( config: GenerationConfig, baseline_commit: str, + description_path: str, repo_url: str = "https://github.com/googleapis/googleapis.git", -) -> str: +) -> None: + """ + Generate pull request description from baseline_commit (exclusive) to the + googleapis commit (inclusive) in the given generation config. + + :param config: a GenerationConfig object. The googleapis commit in this + configuration is the latest commit, inclusively, from which the commit + message is considered. + :param baseline_commit: The baseline (oldest) commit, exclusively, from + which the commit message is considered. This commit should be an ancestor + of googleapis commit in configuration. + :param description_path: + :param repo_url: the GitHub repository from which retrieves the commit + history. + :return: + """ paths = config.get_proto_path_to_library_name() - return __get_commit_messages( + description = __get_commit_messages( repo_url=repo_url, latest_commit=config.googleapis_commitish, baseline_commit=baseline_commit, @@ -36,6 +52,9 @@ def generate_pr_descriptions( is_monorepo=config.is_monorepo, ) + with open(f"{description_path}/pr_description.txt", "w+") as f: + f.write(description) + def __get_commit_messages( repo_url: str, diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index f0b0e2a5c1..d196815fcf 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -25,12 +25,12 @@ def generate_from_yaml( target_library_names: list[str] = None, ) -> None: """ - Parses a config yaml and generates libraries via + Based on the generation config, generates libraries via generate_composed_library.py - :param config: - :param repository_path: If specified, the generated files will be sent to - this location. If not specified, the repository will be generated to the - current working directory. + + :param config: a GenerationConfig object. + :param repository_path: The repository path to which the generated files + will be sent. :param target_library_names: a list of libraries to be generated. If specified, only the library whose library_name is in target-library-names will be generated. From 531244defcd50f1d5921c1f60456df572b2ce003 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Apr 2024 19:34:27 -0400 Subject: [PATCH 05/34] fix integration test --- library_generation/test/integration_tests.py | 48 +++------------ .../baseline_generation_config.yaml | 58 +++++++++++++++++++ 2 files changed, 66 insertions(+), 40 deletions(-) create mode 100644 library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 2acfa7d3b9..8175f4ed9a 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -42,9 +42,8 @@ "java-bigtable": "chore/test-hermetic-build", } config_dir = f"{script_dir}/resources/integration" -config_name = "generation_config.yaml" -monorepo_baseline_commit = "a17d4caf184b050d50cacf2b0d579ce72c31ce74" -split_repo_baseline_commit = "679060c64136e85b52838f53cfe612ce51e60d1d" +baseline_config_name = "baseline_generation_config.yaml" +latest_config_name = "latest_generation_config.yaml" class IntegrationTest(unittest.TestCase): @@ -74,15 +73,11 @@ def test_entry_point_running_in_container(self): ) repo_volumes = f"-v repo-{repo}:/workspace/{repo} -v config-{repo}:/workspace/config-{repo}" # 4. run entry_point.py in docker container - baseline_commit = ( - monorepo_baseline_commit - if repo == "google-cloud-java" - else split_repo_baseline_commit - ) self.__run_entry_point_in_docker_container( repo=repo, repo_volumes=repo_volumes, - baseline_commit=baseline_commit, + baseline_config=baseline_config_name, + latest_config=latest_config_name, ) # 5. compare generation result with golden files print( @@ -243,7 +238,7 @@ def __bind_device_to_volumes(cls, volume_name: str, device_dir: str): @classmethod def __run_entry_point_in_docker_container( - cls, repo: str, repo_volumes: str, baseline_commit: str + cls, repo: str, repo_volumes: str, baseline_config: str, latest_config: str ): subprocess.check_call( [ @@ -266,41 +261,14 @@ def __run_entry_point_in_docker_container( "/src", image_tag, "python", - "/src/generate_repo.py", + "/src/cli/entry_point.py", "generate", - f"--generation-config-yaml=/workspace/config-{repo}/{config_name}", + f"--baseline-generation-config=/workspace/config-{repo}/{baseline_config}", + f"--latest-generation-config=/workspace/config-{repo}/{latest_config}", f"--repository-path=/workspace/{repo}", ] ) - subprocess.check_call( - [ - "docker", - "run", - "--rm", - "-v", - f"repo-{repo}:/workspace/{repo}", - "-v", - f"config-{repo}:/workspace/config-{repo}", - "-v", - "/tmp:/tmp", - "-v", - "/var/run/docker.sock:/var/run/docker.sock", - "-e", - "RUNNING_IN_DOCKER=true", - "-e", - f"REPO_BINDING_VOLUMES={repo_volumes}", - "-w", - "/src", - image_tag, - "python", - "/src/generate_pr_description.py", - "generate", - f"--generation-config-yaml=/workspace/config-{repo}/{config_name}", - f"--baseline-commit={baseline_commit}", - ] - ) - @classmethod def __get_config_files(cls, path: str) -> list[tuple[str, str]]: config_files = [] diff --git a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml new file mode 100644 index 0000000000..839f80996e --- /dev/null +++ b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml @@ -0,0 +1,58 @@ +gapic_generator_version: 2.37.0 +protobuf_version: 25.2 +googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc +owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 +synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e +template_excludes: + - ".github/*" + - ".kokoro/*" + - "samples/*" + - "CODE_OF_CONDUCT.md" + - "CONTRIBUTING.md" + - "LICENSE" + - "SECURITY.md" + - "java.header" + - "license-checks.xml" + - "renovate.json" + - ".gitignore" +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." + release_level: "stable" + library_name: "apigee-connect" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 + + - api_shortname: alloydb + name_pretty: AlloyDB + product_documentation: https://cloud.google.com/alloydb/ + api_description: AlloyDB is a fully managed, PostgreSQL-compatible database service + with industry-leading performance, availability, and scale. + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/v1 + - proto_path: google/cloud/alloydb/v1alpha + - proto_path: google/cloud/alloydb/v1beta + + - api_shortname: alloydb + name_pretty: AlloyDB connectors + product_documentation: https://cloud.google.com/alloydb/docs + api_description: AlloyDB is a fully-managed, PostgreSQL-compatible database for + demanding transactional workloads. It provides enterprise-grade performance and + availability while maintaining 100% compatibility with open-source PostgreSQL. + library_name: alloydb-connectors + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/connectors/v1 + - proto_path: google/cloud/alloydb/connectors/v1alpha + - proto_path: google/cloud/alloydb/connectors/v1beta + + - api_shortname: cloudcontrolspartner + name_pretty: Cloud Controls Partner API + product_documentation: https://cloud.google.com/sovereign-controls-by-partners/docs/sovereign-partners + api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. + GAPICs: + - proto_path: google/cloud/cloudcontrolspartner/v1 + - proto_path: google/cloud/cloudcontrolspartner/v1beta \ No newline at end of file From a6811d5122b5f59a58269e36002c26288524e94a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Apr 2024 19:38:28 -0400 Subject: [PATCH 06/34] rename config --- library_generation/test/integration_tests.py | 2 +- .../google-cloud-java/baseline_generation_config.yaml | 2 +- .../{generation_config.yaml => latest_generation_config.yaml} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename library_generation/test/resources/integration/google-cloud-java/{generation_config.yaml => latest_generation_config.yaml} (100%) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 8175f4ed9a..d4a3181219 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -278,7 +278,7 @@ def __get_config_files(cls, path: str) -> list[tuple[str, str]]: repo = sub_dir.name if repo in ["golden", "java-bigtable"]: continue - config = f"{sub_dir}/{config_name}" + config = f"{sub_dir}/{latest_config_name}" config_files.append((repo, config)) return config_files diff --git a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml index 839f80996e..ad9eafb31e 100644 --- a/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/baseline_generation_config.yaml @@ -1,6 +1,6 @@ gapic_generator_version: 2.37.0 protobuf_version: 25.2 -googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc +googleapis_commitish: a17d4caf184b050d50cacf2b0d579ce72c31ce74 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e template_excludes: diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/latest_generation_config.yaml similarity index 100% rename from library_generation/test/resources/integration/google-cloud-java/generation_config.yaml rename to library_generation/test/resources/integration/google-cloud-java/latest_generation_config.yaml From 14cdfb9d5bf47a90b9821a7c3ae1db8258cb84c3 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Apr 2024 19:49:17 -0400 Subject: [PATCH 07/34] change pr description file path --- library_generation/generate_pr_description.py | 4 +++- library_generation/test/integration_tests.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index 2a1008cca6..af6b805c4c 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -52,7 +52,9 @@ def generate_pr_descriptions( is_monorepo=config.is_monorepo, ) - with open(f"{description_path}/pr_description.txt", "w+") as f: + description_file = f"{description_path}/pr_description.txt" + print(f"Writing pull request description to {description_file}") + with open(description_file, "w+") as f: f.write(description) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index d4a3181219..3d129a6d0e 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -157,7 +157,7 @@ def test_entry_point_running_in_container(self): ) print(" pom.xml comparison succeed.") # compare PR description - description_file = f"{config_dir}/{repo}/pr_description.txt" + description_file = f"{output_dir}/{repo}/pr_description.txt" self.assertTrue( cmp( f"{config_dir}/{repo}/pr-description-golden.txt", From 24f2070dcdd3fc1d8c1f4f4cd8535911147a600e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Apr 2024 21:11:16 -0400 Subject: [PATCH 08/34] add comments --- library_generation/generate_pr_description.py | 7 +++++-- library_generation/generate_repo.py | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index af6b805c4c..db771ee8c5 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -32,16 +32,19 @@ def generate_pr_descriptions( Generate pull request description from baseline_commit (exclusive) to the googleapis commit (inclusive) in the given generation config. + The pull request description will be generated into + description_path/pr_description.txt. + :param config: a GenerationConfig object. The googleapis commit in this configuration is the latest commit, inclusively, from which the commit message is considered. :param baseline_commit: The baseline (oldest) commit, exclusively, from which the commit message is considered. This commit should be an ancestor of googleapis commit in configuration. - :param description_path: + :param description_path: the path to which the pull request description + file goes. :param repo_url: the GitHub repository from which retrieves the commit history. - :return: """ paths = config.get_proto_path_to_library_name() description = __get_commit_messages( diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index d196815fcf..2be89a7ff4 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -32,8 +32,8 @@ def generate_from_yaml( :param repository_path: The repository path to which the generated files will be sent. :param target_library_names: a list of libraries to be generated. - If specified, only the library whose library_name is in - target-library-names will be generated. + If specified, only the library whose library_name is in target_library_names + will be generated. If specified with an empty list, then no library will be generated. If not specified, all libraries in the configuration yaml will be generated. """ @@ -45,7 +45,7 @@ def generate_from_yaml( ) for library_path, library in repo_config.libraries.items(): - print(f"generating library {library.api_shortname}") + print(f"generating library {library.get_library_name()}") generate_composed_library( config=config, From 853b70c5599c271a9571cbd9a47eaef8cef9a223 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Apr 2024 10:58:03 -0400 Subject: [PATCH 09/34] restore CLI --- library_generation/generate_pr_description.py | 49 ++++++++++++- library_generation/generate_repo.py | 70 ++++++++++++++++++- 2 files changed, 117 insertions(+), 2 deletions(-) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index db771ee8c5..ba0a4e2b77 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -14,13 +14,56 @@ # limitations under the License. import os import shutil +import click as click from typing import Dict from git import Commit, Repo -from library_generation.model.generation_config import GenerationConfig +from library_generation.model.generation_config import GenerationConfig, \ + from_yaml from library_generation.utils.proto_path_utils import find_versioned_proto_path from library_generation.utils.commit_message_formatter import format_commit_message from library_generation.utils.commit_message_formatter import wrap_override_commit +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--generation-config-yaml", + required=True, + type=str, + help=""" + Path to generation_config.yaml that contains the metadata about + library generation. + The googleapis commit in the configuration is the latest commit, + inclusively, from which the commit message is considered. + """, +) +@click.option( + "--baseline-commit", + required=True, + type=str, + help=""" + The baseline (oldest) commit, exclusively, from which the commit message is + considered. + This commit should be an ancestor of googleapis commit in configuration. + """, +) +def generate( + generation_config_yaml: str, + baseline_commit: str, +) -> None: + idx = generation_config_yaml.rfind("/") + config_path = generation_config_yaml[:idx] + generate_pr_descriptions( + config=from_yaml(generation_config_yaml), + baseline_commit=baseline_commit, + description_path=config_path + ) + def generate_pr_descriptions( config: GenerationConfig, @@ -147,3 +190,7 @@ def __combine_commit_messages( ) return "\n".join(messages) + + +if __name__ == "__main__": + main() diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index 2be89a7ff4..9cfb64cb12 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -12,13 +12,77 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import click as click import library_generation.utils.utilities as util from library_generation.generate_composed_library import generate_composed_library -from library_generation.model.generation_config import GenerationConfig +from library_generation.model.generation_config import GenerationConfig, \ + from_yaml from library_generation.model.library_config import LibraryConfig from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--generation-config-yaml", + required=True, + type=str, + help=""" + Path to generation_config.yaml that contains the metadata about + library generation + """, +) +@click.option( + "--target-library-names", + required=False, + default=None, + type=str, + help=""" + A list of libraries will be generated. + + If specified, only the `library` whose library_name is in + target-library-names will be generated. + If not specified, all libraries in the configuration yaml will be generated. + + The input string will be parsed to a list of string with comma as the + separator. + + For example, apigeeconnect,alloydb-connectors will be parsed as a + list of two strings, apigeeconnect and alloydb-connectors. + """, +) +@click.option( + "--repository-path", + required=False, + default=".", + type=str, + help=""" + If specified, the generated files will be sent to this location. + If not specified, the repository will be generated to the current working + directory. + """, +) +def generate( + generation_config_yaml: str, + target_library_names: str, + repository_path: str, +): + config = from_yaml(generation_config_yaml) + generate_from_yaml( + config=config, + repository_path=repository_path, + target_library_names=target_library_names.split(",") + if target_library_names is not None + else target_library_names, + ) + + def generate_from_yaml( config: GenerationConfig, repository_path: str, @@ -83,3 +147,7 @@ def get_target_libraries( for library in config.libraries if library.get_library_name() in target_libraries ] + + +if __name__ == "__main__": + main() From 923f14908e36490d4ca8d1b65f92337b4507a319 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Apr 2024 10:59:54 -0400 Subject: [PATCH 10/34] format code --- library_generation/generate_pr_description.py | 6 +++--- library_generation/generate_repo.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index ba0a4e2b77..b976e4fe80 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -17,12 +17,12 @@ import click as click from typing import Dict from git import Commit, Repo -from library_generation.model.generation_config import GenerationConfig, \ - from_yaml +from library_generation.model.generation_config import GenerationConfig, from_yaml from library_generation.utils.proto_path_utils import find_versioned_proto_path from library_generation.utils.commit_message_formatter import format_commit_message from library_generation.utils.commit_message_formatter import wrap_override_commit + @click.group(invoke_without_command=False) @click.pass_context @click.version_option(message="%(version)s") @@ -61,7 +61,7 @@ def generate( generate_pr_descriptions( config=from_yaml(generation_config_yaml), baseline_commit=baseline_commit, - description_path=config_path + description_path=config_path, ) diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index 9cfb64cb12..a15cfb5f2a 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -15,8 +15,7 @@ import click as click import library_generation.utils.utilities as util from library_generation.generate_composed_library import generate_composed_library -from library_generation.model.generation_config import GenerationConfig, \ - from_yaml +from library_generation.model.generation_config import GenerationConfig, from_yaml from library_generation.model.library_config import LibraryConfig from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing From 1cd3859d09aff9db541dc3f26cf3d4a4278703c9 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Apr 2024 19:21:54 -0400 Subject: [PATCH 11/34] change input to current_generation_config --- library_generation/cli/entry_point.py | 16 +-- library_generation/model/config_change.py | 14 +-- library_generation/test/integration_tests.py | 10 +- .../test/model/config_change_unit_tests.py | 30 ++--- ...ig.yaml => current_generation_config.yaml} | 0 ...generation_config_comparator_unit_tests.py | 114 +++++++++--------- .../utils/generation_config_comparator.py | 80 ++++++------ 7 files changed, 132 insertions(+), 132 deletions(-) rename library_generation/test/resources/integration/google-cloud-java/{latest_generation_config.yaml => current_generation_config.yaml} (100%) diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 35e2d43429..89d7d42602 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -35,18 +35,18 @@ def main(ctx): help=""" Path to generation_config.yaml that contains the metadata about library generation. - The googleapis commit in the configuration is the baseline commit, + The googleapis commit in the configuration is the oldest commit, exclusively, from which the commit message is considered. """, ) @click.option( - "--latest-generation-config", + "--current-generation-config", required=True, type=str, help=""" Path to generation_config.yaml that contains the metadata about library generation. - The googleapis commit in the configuration is the latest commit, + The googleapis commit in the configuration is the newest commit, inclusively, to which the commit message is considered. """, ) @@ -64,27 +64,27 @@ def main(ctx): ) def generate( baseline_generation_config: str, - latest_generation_config: str, + current_generation_config: str, repository_path: str, ): # convert paths to absolute paths, so they can be correctly referenced in # downstream scripts baseline_generation_config = os.path.abspath(baseline_generation_config) - latest_generation_config = os.path.abspath(latest_generation_config) + current_generation_config = os.path.abspath(current_generation_config) repository_path = os.path.abspath(repository_path) config_change = compare_config( baseline_config=from_yaml(baseline_generation_config), - latest_config=from_yaml(latest_generation_config), + current_config=from_yaml(current_generation_config), ) # generate libraries generate_from_yaml( - config=config_change.latest_config, + config=config_change.current_config, repository_path=repository_path, target_library_names=config_change.get_changed_libraries(), ) # generate pull request description generate_pr_descriptions( - config=config_change.latest_config, + config=config_change.current_config, baseline_commit=config_change.baseline_config.googleapis_commitish, description_path=repository_path, ) diff --git a/library_generation/model/config_change.py b/library_generation/model/config_change.py index 5b1a97d22a..bfb83a87e6 100644 --- a/library_generation/model/config_change.py +++ b/library_generation/model/config_change.py @@ -47,9 +47,9 @@ def __init__(self, hash_value: int, library: LibraryConfig): class LibraryChange: - def __init__(self, changed_param: str, latest_value: str, library_name: str = ""): + def __init__(self, changed_param: str, current_value: str, library_name: str = ""): self.changed_param = changed_param - self.latest_value = latest_value + self.current_value = current_value self.library_name = library_name @@ -66,17 +66,17 @@ def __init__( self, change_to_libraries: dict[ChangeType, list[LibraryChange]], baseline_config: GenerationConfig, - latest_config: GenerationConfig, + current_config: GenerationConfig, ): self.change_to_libraries = change_to_libraries self.baseline_config = baseline_config - self.latest_config = latest_config + self.current_config = current_config def get_changed_libraries(self) -> Optional[list[str]]: """ Returns a unique, sorted list of library name of changed libraries. None if there is a repository level change, which means all libraries - in the latest_config will be generated. + in the current_config will be generated. :return: library names of change libraries. """ if ChangeType.REPO_LEVEL_CHANGE in self.change_to_libraries: @@ -108,8 +108,8 @@ def get_qualified_commits( os.mkdir(tmp_dir) # we only need commit history, thus shadow clone is enough. repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"]) - commit = repo.commit(self.latest_config.googleapis_commitish) - proto_paths = self.latest_config.get_proto_path_to_library_name() + commit = repo.commit(self.current_config.googleapis_commitish) + proto_paths = self.current_config.get_proto_path_to_library_name() qualified_commits = [] while str(commit.hexsha) != self.baseline_config.googleapis_commitish: qualified_commit = ConfigChange.__create_qualified_commit( diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 3d129a6d0e..3c10e1fdf1 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -43,7 +43,7 @@ } config_dir = f"{script_dir}/resources/integration" baseline_config_name = "baseline_generation_config.yaml" -latest_config_name = "latest_generation_config.yaml" +current_config_name = "current_generation_config.yaml" class IntegrationTest(unittest.TestCase): @@ -77,7 +77,7 @@ def test_entry_point_running_in_container(self): repo=repo, repo_volumes=repo_volumes, baseline_config=baseline_config_name, - latest_config=latest_config_name, + current_config=current_config_name, ) # 5. compare generation result with golden files print( @@ -238,7 +238,7 @@ def __bind_device_to_volumes(cls, volume_name: str, device_dir: str): @classmethod def __run_entry_point_in_docker_container( - cls, repo: str, repo_volumes: str, baseline_config: str, latest_config: str + cls, repo: str, repo_volumes: str, baseline_config: str, current_config: str ): subprocess.check_call( [ @@ -264,7 +264,7 @@ def __run_entry_point_in_docker_container( "/src/cli/entry_point.py", "generate", f"--baseline-generation-config=/workspace/config-{repo}/{baseline_config}", - f"--latest-generation-config=/workspace/config-{repo}/{latest_config}", + f"--current-generation-config=/workspace/config-{repo}/{current_config}", f"--repository-path=/workspace/{repo}", ] ) @@ -278,7 +278,7 @@ def __get_config_files(cls, path: str) -> list[tuple[str, str]]: repo = sub_dir.name if repo in ["golden", "java-bigtable"]: continue - config = f"{sub_dir}/{latest_config_name}" + config = f"{sub_dir}/{current_config_name}" config_files.append((repo, config)) return config_files diff --git a/library_generation/test/model/config_change_unit_tests.py b/library_generation/test/model/config_change_unit_tests.py index d6309c98c1..5df6e8b021 100644 --- a/library_generation/test/model/config_change_unit_tests.py +++ b/library_generation/test/model/config_change_unit_tests.py @@ -33,13 +33,13 @@ def test_get_changed_libraries_with_repo_level_change_returns_all_libraries_chan ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="test-library", ) ], }, baseline_config=ConfigChangeTest.__get_a_gen_config(), - latest_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), ) self.assertEqual( ConfigChange.ALL_LIBRARIES_CHANGED, @@ -52,32 +52,32 @@ def test_get_changed_libraries_with_library_level_change_returns_list(self): ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="a-library", ), LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="another-library", ), ], ChangeType.LIBRARIES_ADDITION: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="new-library", ), ], ChangeType.GAPIC_ADDITION: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="library-with-new-version", ), ], }, baseline_config=ConfigChangeTest.__get_a_gen_config(), - latest_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), ) self.assertEqual( ["a-library", "another-library", "library-with-new-version", "new-library"], @@ -92,18 +92,18 @@ def test_get_changed_libraries_with_duplicated_library_name_returns_unique_name( ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="a-param", - latest_value="new_test", + current_value="new_test", library_name="a-library", ), LibraryChange( changed_param="another-param", - latest_value="new_value", + current_value="new_value", library_name="a-library", ), ], }, baseline_config=ConfigChangeTest.__get_a_gen_config(), - latest_config=ConfigChangeTest.__get_a_gen_config(), + current_config=ConfigChangeTest.__get_a_gen_config(), ) self.assertEqual( ["a-library"], @@ -119,14 +119,14 @@ def test_get_changed_libraries_with_mix_changes_returns_list(self): ChangeType.LIBRARY_LEVEL_CHANGE: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="a-library", ) ], ChangeType.LIBRARIES_ADDITION: [ LibraryChange( changed_param="test", - latest_value="test", + current_value="test", library_name="new-library", ), ], @@ -134,7 +134,7 @@ def test_get_changed_libraries_with_mix_changes_returns_list(self): baseline_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=baseline_commit ), - latest_config=ConfigChangeTest.__get_a_gen_config( + current_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=latest_commit, libraries=[ ConfigChangeTest.__get_a_library_config( @@ -163,7 +163,7 @@ def test_get_qualified_commits_success(self): baseline_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=baseline_commit ), - latest_config=ConfigChangeTest.__get_a_gen_config( + current_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=latest_commit, libraries=[ ConfigChangeTest.__get_a_library_config( @@ -216,7 +216,7 @@ def test_get_qualified_commits_build_only_commit_returns_empty_list(self): baseline_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=baseline_commit ), - latest_config=ConfigChangeTest.__get_a_gen_config( + current_config=ConfigChangeTest.__get_a_gen_config( googleapis_commitish=latest_commit, libraries=[ ConfigChangeTest.__get_a_library_config( diff --git a/library_generation/test/resources/integration/google-cloud-java/latest_generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml similarity index 100% rename from library_generation/test/resources/integration/google-cloud-java/latest_generation_config.yaml rename to library_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml diff --git a/library_generation/test/utils/generation_config_comparator_unit_tests.py b/library_generation/test/utils/generation_config_comparator_unit_tests.py index e526d6856b..c25c3eff98 100644 --- a/library_generation/test/utils/generation_config_comparator_unit_tests.py +++ b/library_generation/test/utils/generation_config_comparator_unit_tests.py @@ -62,7 +62,7 @@ def setUp(self) -> None: def test_compare_config_not_change(self): result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue(len(result.change_to_libraries) == 0) @@ -75,7 +75,7 @@ def test_compare_config_googleapis_update(self): ) result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertEqual({ChangeType.GOOGLEAPIS_COMMIT: []}, result.change_to_libraries) @@ -84,70 +84,70 @@ def test_compare_config_generator_update(self): self.latest_config.gapic_generator_version = "1.2.4" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("gapic_generator_version", config_change.changed_param) - self.assertEqual("1.2.4", config_change.latest_value) + self.assertEqual("1.2.4", config_change.current_value) def test_compare_config_owlbot_cli_update(self): self.baseline_config.owlbot_cli_image = "image_version_123" self.latest_config.owlbot_cli_image = "image_version_456" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("owlbot_cli_image", config_change.changed_param) - self.assertEqual("image_version_456", config_change.latest_value) + self.assertEqual("image_version_456", config_change.current_value) def test_compare_config_synthtool_update(self): self.baseline_config.synthtool_commitish = "commit123" self.latest_config.synthtool_commitish = "commit456" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("synthtool_commitish", config_change.changed_param) - self.assertEqual("commit456", config_change.latest_value) + self.assertEqual("commit456", config_change.current_value) def test_compare_protobuf_update(self): self.baseline_config.protobuf_version = "3.25.2" self.latest_config.protobuf_version = "3.27.0" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("protobuf_version", config_change.changed_param) - self.assertEqual("3.27.0", config_change.latest_value) + self.assertEqual("3.27.0", config_change.current_value) def test_compare_config_grpc_update(self): self.baseline_config.grpc_version = "1.60.0" self.latest_config.grpc_version = "1.61.0" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE][0] self.assertEqual("grpc_version", config_change.changed_param) - self.assertEqual("1.61.0", config_change.latest_value) + self.assertEqual("1.61.0", config_change.current_value) def test_compare_config_template_excludes_update(self): self.baseline_config.template_excludes = [".github/*", ".kokoro/*"] @@ -159,7 +159,7 @@ def test_compare_config_template_excludes_update(self): ] result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.REPO_LEVEL_CHANGE]) == 1 @@ -173,7 +173,7 @@ def test_compare_config_template_excludes_update(self): "samples/*", "CODE_OF_CONDUCT.md", ], - config_change.latest_value, + config_change.current_value, ) def test_compare_config_library_addition(self): @@ -188,7 +188,7 @@ def test_compare_config_library_addition(self): ) result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 @@ -200,7 +200,7 @@ def test_compare_config_api_shortname_update_without_library_name(self): self.latest_config.libraries[0].api_shortname = "new_api_shortname" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 @@ -224,7 +224,7 @@ def test_compare_config_library_name_update(self): self.latest_config.libraries[0].library_name = "new_library_name" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARIES_ADDITION]) == 1 @@ -236,266 +236,266 @@ def test_compare_config_api_description_update(self): self.latest_config.libraries[0].api_description = "updated description" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_description", config_change.changed_param) - self.assertEqual("updated description", config_change.latest_value) + self.assertEqual("updated description", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_name_pretty_update(self): self.latest_config.libraries[0].name_pretty = "new name" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("name_pretty", config_change.changed_param) - self.assertEqual("new name", config_change.latest_value) + self.assertEqual("new name", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_product_docs_update(self): self.latest_config.libraries[0].product_documentation = "new docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("product_documentation", config_change.changed_param) - self.assertEqual("new docs", config_change.latest_value) + self.assertEqual("new docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_library_type_update(self): self.latest_config.libraries[0].library_type = "GAPIC_COMBO" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("library_type", config_change.changed_param) - self.assertEqual("GAPIC_COMBO", config_change.latest_value) + self.assertEqual("GAPIC_COMBO", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_release_level_update(self): self.latest_config.libraries[0].release_level = "STABLE" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("release_level", config_change.changed_param) - self.assertEqual("STABLE", config_change.latest_value) + self.assertEqual("STABLE", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_api_id_update(self): self.latest_config.libraries[0].api_id = "new_id" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_id", config_change.changed_param) - self.assertEqual("new_id", config_change.latest_value) + self.assertEqual("new_id", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_api_reference_update(self): self.latest_config.libraries[0].api_reference = "new api_reference" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("api_reference", config_change.changed_param) - self.assertEqual("new api_reference", config_change.latest_value) + self.assertEqual("new api_reference", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_code_owner_team_update(self): self.latest_config.libraries[0].codeowner_team = "new team" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("codeowner_team", config_change.changed_param) - self.assertEqual("new team", config_change.latest_value) + self.assertEqual("new team", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_excluded_deps_update(self): self.latest_config.libraries[0].excluded_dependencies = "group:artifact" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("excluded_dependencies", config_change.changed_param) - self.assertEqual("group:artifact", config_change.latest_value) + self.assertEqual("group:artifact", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_excluded_poms_update(self): self.latest_config.libraries[0].excluded_poms = "pom.xml" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("excluded_poms", config_change.changed_param) - self.assertEqual("pom.xml", config_change.latest_value) + self.assertEqual("pom.xml", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_client_docs_update(self): self.latest_config.libraries[0].client_documentation = "new client docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("client_documentation", config_change.changed_param) - self.assertEqual("new client docs", config_change.latest_value) + self.assertEqual("new client docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_distribution_name_update(self): self.latest_config.libraries[0].distribution_name = "new_group:new_artifact" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("distribution_name", config_change.changed_param) - self.assertEqual("new_group:new_artifact", config_change.latest_value) + self.assertEqual("new_group:new_artifact", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_group_id_update(self): self.latest_config.libraries[0].group_id = "new_group" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("group_id", config_change.changed_param) - self.assertEqual("new_group", config_change.latest_value) + self.assertEqual("new_group", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_issue_tracker_update(self): self.latest_config.libraries[0].issue_tracker = "new issue tracker" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("issue_tracker", config_change.changed_param) - self.assertEqual("new issue tracker", config_change.latest_value) + self.assertEqual("new issue tracker", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_rest_docs_update(self): self.latest_config.libraries[0].rest_documentation = "new rest docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("rest_documentation", config_change.changed_param) - self.assertEqual("new rest docs", config_change.latest_value) + self.assertEqual("new rest docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_rpc_docs_update(self): self.latest_config.libraries[0].rpc_documentation = "new rpc docs" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("rpc_documentation", config_change.changed_param) - self.assertEqual("new rpc docs", config_change.latest_value) + self.assertEqual("new rpc docs", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_cloud_api_update(self): self.latest_config.libraries[0].cloud_api = False result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("cloud_api", config_change.changed_param) - self.assertEqual(False, config_change.latest_value) + self.assertEqual(False, config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_requires_billing_update(self): self.latest_config.libraries[0].requires_billing = False result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("requires_billing", config_change.changed_param) - self.assertEqual(False, config_change.latest_value) + self.assertEqual(False, config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_extra_versioned_mod_update(self): self.latest_config.libraries[0].extra_versioned_modules = "extra module" result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue( len(result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE]) == 1 ) config_change = result.change_to_libraries[ChangeType.LIBRARY_LEVEL_CHANGE][0] self.assertEqual("extra_versioned_modules", config_change.changed_param) - self.assertEqual("extra module", config_change.latest_value) + self.assertEqual("extra module", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) def test_compare_config_version_addition(self): @@ -504,10 +504,10 @@ def test_compare_config_version_addition(self): ] result = compare_config( baseline_config=self.baseline_config, - latest_config=self.latest_config, + current_config=self.latest_config, ) self.assertTrue(len(result.change_to_libraries[ChangeType.GAPIC_ADDITION]) == 1) config_change = result.change_to_libraries[ChangeType.GAPIC_ADDITION][0] self.assertEqual("", config_change.changed_param) - self.assertEqual("google/new/library/v1", config_change.latest_value) + self.assertEqual("google/new/library/v1", config_change.current_value) self.assertEqual("existing_library", config_change.library_name) diff --git a/library_generation/utils/generation_config_comparator.py b/library_generation/utils/generation_config_comparator.py index 79a5f3c1aa..09b8a9c514 100644 --- a/library_generation/utils/generation_config_comparator.py +++ b/library_generation/utils/generation_config_comparator.py @@ -25,49 +25,49 @@ def compare_config( - baseline_config: GenerationConfig, latest_config: GenerationConfig + baseline_config: GenerationConfig, current_config: GenerationConfig ) -> ConfigChange: """ Compare two GenerationConfig object and output a mapping from ConfigChange to a list of ConfigChange objects. - All libraries in the latest configuration will be affected if one of the + All libraries in the current configuration will be affected if one of the repository level parameters is changed. :param baseline_config: the baseline GenerationConfig object - :param latest_config: the latest GenerationConfig object + :param current_config: the current GenerationConfig object :return: a ConfigChange objects. """ diff = defaultdict(list[LibraryChange]) baseline_params = __convert_params_to_sorted_list(baseline_config) - latest_params = __convert_params_to_sorted_list(latest_config) - for baseline_param, latest_param in zip(baseline_params, latest_params): - if baseline_param == latest_param: + current_params = __convert_params_to_sorted_list(current_config) + for baseline_param, current_param in zip(baseline_params, current_params): + if baseline_param == current_param: continue if baseline_param[0] == "googleapis_commitish": diff[ChangeType.GOOGLEAPIS_COMMIT] = [] else: config_change = LibraryChange( - changed_param=latest_param[0], - latest_value=latest_param[1], + changed_param=current_param[0], + current_value=current_param[1], ) diff[ChangeType.REPO_LEVEL_CHANGE].append(config_change) __compare_libraries( diff=diff, baseline_library_configs=baseline_config.libraries, - latest_library_configs=latest_config.libraries, + current_library_configs=current_config.libraries, ) return ConfigChange( change_to_libraries=diff, baseline_config=baseline_config, - latest_config=latest_config, + current_config=current_config, ) def __compare_libraries( diff: Dict[ChangeType, list[LibraryChange]], baseline_library_configs: List[LibraryConfig], - latest_library_configs: List[LibraryConfig], + current_library_configs: List[LibraryConfig], ) -> None: """ Compare two lists of LibraryConfig and put the difference into a @@ -75,46 +75,46 @@ def __compare_libraries( :param diff: a mapping from ConfigChange to a list of ConfigChange objects. :param baseline_library_configs: a list of LibraryConfig object. - :param latest_library_configs: a list of LibraryConfig object. + :param current_library_configs: a list of LibraryConfig object. """ baseline_libraries = __convert_to_hashed_library_dict(baseline_library_configs) - latest_libraries = __convert_to_hashed_library_dict(latest_library_configs) + current_libraries = __convert_to_hashed_library_dict(current_library_configs) changed_libraries = [] # 1st round comparison. for library_name, hash_library in baseline_libraries.items(): # 1. find any library removed from baseline_libraries. # a library is removed from baseline_libraries if the library_name - # is not in latest_libraries. + # is not in current_libraries. # please see the reason of comment out these lines of code in the # comment of ChangeType.LIBRARIES_REMOVAL. - # if library_name not in latest_libraries: + # if library_name not in current_libraries: # config_change = ConfigChange( - # changed_param="", latest_value="", library_name=library_name + # changed_param="", current_value="", library_name=library_name # ) # diff[ChangeType.LIBRARIES_REMOVAL].append(config_change) # 2. find any library that exists in both configs but at least one # parameter is changed, which means the hash value is different. if ( - library_name in latest_libraries - and hash_library.hash_value != latest_libraries[library_name].hash_value + library_name in current_libraries + and hash_library.hash_value != current_libraries[library_name].hash_value ): changed_libraries.append(library_name) # 2nd round comparison. - for library_name in latest_libraries: - # find any library added to latest_libraries. - # a library is added to latest_libraries if the library_name + for library_name in current_libraries: + # find any library added to current_libraries. + # a library is added to current_libraries if the library_name # is not in baseline_libraries. if library_name not in baseline_libraries: config_change = LibraryChange( - changed_param="", latest_value="", library_name=library_name + changed_param="", current_value="", library_name=library_name ) diff[ChangeType.LIBRARIES_ADDITION].append(config_change) # 3rd round comparison. __compare_changed_libraries( diff=diff, baseline_libraries=baseline_libraries, - latest_libraries=latest_libraries, + current_libraries=current_libraries, changed_libraries=changed_libraries, ) @@ -139,7 +139,7 @@ def __convert_to_hashed_library_dict( def __compare_changed_libraries( diff: Dict[ChangeType, list[LibraryChange]], baseline_libraries: Dict[str, HashLibrary], - latest_libraries: Dict[str, HashLibrary], + current_libraries: Dict[str, HashLibrary], changed_libraries: List[str], ) -> None: """ @@ -149,18 +149,18 @@ def __compare_changed_libraries( :param diff: a mapping from ConfigChange to a list of ConfigChange objects. :param baseline_libraries: a mapping from library_name to HashLibrary object. - :param latest_libraries: a mapping from library_name to HashLibrary object. + :param current_libraries: a mapping from library_name to HashLibrary object. :param changed_libraries: a list of library_name of changed libraries. :raise ValueError: if api_shortname of a library is changed but library_name remains the same. """ for library_name in changed_libraries: baseline_library = baseline_libraries[library_name].library - latest_library = latest_libraries[library_name].library + current_library = current_libraries[library_name].library baseline_params = __convert_params_to_sorted_list(baseline_library) - latest_params = __convert_params_to_sorted_list(latest_library) - for baseline_param, latest_param in zip(baseline_params, latest_params): - if baseline_param == latest_param: + current_params = __convert_params_to_sorted_list(current_library) + for baseline_param, current_param in zip(baseline_params, current_params): + if baseline_param == current_param: continue if baseline_param[0] == "api_shortname": raise ValueError( @@ -168,20 +168,20 @@ def __compare_changed_libraries( ) else: config_change = LibraryChange( - changed_param=latest_param[0], - latest_value=latest_param[1], + changed_param=current_param[0], + current_value=current_param[1], library_name=library_name, ) diff[ChangeType.LIBRARY_LEVEL_CHANGE].append(config_change) # compare gapic_configs baseline_gapic_configs = baseline_library.gapic_configs - latest_gapic_configs = latest_library.gapic_configs + current_gapic_configs = current_library.gapic_configs __compare_gapic_configs( diff=diff, library_name=library_name, baseline_gapic_configs=baseline_gapic_configs, - latest_gapic_configs=latest_gapic_configs, + current_gapic_configs=current_gapic_configs, ) @@ -189,29 +189,29 @@ def __compare_gapic_configs( diff: Dict[ChangeType, list[LibraryChange]], library_name: str, baseline_gapic_configs: List[GapicConfig], - latest_gapic_configs: List[GapicConfig], + current_gapic_configs: List[GapicConfig], ) -> None: baseline_proto_paths = {config.proto_path for config in baseline_gapic_configs} - latest_proto_paths = {config.proto_path for config in latest_gapic_configs} + current_proto_paths = {config.proto_path for config in current_gapic_configs} # 1st round of comparison, find any versioned proto_path is removed # from baseline gapic configs. # please see the reason of comment out these lines of code in the # comment of ChangeType.GAPIC_REMOVAL. # for proto_path in baseline_proto_paths: - # if proto_path in latest_proto_paths: + # if proto_path in current_proto_paths: # continue # config_change = ConfigChange( - # changed_param="", latest_value=proto_path, library_name=library_name + # changed_param="", current_value=proto_path, library_name=library_name # ) # diff[ChangeType.GAPIC_REMOVAL].append(config_change) # 2nd round of comparison, find any versioned proto_path is added - # to latest gapic configs. - for proto_path in latest_proto_paths: + # to current gapic configs. + for proto_path in current_proto_paths: if proto_path in baseline_proto_paths: continue config_change = LibraryChange( - changed_param="", latest_value=proto_path, library_name=library_name + changed_param="", current_value=proto_path, library_name=library_name ) diff[ChangeType.GAPIC_ADDITION].append(config_change) From ffc16ee9c757de268a41424cacb76741d1517f12 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Apr 2024 20:42:36 -0400 Subject: [PATCH 12/34] change to optional inputs --- library_generation/cli/entry_point.py | 59 +++++++++++++++++++-------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 89d7d42602..574d8c5acb 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -30,7 +30,8 @@ def main(ctx): @main.command() @click.option( "--baseline-generation-config", - required=True, + required=False, + default=None, type=str, help=""" Path to generation_config.yaml that contains the metadata about @@ -41,7 +42,8 @@ def main(ctx): ) @click.option( "--current-generation-config", - required=True, + required=False, + default=None, type=str, help=""" Path to generation_config.yaml that contains the metadata about @@ -67,26 +69,47 @@ def generate( current_generation_config: str, repository_path: str, ): - # convert paths to absolute paths, so they can be correctly referenced in - # downstream scripts - baseline_generation_config = os.path.abspath(baseline_generation_config) + default_generation_config = f"{os.getcwd()}/generation_config.yaml" + if baseline_generation_config is None and current_generation_config is None: + if not os.path.isfile(default_generation_config): + raise FileNotFoundError( + f"]{default_generation_config} does not exist when" + f"both baseline_generation_config and current_generation_config" + f" are not specified." + ) + current_generation_config = default_generation_config + elif current_generation_config is None: + # make sure current_generation_config is not None if only one config + # is specified. + current_generation_config = baseline_generation_config + baseline_generation_config = None current_generation_config = os.path.abspath(current_generation_config) repository_path = os.path.abspath(repository_path) - config_change = compare_config( - baseline_config=from_yaml(baseline_generation_config), - current_config=from_yaml(current_generation_config), - ) - # generate libraries + if baseline_generation_config: + # Compare two generation configs and only generate changed libraries. + # Generate pull request description. + baseline_generation_config = os.path.abspath(baseline_generation_config) + config_change = compare_config( + baseline_config=from_yaml(baseline_generation_config), + current_config=from_yaml(current_generation_config), + ) + generate_from_yaml( + config=config_change.current_config, + repository_path=repository_path, + target_library_names=config_change.get_changed_libraries(), + ) + generate_pr_descriptions( + config=config_change.current_config, + baseline_commit=config_change.baseline_config.googleapis_commitish, + description_path=repository_path, + ) + return + # Execute full generation based on current_generation_config if + # baseline_generation_config is not specified. + # Do not generate pull request description. generate_from_yaml( - config=config_change.current_config, + config=from_yaml(current_generation_config), repository_path=repository_path, - target_library_names=config_change.get_changed_libraries(), - ) - # generate pull request description - generate_pr_descriptions( - config=config_change.current_config, - baseline_commit=config_change.baseline_config.googleapis_commitish, - description_path=repository_path, ) From 87c7d1c08fd232c489555c90c4227ba82e3e1cfd Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Apr 2024 21:08:15 -0400 Subject: [PATCH 13/34] raise exception if current commit is older than baseline commit --- library_generation/generate_pr_description.py | 23 +++++++++++-------- .../generate_pr_description_unit_tests.py | 0 2 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 library_generation/test/generate_pr_description_unit_tests.py diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index b976e4fe80..df88dc8c45 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -90,9 +90,9 @@ def generate_pr_descriptions( history. """ paths = config.get_proto_path_to_library_name() - description = __get_commit_messages( + description = get_commit_messages( repo_url=repo_url, - latest_commit=config.googleapis_commitish, + current_commit=config.googleapis_commitish, baseline_commit=baseline_commit, paths=paths, is_monorepo=config.is_monorepo, @@ -104,9 +104,9 @@ def generate_pr_descriptions( f.write(description) -def __get_commit_messages( +def get_commit_messages( repo_url: str, - latest_commit: str, + current_commit: str, baseline_commit: str, paths: Dict[str, str], is_monorepo: bool, @@ -118,7 +118,7 @@ def __get_commit_messages( Note that baseline_commit should be an ancestor of latest_commit. :param repo_url: the url of the repository. - :param latest_commit: the newest commit to be considered in + :param current_commit: the newest commit to be considered in selecting commit message. :param baseline_commit: the oldest commit to be considered in selecting commit message. This commit should be an ancestor of @@ -130,7 +130,12 @@ def __get_commit_messages( shutil.rmtree(tmp_dir, ignore_errors=True) os.mkdir(tmp_dir) repo = Repo.clone_from(repo_url, tmp_dir) - commit = repo.commit(latest_commit) + commit = repo.commit(current_commit) + if ( + commit.committed_datetime.utcnow() + < repo.commit(baseline_commit).committed_datetime.utcnow() + ): + raise ValueError(f"current_commit should be newer than baseline_commit.") qualified_commits = {} while str(commit.hexsha) != baseline_commit: commit_and_name = __filter_qualified_commit(paths=paths, commit=commit) @@ -142,7 +147,7 @@ def __get_commit_messages( commit = commit_parents[0] shutil.rmtree(tmp_dir, ignore_errors=True) return __combine_commit_messages( - latest_commit=latest_commit, + current_commit=current_commit, baseline_commit=baseline_commit, commits=qualified_commits, is_monorepo=is_monorepo, @@ -168,13 +173,13 @@ def __filter_qualified_commit(paths: Dict[str, str], commit: Commit) -> (Commit, def __combine_commit_messages( - latest_commit: str, + current_commit: str, baseline_commit: str, commits: Dict[Commit, str], is_monorepo: bool, ) -> str: messages = [ - f"This pull request is generated with proto changes between googleapis commit {baseline_commit} (exclusive) and {latest_commit} (inclusive).", + f"This pull request is generated with proto changes between googleapis commit {baseline_commit} (exclusive) and {current_commit} (inclusive).", "Qualified commits are:", ] for commit in commits: diff --git a/library_generation/test/generate_pr_description_unit_tests.py b/library_generation/test/generate_pr_description_unit_tests.py new file mode 100644 index 0000000000..e69de29bb2 From 9d723b0d69630a6e4d2dfe84aa38dab21eabdc11 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Apr 2024 21:08:36 -0400 Subject: [PATCH 14/34] add unit test --- .../generate_pr_description_unit_tests.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/library_generation/test/generate_pr_description_unit_tests.py b/library_generation/test/generate_pr_description_unit_tests.py index e69de29bb2..c9883c5a0d 100644 --- a/library_generation/test/generate_pr_description_unit_tests.py +++ b/library_generation/test/generate_pr_description_unit_tests.py @@ -0,0 +1,30 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from library_generation.generate_pr_description import get_commit_messages + + +class GeneratePrDescriptionTest(unittest.TestCase): + def test_get_commit_messages_current_is_older_raise_exception(self): + self.assertRaisesRegex( + ValueError, + "newer than", + get_commit_messages, + "https://github.com/googleapis/googleapis.git", + "36441693dddaf0ed73951ad3a15c215a332756aa", + "8d326d5a7e7146e41ecce7f921e51c97504f7487", + {}, + True, + ) From fd7897a8e8f61f6e1a914b5c0b979f421045a596 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Apr 2024 21:31:27 -0400 Subject: [PATCH 15/34] add error log --- library_generation/generate_pr_description.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index df88dc8c45..ae2f3c272d 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -131,11 +131,12 @@ def get_commit_messages( os.mkdir(tmp_dir) repo = Repo.clone_from(repo_url, tmp_dir) commit = repo.commit(current_commit) - if ( - commit.committed_datetime.utcnow() - < repo.commit(baseline_commit).committed_datetime.utcnow() - ): - raise ValueError(f"current_commit should be newer than baseline_commit.") + current_commit_time = commit.committed_datetime.utcnow() + baseline_commit_time = repo.commit(baseline_commit).committed_datetime.utcnow() + if current_commit_time < baseline_commit_time: + raise ValueError( + f"current_commit ({current_commit[:7]}, committed on {current_commit_time}) should be newer than baseline_commit ({baseline_commit[:7]}, committed on {baseline_commit_time})." + ) qualified_commits = {} while str(commit.hexsha) != baseline_commit: commit_and_name = __filter_qualified_commit(paths=paths, commit=commit) From ca902ec5510432dfd9630b1a4a3c346cfc72d71f Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Apr 2024 21:45:57 -0400 Subject: [PATCH 16/34] convert time to int --- library_generation/generate_pr_description.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index ae2f3c272d..c65142e249 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import calendar import os import shutil import click as click @@ -131,11 +132,15 @@ def get_commit_messages( os.mkdir(tmp_dir) repo = Repo.clone_from(repo_url, tmp_dir) commit = repo.commit(current_commit) - current_commit_time = commit.committed_datetime.utcnow() - baseline_commit_time = repo.commit(baseline_commit).committed_datetime.utcnow() + current_commit_time = calendar.timegm(commit.committed_datetime.utctimetuple()) + baseline_commit_time = calendar.timegm( + repo.commit(baseline_commit).committed_datetime.utctimetuple() + ) if current_commit_time < baseline_commit_time: raise ValueError( - f"current_commit ({current_commit[:7]}, committed on {current_commit_time}) should be newer than baseline_commit ({baseline_commit[:7]}, committed on {baseline_commit_time})." + f"current_commit ({current_commit[:7]}, committed on " + f"{current_commit_time}) should be newer than baseline_commit " + f"({baseline_commit[:7]}, committed on {baseline_commit_time})." ) qualified_commits = {} while str(commit.hexsha) != baseline_commit: From 4895971997594f2c4ac814b693b505be330adbb4 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Apr 2024 21:48:30 -0400 Subject: [PATCH 17/34] add comment --- library_generation/generate_pr_description.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index c65142e249..0d92c6fc2e 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -132,6 +132,8 @@ def get_commit_messages( os.mkdir(tmp_dir) repo = Repo.clone_from(repo_url, tmp_dir) commit = repo.commit(current_commit) + # Convert datetime to UTC timestamp. For more info: + # https://stackoverflow.com/questions/5067218/get-utc-timestamp-in-python-with-datetime current_commit_time = calendar.timegm(commit.committed_datetime.utctimetuple()) baseline_commit_time = calendar.timegm( repo.commit(baseline_commit).committed_datetime.utctimetuple() From 87c0bdd7af953c60149c837f055c4c6bfaa08e26 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 4 Apr 2024 09:07:19 -0400 Subject: [PATCH 18/34] extract helper method --- library_generation/generate_pr_description.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index 0d92c6fc2e..e6999085ac 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -132,12 +132,8 @@ def get_commit_messages( os.mkdir(tmp_dir) repo = Repo.clone_from(repo_url, tmp_dir) commit = repo.commit(current_commit) - # Convert datetime to UTC timestamp. For more info: - # https://stackoverflow.com/questions/5067218/get-utc-timestamp-in-python-with-datetime - current_commit_time = calendar.timegm(commit.committed_datetime.utctimetuple()) - baseline_commit_time = calendar.timegm( - repo.commit(baseline_commit).committed_datetime.utctimetuple() - ) + current_commit_time = __get_commit_timestamp(commit) + baseline_commit_time = __get_commit_timestamp(repo.commit(baseline_commit)) if current_commit_time < baseline_commit_time: raise ValueError( f"current_commit ({current_commit[:7]}, committed on " @@ -205,5 +201,16 @@ def __combine_commit_messages( return "\n".join(messages) +def __get_commit_timestamp(commit: Commit) -> int: + """ + # Convert datetime to UTC timestamp. For more info: + # https://stackoverflow.com/questions/5067218/get-utc-timestamp-in-python-with-datetime + + :param commit: a Commit object + :return: the timestamp of the commit + """ + return calendar.timegm(commit.committed_datetime.utctimetuple()) + + if __name__ == "__main__": main() From 69fd09b189f636a60bd473e21e06b14b6567f319 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 4 Apr 2024 09:11:17 -0400 Subject: [PATCH 19/34] refactor tests --- .../test/generate_pr_description_unit_tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/library_generation/test/generate_pr_description_unit_tests.py b/library_generation/test/generate_pr_description_unit_tests.py index c9883c5a0d..b3bb958648 100644 --- a/library_generation/test/generate_pr_description_unit_tests.py +++ b/library_generation/test/generate_pr_description_unit_tests.py @@ -18,13 +18,17 @@ class GeneratePrDescriptionTest(unittest.TestCase): def test_get_commit_messages_current_is_older_raise_exception(self): + # committed on April 1st, 2024 + current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + # committed on April 2nd, 2024 + baseline_commit = "d5020fff4cbe108bdf506074791c56cff7840bef" self.assertRaisesRegex( ValueError, "newer than", get_commit_messages, "https://github.com/googleapis/googleapis.git", - "36441693dddaf0ed73951ad3a15c215a332756aa", - "8d326d5a7e7146e41ecce7f921e51c97504f7487", + current_commit, + baseline_commit, {}, True, ) From d2b39631d2385bfa9232b094df589f16ab8a744a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 4 Apr 2024 09:25:52 -0400 Subject: [PATCH 20/34] add integration test --- library_generation/test/integration_tests.py | 31 ++++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 3c10e1fdf1..dccbc2d406 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -47,9 +47,20 @@ class IntegrationTest(unittest.TestCase): - def test_entry_point_running_in_container(self): - self.__build_image(docker_file=build_file, cwd=repo_root_dir) + @classmethod + def setUpClass(cls) -> None: + IntegrationTest.__build_image(docker_file=build_file, cwd=repo_root_dir) + + def test_entry_point_without_config_raise_file_exception(self): + self.assertRaisesRegex( + FileNotFoundError, + "does not exist", + self.__run_entry_point_in_docker_container, + ".", + "", + ) + def test_entry_point_running_in_container(self): shutil.rmtree(f"{golden_dir}", ignore_errors=True) os.makedirs(f"{golden_dir}", exist_ok=True) config_files = self.__get_config_files(config_dir) @@ -238,9 +249,17 @@ def __bind_device_to_volumes(cls, volume_name: str, device_dir: str): @classmethod def __run_entry_point_in_docker_container( - cls, repo: str, repo_volumes: str, baseline_config: str, current_config: str + cls, + repo: str, + repo_volumes: str, + baseline_config: str = None, + current_config: str = None, ): - subprocess.check_call( + if not baseline_config: + baseline_config = f"/workspace/config-{repo}/{baseline_config}" + if not current_config: + current_config = f"/workspace/config-{repo}/{current_config}" + return subprocess.check_output( [ "docker", "run", @@ -263,8 +282,8 @@ def __run_entry_point_in_docker_container( "python", "/src/cli/entry_point.py", "generate", - f"--baseline-generation-config=/workspace/config-{repo}/{baseline_config}", - f"--current-generation-config=/workspace/config-{repo}/{current_config}", + f"--baseline-generation-config={baseline_config}", + f"--current-generation-config={current_config}", f"--repository-path=/workspace/{repo}", ] ) From b4b907bdbedd1442c9442e3b963a5d7c129737ff Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 4 Apr 2024 09:36:25 -0400 Subject: [PATCH 21/34] change if --- library_generation/test/integration_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index dccbc2d406..c3ed0ff4c6 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -255,9 +255,9 @@ def __run_entry_point_in_docker_container( baseline_config: str = None, current_config: str = None, ): - if not baseline_config: + if baseline_config: baseline_config = f"/workspace/config-{repo}/{baseline_config}" - if not current_config: + if current_config: current_config = f"/workspace/config-{repo}/{current_config}" return subprocess.check_output( [ From ce25efa799ccb3ac547fec3b572bb1a367bdc440 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 4 Apr 2024 09:46:21 -0400 Subject: [PATCH 22/34] refactor --- library_generation/test/integration_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index c3ed0ff4c6..f8687aaf64 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -136,7 +136,7 @@ def test_entry_point_running_in_container(self): self.assertTrue(len(generated_only) == 0) self.assertTrue(len(diff_files) == 0) - print(" No differences found in {library_name}") + print(f" No differences found in {library_name}") # compare .repo-metadata.json self.assertTrue( self.__compare_json_files( From 8d1d3f698b028b97650cb73020bffdd329c0cf7f Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 4 Apr 2024 15:04:57 +0000 Subject: [PATCH 23/34] restore integration test --- library_generation/cli/entry_point.py | 2 +- library_generation/test/cli/__init__.py | 0 .../test/cli/entry_point_unit_tests.py | 18 +++++++++++++++ library_generation/test/integration_tests.py | 23 ++++--------------- 4 files changed, 24 insertions(+), 19 deletions(-) create mode 100644 library_generation/test/cli/__init__.py create mode 100644 library_generation/test/cli/entry_point_unit_tests.py diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 574d8c5acb..5054c9c995 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -73,7 +73,7 @@ def generate( if baseline_generation_config is None and current_generation_config is None: if not os.path.isfile(default_generation_config): raise FileNotFoundError( - f"]{default_generation_config} does not exist when" + f"{default_generation_config} does not exist when " f"both baseline_generation_config and current_generation_config" f" are not specified." ) diff --git a/library_generation/test/cli/__init__.py b/library_generation/test/cli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py new file mode 100644 index 0000000000..bc42699f14 --- /dev/null +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + + +class EntryPointTest(unittest.TestCase): + pass diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index f8687aaf64..2c356d24a1 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -51,15 +51,6 @@ class IntegrationTest(unittest.TestCase): def setUpClass(cls) -> None: IntegrationTest.__build_image(docker_file=build_file, cwd=repo_root_dir) - def test_entry_point_without_config_raise_file_exception(self): - self.assertRaisesRegex( - FileNotFoundError, - "does not exist", - self.__run_entry_point_in_docker_container, - ".", - "", - ) - def test_entry_point_running_in_container(self): shutil.rmtree(f"{golden_dir}", ignore_errors=True) os.makedirs(f"{golden_dir}", exist_ok=True) @@ -252,14 +243,10 @@ def __run_entry_point_in_docker_container( cls, repo: str, repo_volumes: str, - baseline_config: str = None, - current_config: str = None, + baseline_config: str, + current_config: str, ): - if baseline_config: - baseline_config = f"/workspace/config-{repo}/{baseline_config}" - if current_config: - current_config = f"/workspace/config-{repo}/{current_config}" - return subprocess.check_output( + subprocess.check_call( [ "docker", "run", @@ -282,8 +269,8 @@ def __run_entry_point_in_docker_container( "python", "/src/cli/entry_point.py", "generate", - f"--baseline-generation-config={baseline_config}", - f"--current-generation-config={current_config}", + f"--baseline-generation-config=/workspace/config-{repo}/{baseline_config}", + f"--current-generation-config=/workspace/config-{repo}/{current_config}", f"--repository-path=/workspace/{repo}", ] ) From e155ea0939448f41428026dd6a433f2bf3242856 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 4 Apr 2024 15:22:44 +0000 Subject: [PATCH 24/34] add unit tests --- library_generation/test/cli/entry_point_unit_tests.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py index bc42699f14..56d3aa4e45 100644 --- a/library_generation/test/cli/entry_point_unit_tests.py +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -12,7 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. import unittest +from click.testing import CliRunner +from library_generation.cli.entry_point import generate class EntryPointTest(unittest.TestCase): - pass + def test_entry_point_without_config_raise_file_exception(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke(generate, ["--repository-path=."]) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) From 8de0c3d0aa3b05ae46c80de8f13e008d5613dd02 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 4 Apr 2024 11:35:53 -0400 Subject: [PATCH 25/34] add unit tests --- .../test/cli/entry_point_unit_tests.py | 11 ++++ .../entry_point/generation_config.yaml | 58 +++++++++++++++++++ .../test/resources/entry_point/versions.txt | 0 3 files changed, 69 insertions(+) create mode 100644 library_generation/test/resources/entry_point/generation_config.yaml create mode 100644 library_generation/test/resources/entry_point/versions.txt diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py index 56d3aa4e45..73c41e9107 100644 --- a/library_generation/test/cli/entry_point_unit_tests.py +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -11,10 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import unittest from click.testing import CliRunner from library_generation.cli.entry_point import generate +script_dir = os.path.dirname(os.path.realpath(__file__)) +resource_dir = os.path.join(script_dir, "..", "resources", "entry_point") + class EntryPointTest(unittest.TestCase): def test_entry_point_without_config_raise_file_exception(self): @@ -23,3 +27,10 @@ def test_entry_point_without_config_raise_file_exception(self): result = runner.invoke(generate, ["--repository-path=."]) self.assertEqual(1, result.exit_code) self.assertEqual(FileNotFoundError, result.exc_info[0]) + + def test_entry_point_with_default_config_success(self): + os.chdir(resource_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke(generate, ["--repository-path=."]) + self.assertEqual(0, result.exit_code) diff --git a/library_generation/test/resources/entry_point/generation_config.yaml b/library_generation/test/resources/entry_point/generation_config.yaml new file mode 100644 index 0000000000..839f80996e --- /dev/null +++ b/library_generation/test/resources/entry_point/generation_config.yaml @@ -0,0 +1,58 @@ +gapic_generator_version: 2.37.0 +protobuf_version: 25.2 +googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc +owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 +synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e +template_excludes: + - ".github/*" + - ".kokoro/*" + - "samples/*" + - "CODE_OF_CONDUCT.md" + - "CONTRIBUTING.md" + - "LICENSE" + - "SECURITY.md" + - "java.header" + - "license-checks.xml" + - "renovate.json" + - ".gitignore" +libraries: + - api_shortname: apigeeconnect + name_pretty: Apigee Connect + product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" + api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." + release_level: "stable" + library_name: "apigee-connect" + GAPICs: + - proto_path: google/cloud/apigeeconnect/v1 + + - api_shortname: alloydb + name_pretty: AlloyDB + product_documentation: https://cloud.google.com/alloydb/ + api_description: AlloyDB is a fully managed, PostgreSQL-compatible database service + with industry-leading performance, availability, and scale. + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/v1 + - proto_path: google/cloud/alloydb/v1alpha + - proto_path: google/cloud/alloydb/v1beta + + - api_shortname: alloydb + name_pretty: AlloyDB connectors + product_documentation: https://cloud.google.com/alloydb/docs + api_description: AlloyDB is a fully-managed, PostgreSQL-compatible database for + demanding transactional workloads. It provides enterprise-grade performance and + availability while maintaining 100% compatibility with open-source PostgreSQL. + library_name: alloydb-connectors + rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest + GAPICs: + - proto_path: google/cloud/alloydb/connectors/v1 + - proto_path: google/cloud/alloydb/connectors/v1alpha + - proto_path: google/cloud/alloydb/connectors/v1beta + + - api_shortname: cloudcontrolspartner + name_pretty: Cloud Controls Partner API + product_documentation: https://cloud.google.com/sovereign-controls-by-partners/docs/sovereign-partners + api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. + GAPICs: + - proto_path: google/cloud/cloudcontrolspartner/v1 + - proto_path: google/cloud/cloudcontrolspartner/v1beta \ No newline at end of file diff --git a/library_generation/test/resources/entry_point/versions.txt b/library_generation/test/resources/entry_point/versions.txt new file mode 100644 index 0000000000..e69de29bb2 From e518d3fe49a520910e2abbebe97b58f3de2dea18 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 4 Apr 2024 11:41:28 -0400 Subject: [PATCH 26/34] test setup --- library_generation/test/cli/entry_point_unit_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py index 73c41e9107..3d3ed4cf86 100644 --- a/library_generation/test/cli/entry_point_unit_tests.py +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -21,6 +21,9 @@ class EntryPointTest(unittest.TestCase): + def setUp(self) -> None: + os.chdir(script_dir) + def test_entry_point_without_config_raise_file_exception(self): runner = CliRunner() # noinspection PyTypeChecker From 22eb1279435fc160b547154166bb32458e035637 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 4 Apr 2024 11:49:17 -0400 Subject: [PATCH 27/34] restore unit tests --- .../test/cli/entry_point_unit_tests.py | 14 ----- .../entry_point/generation_config.yaml | 58 ------------------- .../test/resources/entry_point/versions.txt | 0 3 files changed, 72 deletions(-) delete mode 100644 library_generation/test/resources/entry_point/generation_config.yaml delete mode 100644 library_generation/test/resources/entry_point/versions.txt diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py index 3d3ed4cf86..56d3aa4e45 100644 --- a/library_generation/test/cli/entry_point_unit_tests.py +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -11,29 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest from click.testing import CliRunner from library_generation.cli.entry_point import generate -script_dir = os.path.dirname(os.path.realpath(__file__)) -resource_dir = os.path.join(script_dir, "..", "resources", "entry_point") - class EntryPointTest(unittest.TestCase): - def setUp(self) -> None: - os.chdir(script_dir) - def test_entry_point_without_config_raise_file_exception(self): runner = CliRunner() # noinspection PyTypeChecker result = runner.invoke(generate, ["--repository-path=."]) self.assertEqual(1, result.exit_code) self.assertEqual(FileNotFoundError, result.exc_info[0]) - - def test_entry_point_with_default_config_success(self): - os.chdir(resource_dir) - runner = CliRunner() - # noinspection PyTypeChecker - result = runner.invoke(generate, ["--repository-path=."]) - self.assertEqual(0, result.exit_code) diff --git a/library_generation/test/resources/entry_point/generation_config.yaml b/library_generation/test/resources/entry_point/generation_config.yaml deleted file mode 100644 index 839f80996e..0000000000 --- a/library_generation/test/resources/entry_point/generation_config.yaml +++ /dev/null @@ -1,58 +0,0 @@ -gapic_generator_version: 2.37.0 -protobuf_version: 25.2 -googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc -owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 -synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e -template_excludes: - - ".github/*" - - ".kokoro/*" - - "samples/*" - - "CODE_OF_CONDUCT.md" - - "CONTRIBUTING.md" - - "LICENSE" - - "SECURITY.md" - - "java.header" - - "license-checks.xml" - - "renovate.json" - - ".gitignore" -libraries: - - api_shortname: apigeeconnect - name_pretty: Apigee Connect - product_documentation: "https://cloud.google.com/apigee/docs/hybrid/v1.3/apigee-connect/" - api_description: "allows the Apigee hybrid management plane to connect securely to the MART service in the runtime plane without requiring you to expose the MART endpoint on the internet." - release_level: "stable" - library_name: "apigee-connect" - GAPICs: - - proto_path: google/cloud/apigeeconnect/v1 - - - api_shortname: alloydb - name_pretty: AlloyDB - product_documentation: https://cloud.google.com/alloydb/ - api_description: AlloyDB is a fully managed, PostgreSQL-compatible database service - with industry-leading performance, availability, and scale. - rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest - GAPICs: - - proto_path: google/cloud/alloydb/v1 - - proto_path: google/cloud/alloydb/v1alpha - - proto_path: google/cloud/alloydb/v1beta - - - api_shortname: alloydb - name_pretty: AlloyDB connectors - product_documentation: https://cloud.google.com/alloydb/docs - api_description: AlloyDB is a fully-managed, PostgreSQL-compatible database for - demanding transactional workloads. It provides enterprise-grade performance and - availability while maintaining 100% compatibility with open-source PostgreSQL. - library_name: alloydb-connectors - rest_documentation: https://cloud.google.com/alloydb/docs/reference/rest - GAPICs: - - proto_path: google/cloud/alloydb/connectors/v1 - - proto_path: google/cloud/alloydb/connectors/v1alpha - - proto_path: google/cloud/alloydb/connectors/v1beta - - - api_shortname: cloudcontrolspartner - name_pretty: Cloud Controls Partner API - product_documentation: https://cloud.google.com/sovereign-controls-by-partners/docs/sovereign-partners - api_description: Provides insights about your customers and their Assured Workloads based on your Sovereign Controls by Partners offering. - GAPICs: - - proto_path: google/cloud/cloudcontrolspartner/v1 - - proto_path: google/cloud/cloudcontrolspartner/v1beta \ No newline at end of file diff --git a/library_generation/test/resources/entry_point/versions.txt b/library_generation/test/resources/entry_point/versions.txt deleted file mode 100644 index e69de29bb2..0000000000 From c25e2bdd0a5653b842a84ebe18642d0da4d189b1 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 4 Apr 2024 16:40:32 -0400 Subject: [PATCH 28/34] refactor --- library_generation/cli/entry_point.py | 39 ++++++++++++++------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 5054c9c995..35f3fd4298 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -85,31 +85,32 @@ def generate( baseline_generation_config = None current_generation_config = os.path.abspath(current_generation_config) repository_path = os.path.abspath(repository_path) - if baseline_generation_config: - # Compare two generation configs and only generate changed libraries. - # Generate pull request description. - baseline_generation_config = os.path.abspath(baseline_generation_config) - config_change = compare_config( - baseline_config=from_yaml(baseline_generation_config), - current_config=from_yaml(current_generation_config), - ) + if not baseline_generation_config: + # Execute full generation based on current_generation_config if + # baseline_generation_config is not specified. + # Do not generate pull request description. generate_from_yaml( - config=config_change.current_config, + config=from_yaml(current_generation_config), repository_path=repository_path, - target_library_names=config_change.get_changed_libraries(), - ) - generate_pr_descriptions( - config=config_change.current_config, - baseline_commit=config_change.baseline_config.googleapis_commitish, - description_path=repository_path, ) return - # Execute full generation based on current_generation_config if - # baseline_generation_config is not specified. - # Do not generate pull request description. + + # Compare two generation configs and only generate changed libraries. + # Generate pull request description. + baseline_generation_config = os.path.abspath(baseline_generation_config) + config_change = compare_config( + baseline_config=from_yaml(baseline_generation_config), + current_config=from_yaml(current_generation_config), + ) generate_from_yaml( - config=from_yaml(current_generation_config), + config=config_change.current_config, repository_path=repository_path, + target_library_names=config_change.get_changed_libraries(), + ) + generate_pr_descriptions( + config=config_change.current_config, + baseline_commit=config_change.baseline_config.googleapis_commitish, + description_path=repository_path, ) From cc137b00a718a5ef2cb9ed9904c68d42cfbbe2df Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 5 Apr 2024 14:46:08 -0400 Subject: [PATCH 29/34] change error message --- library_generation/cli/entry_point.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 35f3fd4298..c336dbb2bb 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -73,9 +73,10 @@ def generate( if baseline_generation_config is None and current_generation_config is None: if not os.path.isfile(default_generation_config): raise FileNotFoundError( - f"{default_generation_config} does not exist when " - f"both baseline_generation_config and current_generation_config" - f" are not specified." + f"{default_generation_config} does not exist. " + "A valid generation config has to be passed in as " + "current_generation_config or exist in the current working " + "directory." ) current_generation_config = default_generation_config elif current_generation_config is None: From 5dbcd2c910beee34582a1a21b8e2d793e76d9267 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 5 Apr 2024 15:05:12 -0400 Subject: [PATCH 30/34] fail if current config is null but baseline is not --- library_generation/cli/entry_point.py | 12 ++++++++---- .../test/cli/entry_point_unit_tests.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index c336dbb2bb..421740ab1d 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -70,6 +70,7 @@ def generate( repository_path: str, ): default_generation_config = f"{os.getcwd()}/generation_config.yaml" + if baseline_generation_config is None and current_generation_config is None: if not os.path.isfile(default_generation_config): raise FileNotFoundError( @@ -80,10 +81,13 @@ def generate( ) current_generation_config = default_generation_config elif current_generation_config is None: - # make sure current_generation_config is not None if only one config - # is specified. - current_generation_config = baseline_generation_config - baseline_generation_config = None + raise FileNotFoundError( + "current_generation_config is not specified when " + "baseline_generation_config is specified. " + "current_generation_config should be the source of truth of " + "library generation." + ) + current_generation_config = os.path.abspath(current_generation_config) repository_path = os.path.abspath(repository_path) if not baseline_generation_config: diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py index 56d3aa4e45..ec65753609 100644 --- a/library_generation/test/cli/entry_point_unit_tests.py +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -23,3 +23,21 @@ def test_entry_point_without_config_raise_file_exception(self): result = runner.invoke(generate, ["--repository-path=."]) self.assertEqual(1, result.exit_code) self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex( + result.exception.args[0], "generation_config.yaml does not exist." + ) + + def test_entry_point_with_baseline_without_current_raise_file_exception(self): + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + generate, + ["--baseline-generation-config=path/to/config.yaml", "--repository-path=."], + ) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex( + result.exception.args[0], + "current_generation_config is not specified when " + "baseline_generation_config is specified.", + ) From 3d3522b52b1e4a539570ad9a7409bcffd9c01838 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 5 Apr 2024 15:11:06 -0400 Subject: [PATCH 31/34] two commits should be different --- library_generation/generate_pr_description.py | 2 +- .../test/generate_pr_description_unit_tests.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/library_generation/generate_pr_description.py b/library_generation/generate_pr_description.py index e6999085ac..03a8d2cfef 100644 --- a/library_generation/generate_pr_description.py +++ b/library_generation/generate_pr_description.py @@ -134,7 +134,7 @@ def get_commit_messages( commit = repo.commit(current_commit) current_commit_time = __get_commit_timestamp(commit) baseline_commit_time = __get_commit_timestamp(repo.commit(baseline_commit)) - if current_commit_time < baseline_commit_time: + if current_commit_time <= baseline_commit_time: raise ValueError( f"current_commit ({current_commit[:7]}, committed on " f"{current_commit_time}) should be newer than baseline_commit " diff --git a/library_generation/test/generate_pr_description_unit_tests.py b/library_generation/test/generate_pr_description_unit_tests.py index b3bb958648..35dd26f952 100644 --- a/library_generation/test/generate_pr_description_unit_tests.py +++ b/library_generation/test/generate_pr_description_unit_tests.py @@ -32,3 +32,18 @@ def test_get_commit_messages_current_is_older_raise_exception(self): {}, True, ) + + def test_get_commit_messages_current_and_baseline_are_same_raise_exception(self): + # committed on April 1st, 2024 + current_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + baseline_commit = "36441693dddaf0ed73951ad3a15c215a332756aa" + self.assertRaisesRegex( + ValueError, + "newer than", + get_commit_messages, + "https://github.com/googleapis/googleapis.git", + current_commit, + baseline_commit, + {}, + True, + ) From 280cee5737ab1f39fde8eb52bf3393af829e0df3 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 5 Apr 2024 15:42:46 -0400 Subject: [PATCH 32/34] change doc --- library_generation/cli/entry_point.py | 29 +++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 421740ab1d..6f1d19370c 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -34,8 +34,8 @@ def main(ctx): default=None, type=str, help=""" - Path to generation_config.yaml that contains the metadata about - library generation. + Absolute or relative path to generation_config.yaml that contains the + metadata about library generation. The googleapis commit in the configuration is the oldest commit, exclusively, from which the commit message is considered. """, @@ -46,8 +46,8 @@ def main(ctx): default=None, type=str, help=""" - Path to generation_config.yaml that contains the metadata about - library generation. + Absolute or relative path to generation_config.yaml that contains the + metadata about library generation. The googleapis commit in the configuration is the newest commit, inclusively, to which the commit message is considered. """, @@ -69,6 +69,27 @@ def generate( current_generation_config: str, repository_path: str, ): + """ + Compare baseline generation config and current generation config and + generate changed libraries based on current generation config with pull + request description. + + If baseline generation config is not specified but current generation + config is specified, generate all libraries based on current generation + config without pull request description. + + If current generation config is not specified but baseline generation + config is specified, raise FileNotFoundError because current generation + config should be the source of truth of library generation. + + If both baseline generation config and current generation config are not + specified, generate all libraries based on the default generation config, + which is generation_config.yaml in the current working directory. Raise + FileNotFoundError if the default config does not exist. + + The pull request description, if generated, will be available in + repository_path/pr_description.txt. + """ default_generation_config = f"{os.getcwd()}/generation_config.yaml" if baseline_generation_config is None and current_generation_config is None: From 881b557d718f60eb391f32891c23c543fdd46a1c Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 5 Apr 2024 17:16:12 -0400 Subject: [PATCH 33/34] change comment --- library_generation/cli/entry_point.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 6f1d19370c..e751a33893 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -34,10 +34,9 @@ def main(ctx): default=None, type=str, help=""" - Absolute or relative path to generation_config.yaml that contains the - metadata about library generation. - The googleapis commit in the configuration is the oldest commit, - exclusively, from which the commit message is considered. + Absolute or relative path to a generation_config.yaml. + This config file is used for commit history generation, not library + generation. """, ) @click.option( @@ -46,10 +45,8 @@ def main(ctx): default=None, type=str, help=""" - Absolute or relative path to generation_config.yaml that contains the + Absolute or relative path to a generation_config.yaml that contains the metadata about library generation. - The googleapis commit in the configuration is the newest commit, - inclusively, to which the commit message is considered. """, ) @click.option( @@ -71,12 +68,12 @@ def generate( ): """ Compare baseline generation config and current generation config and - generate changed libraries based on current generation config with pull - request description. + generate changed libraries based on current generation config with commit + history. If baseline generation config is not specified but current generation config is specified, generate all libraries based on current generation - config without pull request description. + config without commit history. If current generation config is not specified but baseline generation config is specified, raise FileNotFoundError because current generation @@ -87,7 +84,7 @@ def generate( which is generation_config.yaml in the current working directory. Raise FileNotFoundError if the default config does not exist. - The pull request description, if generated, will be available in + The commit history, if generated, will be available in repository_path/pr_description.txt. """ default_generation_config = f"{os.getcwd()}/generation_config.yaml" From 4de89debec24a39649e214ed56053bf8df919289 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 5 Apr 2024 17:44:25 -0400 Subject: [PATCH 34/34] change readme --- library_generation/README.md | 53 ++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/library_generation/README.md b/library_generation/README.md index 75fc32695d..364ec6b511 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -1,8 +1,8 @@ # Generate a repository containing GAPIC Client Libraries -The script, `generate_repo.py`, allows you to generate a repository containing -GAPIC client libraries (a monorepo, for example, google-cloud-java) from a -configuration file. +The script, `entry_point.py`, allows you to generate a repository containing +GAPIC client libraries with googleapis commit history (a monorepo, for example, +google-cloud-java) from a configuration file. ## Environment @@ -17,30 +17,27 @@ In order to generate a version for each library, a versions.txt has to exist in `repository_path`. Please refer to [Repository path](#repository-path--repositorypath---optional) for more information. -## Parameters to generate a repository using `generate_repo.py` +## Parameters to generate a repository using `entry_point.py` -### Generation configuration yaml (`generation_config_yaml`) +### Baseline generation configuration yaml (`baseline_generation_config`) -A path to a configuration file containing parameters to generate the repository. -Please refer [Configuration to generate a repository](#configuration-to-generate-a-repository) -for more information. - -### Target library API shortname (`target_library_api_shortname`), optional +An absolute or relative path to a generation_config.yaml. +This config file is used for commit history generation, not library +generation. -If specified, the libray whose `api_shortname` equals to `target_library_api_shortname` -will be generated; otherwise all libraries in the configuration file will be -generated. -This can be useful when you just want to generate one library for debugging -purposes. +### Current generation configuration yaml (`current_generation_config`) -The default value is an empty string, which means all libraries will be generated. +An absolute or relative path to a configuration file containing parameters to +generate the repository. +Please refer [Configuration to generate a repository](#configuration-to-generate-a-repository) +for more information. ### Repository path (`repository_path`), optional The path to where the generated repository goes. The default value is the current working directory when running the script. -For example, `cd google-cloud-java && python generate_repo.py ...` without +For example, `cd google-cloud-java && python entry_point.py ...` without specifying the `--repository_path` option will modify the `google-cloud-java` repository the user `cd`'d into. @@ -49,7 +46,9 @@ right version for each library. Please refer [here](go/java-client-releasing#versionstxt-manifest) for more info of versions.txt. -## Output of `generate_repo.py` +## Output of `entry_point.py` + +### GAPIC libraries For each module (e.g. `google-cloud-java/java-asset`), the following files/folders will be created/modified: @@ -73,6 +72,11 @@ will be created/modified: | pom.xml (repo root dir) | Always generated from inputs | | versions.txt | New entries will be added if they don’t exist | +### googleapis commit history + +If both `baseline_generation_config` and `current_generation_config` are +specified, and they contain different googleapis commit, the commit history will +be generated into `pr_description.txt` in the `repository_path`. ## Configuration to generate a repository @@ -184,20 +188,23 @@ libraries: - proto_path: google/cloud/asset/v1p7beta1 ``` -## An example to generate a repository using `generate_repo.py` +## An example to generate a repository using `entry_point.py` ```bash # install python module (allows the `library_generation` module to be imported from anywhere) python -m pip install -r library_generation/requirements.in +# install library_generation module +python -m pip install library_generation # generate the repository -python -m library_generation/generate_repo.py generate \ ---generation-config-yaml=/path/to/config-file \ +python -m library_generation/entry_point.py generate \ +--baseline-generation-config=/path/to/baseline_config_file \ +--current-generation-config=/path/to/current_config_file \ --repository-path=/path/to/repository ``` -## An example of generated repository using `generate_repo.py` +## An example of generated repository using `entry_point.py` -If you run `generate_repo.py` with the example [configuration](#an-example-of-generation-configuration) +If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration) shown above, the repository structure is: ``` $repository_path