From 30eb2a2a3a7b0b1707af23350fcce3393ec6f449 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Wed, 13 Mar 2024 14:19:59 -0400 Subject: [PATCH 1/2] chore: remove unused `library_generation/configuration` folder (#2563) --- library_generation/configuration/owlbot-cli-sha | 2 -- library_generation/configuration/python-version | 1 - library_generation/configuration/synthtool-commitish | 1 - 3 files changed, 4 deletions(-) delete mode 100644 library_generation/configuration/owlbot-cli-sha delete mode 100644 library_generation/configuration/python-version delete mode 100644 library_generation/configuration/synthtool-commitish diff --git a/library_generation/configuration/owlbot-cli-sha b/library_generation/configuration/owlbot-cli-sha deleted file mode 100644 index aad884ebfc..0000000000 --- a/library_generation/configuration/owlbot-cli-sha +++ /dev/null @@ -1,2 +0,0 @@ -# owlbot-cli image from 20231010 -sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 diff --git a/library_generation/configuration/python-version b/library_generation/configuration/python-version deleted file mode 100644 index 1e33456831..0000000000 --- a/library_generation/configuration/python-version +++ /dev/null @@ -1 +0,0 @@ -3.11.2 diff --git a/library_generation/configuration/synthtool-commitish b/library_generation/configuration/synthtool-commitish deleted file mode 100644 index 5603b9055b..0000000000 --- a/library_generation/configuration/synthtool-commitish +++ /dev/null @@ -1 +0,0 @@ -59fe44fde9866a26e7ee4e4450fd79f67f8cf599 From 510a8873d3d9b6ebb92b7bd7310cf482eece593c Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Wed, 13 Mar 2024 18:33:29 +0000 Subject: [PATCH 2/2] chore: refactor utility functions (#2557) In this PR: - Code Refactor - Refactor pom generation functions to a separate file. - Refactor monorepo post processing functions to a separate file. - Refactor unit tests to separate files. - Expose CLI tool used in google-cloud-java - Generate root pom.xml - Generate gapic-libraries-bom/pom.xml To use these tools in google-cloud-java (through docker container): ``` python library_generation/cli/generate_monorepo_root_pom.py generate --repository-path=. python library_generation/cli/generate_monorepo_gapic_bom.py generate --repository-path=. --versions-file=./versions.txt ``` --- .../cli/generate_monorepo_gapic_bom.py | 50 ++++++ .../cli/generate_monorepo_root_pom.py | 39 +++++ library_generation/generate_repo.py | 3 +- library_generation/test/test_utils.py | 40 +++++ ...{unit_tests.py => utilities_unit_tests.py} | 72 ++------ .../monorepo_postprocessor_unit_tests.py | 45 +++++ .../test/utils/pom_generator_unit_tests.py | 32 ++++ library_generation/utilities.py | 162 +----------------- library_generation/utils/file_render.py | 24 +++ .../utils/monorepo_postprocessor.py | 29 ++++ library_generation/utils/pom_generator.py | 157 +++++++++++++++++ 11 files changed, 431 insertions(+), 222 deletions(-) create mode 100644 library_generation/cli/generate_monorepo_gapic_bom.py create mode 100644 library_generation/cli/generate_monorepo_root_pom.py create mode 100644 library_generation/test/test_utils.py rename library_generation/test/{unit_tests.py => utilities_unit_tests.py} (90%) create mode 100644 library_generation/test/utils/monorepo_postprocessor_unit_tests.py create mode 100644 library_generation/test/utils/pom_generator_unit_tests.py create mode 100644 library_generation/utils/file_render.py create mode 100644 library_generation/utils/monorepo_postprocessor.py create mode 100644 library_generation/utils/pom_generator.py diff --git a/library_generation/cli/generate_monorepo_gapic_bom.py b/library_generation/cli/generate_monorepo_gapic_bom.py new file mode 100644 index 0000000000..999c9e5ebf --- /dev/null +++ b/library_generation/cli/generate_monorepo_gapic_bom.py @@ -0,0 +1,50 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import click as click + +from library_generation.utils.pom_generator import generate_gapic_bom +from library_generation.utils.pom_generator import generate_root_pom + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--repository-path", + required=True, + type=str, + help=""" + Path to which the generated pom.xml goes. + """, +) +@click.option( + "--versions-file", + required=True, + type=str, + help=""" + The file containing version of libraries. + Throw FileNotFoundError if the file doesn't exist. + """, +) +def generate(repository_path: str, versions_file: str) -> None: + generate_gapic_bom(repository_path=repository_path, versions_file=versions_file) + + +if __name__ == "__main__": + main() diff --git a/library_generation/cli/generate_monorepo_root_pom.py b/library_generation/cli/generate_monorepo_root_pom.py new file mode 100644 index 0000000000..8f129b63f0 --- /dev/null +++ b/library_generation/cli/generate_monorepo_root_pom.py @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import click as click +from library_generation.utils.pom_generator import generate_root_pom + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--repository-path", + required=True, + type=str, + help=""" + Path to which the generated pom.xml goes. + """, +) +def generate(repository_path: str) -> None: + generate_root_pom(repository_path=repository_path) + + +if __name__ == "__main__": + main() diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index c4b2c32ea8..abe51c50f2 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -18,6 +18,7 @@ import os from library_generation.generate_composed_library import generate_composed_library from library_generation.model.generation_config import from_yaml +from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing @click.group(invoke_without_command=False) @@ -112,7 +113,7 @@ def generate_from_yaml( if not config.is_monorepo: return - util.monorepo_postprocessing( + monorepo_postprocessing( repository_path=repository_path, versions_file=repo_config.versions_file ) diff --git a/library_generation/test/test_utils.py b/library_generation/test/test_utils.py new file mode 100644 index 0000000000..890cd95362 --- /dev/null +++ b/library_generation/test/test_utils.py @@ -0,0 +1,40 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from difflib import unified_diff +from pathlib import Path + +from typing import List + + +class FileComparator(unittest.TestCase): + def compare_files(self, expect: str, actual: str): + with open(expect, "r") as f: + expected_lines = f.readlines() + with open(actual, "r") as f: + actual_lines = f.readlines() + + diff = list(unified_diff(expected_lines, actual_lines)) + self.assertEqual( + first=[], second=diff, msg="Unexpected file contents:\n" + "".join(diff) + ) + + +def cleanup(files: List[str]): + for file in files: + path = Path(file).resolve() + if path.is_file(): + path.unlink() + elif path.is_dir(): + path.rmdir() diff --git a/library_generation/test/unit_tests.py b/library_generation/test/utilities_unit_tests.py similarity index 90% rename from library_generation/test/unit_tests.py rename to library_generation/test/utilities_unit_tests.py index 909414e20e..aa6d99ac24 100644 --- a/library_generation/test/unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -21,9 +21,6 @@ import io import contextlib from pathlib import Path -from difflib import unified_diff - -from typing import List from parameterized import parameterized from library_generation import utilities as util from library_generation.model.gapic_config import GapicConfig @@ -31,6 +28,8 @@ from library_generation.model.gapic_inputs import parse as parse_build_file from library_generation.model.generation_config import from_yaml from library_generation.model.library_config import LibraryConfig +from library_generation.test.test_utils import FileComparator +from library_generation.test.test_utils import cleanup from library_generation.utilities import find_versioned_proto_path from library_generation.utilities import get_file_paths @@ -38,6 +37,7 @@ resources_dir = os.path.join(script_dir, "resources") build_file = Path(os.path.join(resources_dir, "misc")).resolve() test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() +file_comparator = FileComparator() library_1 = LibraryConfig( api_shortname="baremetalsolution", name_pretty="Bare Metal Solution", @@ -391,18 +391,6 @@ def test_remove_version_from_returns_self(self): "google/cloud/aiplatform", util.remove_version_from(proto_path) ) - def test_get_version_from_returns_current(self): - versions_file = f"{resources_dir}/misc/versions.txt" - artifact = "gax-grpc" - self.assertEqual( - "2.33.1-SNAPSHOT", util.get_version_from(versions_file, artifact) - ) - - def test_get_version_from_returns_released(self): - versions_file = f"{resources_dir}/misc/versions.txt" - artifact = "gax-grpc" - self.assertEqual("2.34.0", util.get_version_from(versions_file, artifact, True)) - def test_get_library_returns_library_name(self): self.assertEqual("bare-metal-solution", util.get_library_name(library_1)) @@ -414,32 +402,32 @@ def test_generate_prerequisite_files_non_monorepo_success(self): num_libraries=1, library_type="GAPIC_COMBO" ) - self.__compare_files( + file_comparator.compare_files( f"{library_path}/.repo-metadata.json", f"{library_path}/.repo-metadata-non-monorepo-golden.json", ) # since this is a single library, we treat this as HW repository, # meaning that the owlbot yaml will be inside a .github folder - self.__compare_files( + file_comparator.compare_files( f"{library_path}/.github/.OwlBot.yaml", f"{library_path}/.OwlBot-golden.yaml", ) - self.__compare_files( + file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" ) def test_generate_prerequisite_files_monorepo_success(self): library_path = self.__setup_prerequisite_files(num_libraries=2) - self.__compare_files( + file_comparator.compare_files( f"{library_path}/.repo-metadata.json", f"{library_path}/.repo-metadata-monorepo-golden.json", ) - self.__compare_files( + file_comparator.compare_files( f"{library_path}/.OwlBot.yaml", f"{library_path}/.OwlBot-golden.yaml", ) - self.__compare_files( + file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" ) @@ -488,37 +476,6 @@ def test_prepare_repo_split_repo_success(self): library_path = sorted([Path(key).name for key in repo_config.libraries]) self.assertEqual(["misc"], library_path) - def test_monorepo_postprocessing_valid_repository_success(self): - repository_path = f"{resources_dir}/test_monorepo_postprocessing" - versions_file = f"{repository_path}/versions.txt" - files = [ - f"{repository_path}/pom.xml", - f"{repository_path}/gapic-libraries-bom/pom.xml", - ] - self.__cleanup(files) - util.monorepo_postprocessing( - repository_path=repository_path, versions_file=versions_file - ) - self.__compare_files( - expect=f"{repository_path}/pom-golden.xml", - actual=f"{repository_path}/pom.xml", - ) - self.__compare_files( - expect=f"{repository_path}/gapic-libraries-bom/pom-golden.xml", - actual=f"{repository_path}/gapic-libraries-bom/pom.xml", - ) - - def __compare_files(self, expect: str, actual: str): - with open(expect, "r") as f: - expected_lines = f.readlines() - with open(actual, "r") as f: - actual_lines = f.readlines() - - diff = list(unified_diff(expected_lines, actual_lines)) - self.assertEqual( - first=[], second=diff, msg="Unexpected file contents:\n" + "".join(diff) - ) - def __setup_prerequisite_files( self, num_libraries: int, library_type: str = "GAPIC_AUTO" ) -> str: @@ -528,7 +485,7 @@ def __setup_prerequisite_files( f"{library_path}/.OwlBot.yaml", f"{library_path}/owlbot.py", ] - self.__cleanup(files) + cleanup(files) config = self.__get_a_gen_config(num_libraries, library_type=library_type) proto_path = "google/cloud/baremetalsolution/v2" transport = "grpc" @@ -593,15 +550,6 @@ def __get_a_gen_config( libraries=libraries, ) - @staticmethod - def __cleanup(files: List[str]): - for file in files: - path = Path(file).resolve() - if path.is_file(): - path.unlink() - elif path.is_dir(): - path.rmdir() - if __name__ == "__main__": unittest.main() diff --git a/library_generation/test/utils/monorepo_postprocessor_unit_tests.py b/library_generation/test/utils/monorepo_postprocessor_unit_tests.py new file mode 100644 index 0000000000..c7df83acaf --- /dev/null +++ b/library_generation/test/utils/monorepo_postprocessor_unit_tests.py @@ -0,0 +1,45 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest + +from library_generation.test.test_utils import FileComparator +from library_generation.test.test_utils import cleanup +from library_generation.utils.monorepo_postprocessor import monorepo_postprocessing + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") +file_comparator = FileComparator() + + +class MonorepoPostprocessorTest(unittest.TestCase): + def test_monorepo_postprocessing_valid_repository_success(self): + repository_path = f"{resources_dir}/test_monorepo_postprocessing" + versions_file = f"{repository_path}/versions.txt" + files = [ + f"{repository_path}/pom.xml", + f"{repository_path}/gapic-libraries-bom/pom.xml", + ] + cleanup(files) + monorepo_postprocessing( + repository_path=repository_path, versions_file=versions_file + ) + file_comparator.compare_files( + expect=f"{repository_path}/pom-golden.xml", + actual=f"{repository_path}/pom.xml", + ) + file_comparator.compare_files( + expect=f"{repository_path}/gapic-libraries-bom/pom-golden.xml", + actual=f"{repository_path}/gapic-libraries-bom/pom.xml", + ) diff --git a/library_generation/test/utils/pom_generator_unit_tests.py b/library_generation/test/utils/pom_generator_unit_tests.py new file mode 100644 index 0000000000..7a829e58aa --- /dev/null +++ b/library_generation/test/utils/pom_generator_unit_tests.py @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest + +from library_generation.utils.pom_generator import get_version_from + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") + + +class PomGeneratorTest(unittest.TestCase): + def test_get_version_from_returns_current(self): + versions_file = f"{resources_dir}/misc/versions.txt" + artifact = "gax-grpc" + self.assertEqual("2.33.1-SNAPSHOT", get_version_from(versions_file, artifact)) + + def test_get_version_from_returns_released(self): + versions_file = f"{resources_dir}/misc/versions.txt" + artifact = "gax-grpc" + self.assertEqual("2.34.0", get_version_from(versions_file, artifact, True)) diff --git a/library_generation/utilities.py b/library_generation/utilities.py index 2faee66f7a..29e175c1b8 100755 --- a/library_generation/utilities.py +++ b/library_generation/utilities.py @@ -19,121 +19,13 @@ import re from pathlib import Path from typing import Dict - -from lxml import etree -from library_generation.model.bom_config import BomConfig from library_generation.model.generation_config import GenerationConfig from library_generation.model.library_config import LibraryConfig from typing import List -from jinja2 import Environment, FileSystemLoader - from library_generation.model.repo_config import RepoConfig +from library_generation.utils.file_render import render script_dir = os.path.dirname(os.path.realpath(__file__)) -jinja_env = Environment(loader=FileSystemLoader(f"{script_dir}/templates")) -project_tag = "{http://maven.apache.org/POM/4.0.0}" -group_id_tag = "groupId" -artifact_tag = "artifactId" -version_tag = "version" - - -def __render(template_name: str, output_name: str, **kwargs): - template = jinja_env.get_template(template_name) - t = template.stream(kwargs) - directory = os.path.dirname(output_name) - if not os.path.isdir(directory): - os.makedirs(directory) - t.dump(str(output_name)) - - -def __search_for_java_modules( - repository_path: str, -) -> List[str]: - repo = Path(repository_path).resolve() - modules = [] - for sub_dir in repo.iterdir(): - if sub_dir.is_dir() and sub_dir.name.startswith("java-"): - modules.append(sub_dir.name) - return sorted(modules) - - -def __search_for_bom_artifact( - repository_path: str, -) -> List[BomConfig]: - repo = Path(repository_path).resolve() - module_exclusions = ["gapic-libraries-bom"] - group_id_inclusions = [ - "com.google.cloud", - "com.google.analytics", - "com.google.area120", - ] - bom_configs = [] - for module in repo.iterdir(): - if module.is_file() or module.name in module_exclusions: - continue - for sub_module in module.iterdir(): - if sub_module.is_dir() and sub_module.name.endswith("-bom"): - root = etree.parse(f"{sub_module}/pom.xml").getroot() - group_id = root.find(f"{project_tag}{group_id_tag}").text - if group_id not in group_id_inclusions: - continue - artifact_id = root.find(f"{project_tag}{artifact_tag}").text - version = root.find(f"{project_tag}{version_tag}").text - index = artifact_id.rfind("-") - version_annotation = artifact_id[:index] - bom_configs.append( - BomConfig( - group_id=group_id, - artifact_id=artifact_id, - version=version, - version_annotation=version_annotation, - ) - ) - # handle edge case: java-grafeas - bom_configs += __handle_special_bom( - repository_path=repository_path, - module="java-grafeas", - group_id="io.grafeas", - artifact_id="grafeas", - ) - # handle edge case: java-dns - bom_configs += __handle_special_bom( - repository_path=repository_path, - module="java-dns", - group_id="com.google.cloud", - artifact_id="google-cloud-dns", - ) - # handle edge case: java-notification - bom_configs += __handle_special_bom( - repository_path=repository_path, - module="java-notification", - group_id="com.google.cloud", - artifact_id="google-cloud-notification", - ) - - return sorted(bom_configs) - - -def __handle_special_bom( - repository_path: str, - module: str, - group_id: str, - artifact_id: str, -) -> List[BomConfig]: - pom = f"{repository_path}/{module}/pom.xml" - if not Path(pom).exists(): - return [] - root = etree.parse(pom).getroot() - version = root.find(f"{project_tag}{version_tag}").text - return [ - BomConfig( - group_id=group_id, - artifact_id=artifact_id, - version=version, - version_annotation=artifact_id, - is_import=False, - ) - ] def create_argument(arg_key: str, arg_container: object) -> List[str]: @@ -419,7 +311,7 @@ def generate_prerequisite_files( else f"{library_path}/.github/{owlbot_yaml_file}" ) if not os.path.exists(path_to_owlbot_yaml_file): - __render( + render( template_name="owlbot.yaml.monorepo.j2", output_name=path_to_owlbot_yaml_file, artifact_name=distribution_name_short, @@ -431,7 +323,7 @@ def generate_prerequisite_files( # generate owlbot.py py_file = "owlbot.py" if not os.path.exists(f"{library_path}/{py_file}"): - __render( + render( template_name="owlbot.py.j2", output_name=f"{library_path}/{py_file}", should_include_templates=True, @@ -439,54 +331,6 @@ def generate_prerequisite_files( ) -def monorepo_postprocessing( - repository_path: str, - versions_file: str, -) -> None: - """ - Perform repository level post-processing - :param repository_path: the path of the repository - :param versions_file: the versions_txt contains version of modules - :return: None - """ - print("Regenerating root pom.xml") - modules = __search_for_java_modules(repository_path) - __render( - template_name="root-pom.xml.j2", - output_name=f"{repository_path}/pom.xml", - modules=modules, - ) - print("Regenerating gapic-libraries-bom") - bom_configs = __search_for_bom_artifact(repository_path) - monorepo_version = get_version_from( - versions_file=versions_file, - artifact_id="google-cloud-java", - ) - __render( - template_name="gapic-libraries-bom.xml.j2", - output_name=f"{repository_path}/gapic-libraries-bom/pom.xml", - monorepo_version=monorepo_version, - bom_configs=bom_configs, - ) - - -def get_version_from( - versions_file: str, artifact_id: str, is_released: bool = False -) -> str: - """ - Get version of a given artifact from versions.txt - :param versions_file: the path of version.txt - :param artifact_id: the artifact id - :param is_released: whether returns the released or current version - :return: the version of the artifact - """ - index = 1 if is_released else 2 - with open(versions_file, "r") as f: - for line in f.readlines(): - if artifact_id in line: - return line.split(":")[index].strip() - - def get_file_paths(config: GenerationConfig) -> Dict[str, str]: """ Get versioned proto_path to library_name mapping from configuration file. diff --git a/library_generation/utils/file_render.py b/library_generation/utils/file_render.py new file mode 100644 index 0000000000..5c68445753 --- /dev/null +++ b/library_generation/utils/file_render.py @@ -0,0 +1,24 @@ +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from jinja2 import Environment, FileSystemLoader + +script_dir = os.path.dirname(os.path.realpath(__file__)) +jinja_env = Environment(loader=FileSystemLoader(f"{script_dir}/../templates")) + + +def render(template_name: str, output_name: str, **kwargs): + template = jinja_env.get_template(template_name) + t = template.stream(kwargs) + directory = os.path.dirname(output_name) + if not os.path.isdir(directory): + os.makedirs(directory) + t.dump(str(output_name)) diff --git a/library_generation/utils/monorepo_postprocessor.py b/library_generation/utils/monorepo_postprocessor.py new file mode 100644 index 0000000000..cd522b83b5 --- /dev/null +++ b/library_generation/utils/monorepo_postprocessor.py @@ -0,0 +1,29 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from library_generation.utils.pom_generator import generate_gapic_bom +from library_generation.utils.pom_generator import generate_root_pom + + +def monorepo_postprocessing( + repository_path: str, + versions_file: str, +) -> None: + """ + Perform repository level post-processing + :param repository_path: the path of the repository + :param versions_file: the versions_txt contains version of modules + :return: None + """ + generate_root_pom(repository_path=repository_path) + generate_gapic_bom(repository_path=repository_path, versions_file=versions_file) diff --git a/library_generation/utils/pom_generator.py b/library_generation/utils/pom_generator.py new file mode 100644 index 0000000000..8312d4c5a4 --- /dev/null +++ b/library_generation/utils/pom_generator.py @@ -0,0 +1,157 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pathlib import Path + +from lxml import etree +from typing import List + +from library_generation.model.bom_config import BomConfig +from library_generation.utils.file_render import render + +project_tag = "{http://maven.apache.org/POM/4.0.0}" +group_id_tag = "groupId" +artifact_tag = "artifactId" +version_tag = "version" + + +def generate_root_pom(repository_path: str) -> None: + print("Regenerating root pom.xml") + modules = __search_for_java_modules(repository_path) + render( + template_name="root-pom.xml.j2", + output_name=f"{repository_path}/pom.xml", + modules=modules, + ) + + +def generate_gapic_bom(repository_path: str, versions_file: str) -> None: + print("Regenerating gapic-libraries-bom") + bom_configs = __search_for_bom_artifact(repository_path) + monorepo_version = get_version_from( + versions_file=versions_file, + artifact_id="google-cloud-java", + ) + render( + template_name="gapic-libraries-bom.xml.j2", + output_name=f"{repository_path}/gapic-libraries-bom/pom.xml", + monorepo_version=monorepo_version, + bom_configs=bom_configs, + ) + + +def get_version_from( + versions_file: str, artifact_id: str, is_released: bool = False +) -> str: + """ + Get version of a given artifact from versions.txt + :param versions_file: the path of version.txt + :param artifact_id: the artifact id + :param is_released: whether returns the released or current version + :return: the version of the artifact + """ + index = 1 if is_released else 2 + with open(versions_file, "r") as f: + for line in f.readlines(): + if artifact_id in line: + return line.split(":")[index].strip() + + +def __search_for_java_modules( + repository_path: str, +) -> List[str]: + repo = Path(repository_path).resolve() + modules = [] + for sub_dir in repo.iterdir(): + if sub_dir.is_dir() and sub_dir.name.startswith("java-"): + modules.append(sub_dir.name) + return sorted(modules) + + +def __search_for_bom_artifact( + repository_path: str, +) -> List[BomConfig]: + repo = Path(repository_path).resolve() + module_exclusions = ["gapic-libraries-bom"] + group_id_inclusions = [ + "com.google.cloud", + "com.google.analytics", + "com.google.area120", + ] + bom_configs = [] + for module in repo.iterdir(): + if module.is_file() or module.name in module_exclusions: + continue + for sub_module in module.iterdir(): + if sub_module.is_dir() and sub_module.name.endswith("-bom"): + root = etree.parse(f"{sub_module}/pom.xml").getroot() + group_id = root.find(f"{project_tag}{group_id_tag}").text + if group_id not in group_id_inclusions: + continue + artifact_id = root.find(f"{project_tag}{artifact_tag}").text + version = root.find(f"{project_tag}{version_tag}").text + index = artifact_id.rfind("-") + version_annotation = artifact_id[:index] + bom_configs.append( + BomConfig( + group_id=group_id, + artifact_id=artifact_id, + version=version, + version_annotation=version_annotation, + ) + ) + # handle edge case: java-grafeas + bom_configs += __handle_special_bom( + repository_path=repository_path, + module="java-grafeas", + group_id="io.grafeas", + artifact_id="grafeas", + ) + # handle edge case: java-dns + bom_configs += __handle_special_bom( + repository_path=repository_path, + module="java-dns", + group_id="com.google.cloud", + artifact_id="google-cloud-dns", + ) + # handle edge case: java-notification + bom_configs += __handle_special_bom( + repository_path=repository_path, + module="java-notification", + group_id="com.google.cloud", + artifact_id="google-cloud-notification", + ) + + return sorted(bom_configs) + + +def __handle_special_bom( + repository_path: str, + module: str, + group_id: str, + artifact_id: str, +) -> List[BomConfig]: + pom = f"{repository_path}/{module}/pom.xml" + if not Path(pom).exists(): + return [] + root = etree.parse(pom).getroot() + version = root.find(f"{project_tag}{version_tag}").text + return [ + BomConfig( + group_id=group_id, + artifact_id=artifact_id, + version=version, + version_annotation=artifact_id, + is_import=False, + ) + ]