From c139e14cce0bac2e2ba8b24e889199976e1c6ec6 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 10 May 2024 15:52:53 -0400 Subject: [PATCH 01/75] initial moving over into pygen core --- packages/autorest.python/autorest/__init__.py | 98 +----------- packages/autorest.python/autorest/black.py | 12 ++ packages/autorest.python/autorest/codegen.py | 117 ++++++++++++++ packages/autorest.python/autorest/m2r.py | 15 ++ .../autorest.python/autorest/postprocess.py | 13 ++ .../autorest.python/autorest/preprocess.py | 20 +++ packages/pygen/README.md | 1 + packages/pygen/core/__init__.py | 108 +++++++++++++ packages/pygen/core/_utils.py | 149 ++++++++++++++++++ packages/pygen/core/_version.py | 7 + .../autorest => pygen/core}/black/__init__.py | 7 +- .../core}/codegen/__init__.py | 84 +--------- .../autorest => pygen/core}/codegen/_utils.py | 0 .../core}/codegen/models/__init__.py | 0 .../core}/codegen/models/base.py | 0 .../core}/codegen/models/base_builder.py | 0 .../core}/codegen/models/client.py | 0 .../core}/codegen/models/code_model.py | 0 .../core}/codegen/models/combined_type.py | 0 .../core}/codegen/models/constant_type.py | 0 .../core}/codegen/models/credential_types.py | 0 .../core}/codegen/models/dictionary_type.py | 0 .../core}/codegen/models/enum_type.py | 0 .../core}/codegen/models/imports.py | 0 .../core}/codegen/models/list_type.py | 0 .../core}/codegen/models/lro_operation.py | 0 .../codegen/models/lro_paging_operation.py | 0 .../core}/codegen/models/model_type.py | 0 .../core}/codegen/models/operation.py | 0 .../core}/codegen/models/operation_group.py | 0 .../core}/codegen/models/paging_operation.py | 0 .../core}/codegen/models/parameter.py | 0 .../core}/codegen/models/parameter_list.py | 0 .../core}/codegen/models/primitive_types.py | 0 .../core}/codegen/models/property.py | 0 .../core}/codegen/models/request_builder.py | 0 .../models/request_builder_parameter.py | 0 .../core}/codegen/models/response.py | 0 .../core}/codegen/models/utils.py | 0 .../core}/codegen/serializers/__init__.py | 24 +-- .../codegen/serializers/base_serializer.py | 0 .../codegen/serializers/builder_serializer.py | 0 .../codegen/serializers/client_serializer.py | 0 .../codegen/serializers/enum_serializer.py | 0 .../codegen/serializers/general_serializer.py | 0 .../codegen/serializers/import_serializer.py | 0 .../serializers/metadata_serializer.py | 0 .../serializers/model_init_serializer.py | 0 .../codegen/serializers/model_serializer.py | 0 .../operation_groups_serializer.py | 0 .../serializers/operations_init_serializer.py | 0 .../serializers/parameter_serializer.py | 0 .../codegen/serializers/patch_serializer.py | 0 .../request_builders_serializer.py | 0 .../codegen/serializers/sample_serializer.py | 0 .../codegen/serializers/test_serializer.py | 0 .../codegen/serializers/types_serializer.py | 0 .../core}/codegen/serializers/utils.py | 0 .../core}/codegen/templates/client.py.jinja2 | 0 .../templates/client_container.py.jinja2 | 0 .../core}/codegen/templates/config.py.jinja2 | 0 .../templates/config_container.py.jinja2 | 0 .../codegen/templates/conftest.py.jinja2 | 0 .../core}/codegen/templates/enum.py.jinja2 | 0 .../templates/enum_container.py.jinja2 | 0 .../core}/codegen/templates/init.py.jinja2 | 0 .../core}/codegen/templates/keywords.jinja2 | 0 .../codegen/templates/lro_operation.py.jinja2 | 0 .../templates/lro_paging_operation.py.jinja2 | 0 .../core}/codegen/templates/macros.jinja2 | 0 .../codegen/templates/metadata.json.jinja2 | 0 .../codegen/templates/model_base.py.jinja2 | 0 .../templates/model_container.py.jinja2 | 0 .../codegen/templates/model_dpg.py.jinja2 | 0 .../codegen/templates/model_init.py.jinja2 | 0 .../codegen/templates/model_msrest.py.jinja2 | 0 .../codegen/templates/operation.py.jinja2 | 0 .../templates/operation_group.py.jinja2 | 0 .../operation_groups_container.py.jinja2 | 0 .../codegen/templates/operation_tools.jinja2 | 0 .../operations_folder_init.py.jinja2 | 0 .../packaging_templates/CHANGELOG.md.jinja2 | 0 .../packaging_templates/LICENSE.jinja2 | 0 .../packaging_templates/MANIFEST.in.jinja2 | 0 .../packaging_templates/README.md.jinja2 | 0 .../dev_requirements.txt.jinja2 | 0 .../packaging_templates/setup.py.jinja2 | 0 .../templates/paging_operation.py.jinja2 | 0 .../core}/codegen/templates/patch.py.jinja2 | 0 .../codegen/templates/pkgutil_init.py.jinja2 | 0 .../templates/request_builder.py.jinja2 | 0 .../templates/request_builders.py.jinja2 | 0 .../codegen/templates/rest_init.py.jinja2 | 0 .../core}/codegen/templates/sample.py.jinja2 | 0 .../codegen/templates/serialization.py.jinja2 | 0 .../core}/codegen/templates/test.py.jinja2 | 0 .../codegen/templates/testpreparer.py.jinja2 | 0 .../core}/codegen/templates/types.py.jinja2 | 0 .../codegen/templates/validation.py.jinja2 | 0 .../core}/codegen/templates/vendor.py.jinja2 | 0 .../core}/codegen/templates/version.py.jinja2 | 0 .../autorest => pygen/core}/m2r/__init__.py | 13 +- .../core}/postprocess/__init__.py | 7 +- .../core}/postprocess/get_all.py | 0 .../core}/postprocess/venvtools.py | 0 .../core}/preprocess/__init__.py | 12 +- .../core}/preprocess/helpers.py | 0 .../core}/preprocess/python_mappings.py | 2 +- packages/pygen/setup.py | 56 +++++++ 109 files changed, 513 insertions(+), 232 deletions(-) create mode 100644 packages/autorest.python/autorest/black.py create mode 100644 packages/autorest.python/autorest/codegen.py create mode 100644 packages/autorest.python/autorest/m2r.py create mode 100644 packages/autorest.python/autorest/postprocess.py create mode 100644 packages/autorest.python/autorest/preprocess.py create mode 100644 packages/pygen/README.md create mode 100644 packages/pygen/core/__init__.py create mode 100644 packages/pygen/core/_utils.py create mode 100644 packages/pygen/core/_version.py rename packages/{autorest.python/autorest => pygen/core}/black/__init__.py (91%) rename packages/{autorest.python/autorest => pygen/core}/codegen/__init__.py (72%) rename packages/{autorest.python/autorest => pygen/core}/codegen/_utils.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/__init__.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/base.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/base_builder.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/client.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/code_model.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/combined_type.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/constant_type.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/credential_types.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/dictionary_type.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/enum_type.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/imports.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/list_type.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/lro_operation.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/lro_paging_operation.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/model_type.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/operation.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/operation_group.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/paging_operation.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/parameter.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/parameter_list.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/primitive_types.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/property.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/request_builder.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/request_builder_parameter.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/response.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/models/utils.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/__init__.py (97%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/base_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/builder_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/client_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/enum_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/general_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/import_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/metadata_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/model_init_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/model_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/operation_groups_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/operations_init_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/parameter_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/patch_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/request_builders_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/sample_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/test_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/types_serializer.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/serializers/utils.py (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/client.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/client_container.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/config.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/config_container.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/conftest.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/enum.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/enum_container.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/init.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/keywords.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/lro_operation.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/lro_paging_operation.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/macros.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/metadata.json.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/model_base.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/model_container.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/model_dpg.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/model_init.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/model_msrest.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/operation.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/operation_group.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/operation_groups_container.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/operation_tools.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/operations_folder_init.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/packaging_templates/LICENSE.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/packaging_templates/MANIFEST.in.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/packaging_templates/README.md.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/packaging_templates/setup.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/paging_operation.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/patch.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/pkgutil_init.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/request_builder.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/request_builders.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/rest_init.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/sample.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/serialization.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/test.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/testpreparer.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/types.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/validation.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/vendor.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/codegen/templates/version.py.jinja2 (100%) rename packages/{autorest.python/autorest => pygen/core}/m2r/__init__.py (86%) rename packages/{autorest.python/autorest => pygen/core}/postprocess/__init__.py (96%) rename packages/{autorest.python/autorest => pygen/core}/postprocess/get_all.py (100%) rename packages/{autorest.python/autorest => pygen/core}/postprocess/venvtools.py (100%) rename packages/{autorest.python/autorest => pygen/core}/preprocess/__init__.py (97%) rename packages/{autorest.python/autorest => pygen/core}/preprocess/helpers.py (100%) rename packages/{autorest.python/autorest => pygen/core}/preprocess/python_mappings.py (98%) create mode 100644 packages/pygen/setup.py diff --git a/packages/autorest.python/autorest/__init__.py b/packages/autorest.python/autorest/__init__.py index 1fa6e0b83c8..bb2e1883f73 100644 --- a/packages/autorest.python/autorest/__init__.py +++ b/packages/autorest.python/autorest/__init__.py @@ -5,59 +5,16 @@ # -------------------------------------------------------------------------- import logging from pathlib import Path -import json -from abc import ABC, abstractmethod +from abc import abstractmethod from typing import Any, Dict, Union, List import yaml from .jsonrpc import AutorestAPI -from ._version import VERSION +from pygen.core import ReaderAndWriter, Plugin, YamlUpdatePlugin - -__version__ = VERSION _LOGGER = logging.getLogger(__name__) - -class ReaderAndWriter: - def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None: - self.output_folder = Path(output_folder) - self._list_file: List[str] = [] - try: - with open( - Path(self.output_folder) / Path("..") / Path("python.json"), - "r", - encoding="utf-8-sig", - ) as fd: - python_json = json.load(fd) - except Exception: # pylint: disable=broad-except - python_json = {} - self.options = kwargs - if python_json: - _LOGGER.warning("Loading python.json file. This behavior will be depreacted") - self.options.update(python_json) - - def read_file(self, path: Union[str, Path]) -> str: - """Directly reading from disk""" - # make path relative to output folder - try: - with open(self.output_folder / Path(path), "r", encoding="utf-8-sig") as fd: - return fd.read() - except FileNotFoundError: - return "" - - def write_file(self, filename: Union[str, Path], file_content: str) -> None: - """Directly writing to disk""" - file_folder = Path(filename).parent - if not Path.is_dir(self.output_folder / file_folder): - Path.mkdir(self.output_folder / file_folder, parents=True) - with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd: - fd.write(file_content) - - def list_file(self) -> List[str]: - return [str(f) for f in self.output_folder.glob("**/*") if f.is_file()] - - class ReaderAndWriterAutorest(ReaderAndWriter): def __init__(self, *, output_folder: Union[str, Path], autorestapi: AutorestAPI) -> None: super().__init__(output_folder=output_folder) @@ -73,23 +30,6 @@ def list_file(self) -> List[str]: return self._autorestapi.list_inputs() -class Plugin(ReaderAndWriter, ABC): - """A base class for autorest plugin. - - :param autorestapi: An autorest API instance - """ - - @abstractmethod - def process(self) -> bool: - """The plugin process. - - :rtype: bool - :returns: True if everything's ok, False optherwise - :raises Exception: Could raise any exception, stacktrace will be sent to autorest API - """ - raise NotImplementedError() - - class PluginAutorest(Plugin, ReaderAndWriterAutorest): """For our Autorest plugins, we want to take autorest api as input as options, then pass it to the Plugin""" @@ -101,40 +41,6 @@ def __init__(self, autorestapi: AutorestAPI, *, output_folder: Union[str, Path]) def get_options(self) -> Dict[str, Any]: """Get the options bag using the AutorestAPI that we send to the parent plugin""" - -class YamlUpdatePlugin(Plugin): - """A plugin that update the YAML as input.""" - - def get_yaml(self) -> Dict[str, Any]: - # cadl file doesn't have to be relative to output folder - with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd: - return yaml.safe_load(fd.read()) - - def write_yaml(self, yaml_string: str) -> None: - with open(self.options["cadl_file"], "w", encoding="utf-8-sig") as fd: - fd.write(yaml_string) - - def process(self) -> bool: - # List the input file, should be only one - yaml_data = self.get_yaml() - - self.update_yaml(yaml_data) - - yaml_string = yaml.safe_dump(yaml_data) - - self.write_yaml(yaml_string) - return True - - @abstractmethod - def update_yaml(self, yaml_data: Dict[str, Any]) -> None: - """The code-model-v4-no-tags yaml model tree. - - :rtype: updated yaml - :raises Exception: Could raise any exception, stacktrace will be sent to autorest API - """ - raise NotImplementedError() - - class YamlUpdatePluginAutorest(YamlUpdatePlugin, PluginAutorest): # pylint: disable=abstract-method def get_yaml(self) -> Dict[str, Any]: return yaml.safe_load(self.read_file("code-model-v4-no-tags.yaml")) diff --git a/packages/autorest.python/autorest/black.py b/packages/autorest.python/autorest/black.py new file mode 100644 index 00000000000..8dc22c034fb --- /dev/null +++ b/packages/autorest.python/autorest/black.py @@ -0,0 +1,12 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Dict, Any +from pygen.core.black import BlackScriptPlugin +from . import PluginAutorest + +class BlackScriptPluginAutorest(BlackScriptPlugin, PluginAutorest): + def get_options(self) -> Dict[str, Any]: + return {"output_folder": self._autorestapi.get_value("outputFolderUri")} diff --git a/packages/autorest.python/autorest/codegen.py b/packages/autorest.python/autorest/codegen.py new file mode 100644 index 00000000000..116d480d820 --- /dev/null +++ b/packages/autorest.python/autorest/codegen.py @@ -0,0 +1,117 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from typing import Dict, Any, Union +from pathlib import Path +import yaml +from . import ReaderAndWriterAutorest +from .jsonrpc import AutorestAPI +from pygen.core.codegen.models import CodeModel +from pygen.core.codegen.serializers import JinjaSerializer + + +from . import PluginAutorest +from pygen.core.codegen import CodeGenerator + + +_LOGGER = logging.getLogger(__name__) + +class JinjaSerializerAutorest(JinjaSerializer, ReaderAndWriterAutorest): + def __init__( + self, + autorestapi: AutorestAPI, + code_model: CodeModel, + *, + output_folder: Union[str, Path], + **kwargs: Any, + ) -> None: + super().__init__( + autorestapi=autorestapi, + code_model=code_model, + output_folder=output_folder, + **kwargs, + ) + + + +class CodeGeneratorAutorest(CodeGenerator, PluginAutorest): + def get_options(self) -> Dict[str, Any]: + if self._autorestapi.get_boolean_value("python3-only") is False: + _LOGGER.warning("You have passed in --python3-only=False. We have force overriden this to True.") + if self._autorestapi.get_boolean_value("add-python3-operation-files"): + _LOGGER.warning( + "You have passed in --add-python3-operation-files. " + "This flag no longer has an effect bc all SDKs are now Python3 only." + ) + if self._autorestapi.get_boolean_value("reformat-next-link"): + _LOGGER.warning( + "You have passed in --reformat-next-link. We have force overriden " + "this to False because we no longer reformat initial query parameters into next " + "calls unless explicitly defined in the service definition." + ) + options = { + "azure-arm": self._autorestapi.get_boolean_value("azure-arm"), + "header-text": self._autorestapi.get_value("header-text"), + "low-level-client": self._autorestapi.get_boolean_value("low-level-client", False), + "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant", True), + "show-operations": self._autorestapi.get_boolean_value("show-operations"), + "python3-only": self._autorestapi.get_boolean_value("python3-only"), + "head-as-boolean": self._autorestapi.get_boolean_value("head-as-boolean", False), + "keep-version-file": self._autorestapi.get_boolean_value("keep-version-file"), + "no-async": self._autorestapi.get_boolean_value("no-async"), + "no-namespace-folders": self._autorestapi.get_boolean_value("no-namespace-folders"), + "basic-setup-py": self._autorestapi.get_boolean_value("basic-setup-py"), + "package-name": self._autorestapi.get_value("package-name"), + "package-version": self._autorestapi.get_value("package-version"), + "client-side-validation": self._autorestapi.get_boolean_value("client-side-validation"), + "tracing": self._autorestapi.get_boolean_value("trace"), + "multiapi": self._autorestapi.get_boolean_value("multiapi", False), + "polymorphic-examples": self._autorestapi.get_value("polymorphic-examples"), + "models-mode": self._autorestapi.get_value("models-mode"), + "builders-visibility": self._autorestapi.get_value("builders-visibility"), + "show-send-request": self._autorestapi.get_boolean_value("show-send-request"), + "only-path-and-body-params-positional": self._autorestapi.get_boolean_value( + "only-path-and-body-params-positional" + ), + "combine-operation-files": self._autorestapi.get_boolean_value("combine-operation-files"), + "package-mode": self._autorestapi.get_value("package-mode"), + "package-pprint-name": self._autorestapi.get_value("package-pprint-name"), + "packaging-files-config": self._autorestapi.get_value("package-configuration"), + "default-optional-constants-to-none": self._autorestapi.get_boolean_value( + "default-optional-constants-to-none" + ), + "generate-sample": self._autorestapi.get_boolean_value("generate-sample"), + "generate-test": self._autorestapi.get_boolean_value("generate-test"), + "default-api-version": self._autorestapi.get_value("default-api-version"), + } + return {k: v for k, v in options.items() if v is not None} + + def get_yaml(self) -> Dict[str, Any]: + inputs = self._autorestapi.list_inputs() + _LOGGER.debug("Possible Inputs: %s", inputs) + if "code-model-v4-no-tags.yaml" not in inputs: + raise ValueError("code-model-v4-no-tags.yaml must be a possible input") + + if self._autorestapi.get_value("input-yaml"): + input_yaml = self._autorestapi.get_value("input-yaml") + file_content = self._autorestapi.read_file(input_yaml) + else: + inputs = self._autorestapi.list_inputs() + _LOGGER.debug("Possible Inputs: %s", inputs) + if "code-model-v4-no-tags.yaml" not in inputs: + raise ValueError("code-model-v4-no-tags.yaml must be a possible input") + + file_content = self._autorestapi.read_file("code-model-v4-no-tags.yaml") + + # Parse the received YAML + return yaml.safe_load(file_content) + + def get_serializer(self, code_model: CodeModel): + return JinjaSerializerAutorest( + self._autorestapi, + code_model, + output_folder=self.output_folder, + ) diff --git a/packages/autorest.python/autorest/m2r.py b/packages/autorest.python/autorest/m2r.py new file mode 100644 index 00000000000..82e415c1d42 --- /dev/null +++ b/packages/autorest.python/autorest/m2r.py @@ -0,0 +1,15 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""An autorest MD to RST plugin. +""" +from typing import Any, Dict + +from pygen.core.m2r import M2R +from . import YamlUpdatePluginAutorest + +class M2RAutorest(YamlUpdatePluginAutorest, M2R): + def get_options(self) -> Dict[str, Any]: + return {} diff --git a/packages/autorest.python/autorest/postprocess.py b/packages/autorest.python/autorest/postprocess.py new file mode 100644 index 00000000000..78436b200a0 --- /dev/null +++ b/packages/autorest.python/autorest/postprocess.py @@ -0,0 +1,13 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict + +from pygen.core.postprocess import PostProcessPlugin +from . import PluginAutorest + +class PostProcessPluginAutorest(PostProcessPlugin, PluginAutorest): + def get_options(self) -> Dict[str, Any]: + return {"outputFolderUri": self._autorestapi.get_value("outputFolderUri")} diff --git a/packages/autorest.python/autorest/preprocess.py b/packages/autorest.python/autorest/preprocess.py new file mode 100644 index 00000000000..3eb4dd725a8 --- /dev/null +++ b/packages/autorest.python/autorest/preprocess.py @@ -0,0 +1,20 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""The preprocessing autorest plugin. +""" +from typing import Dict, Any +from . import YamlUpdatePluginAutorest +from pygen.core.preprocess import PreProcessPlugin + + +class PreProcessPluginAutorest(YamlUpdatePluginAutorest, PreProcessPlugin): + def get_options(self) -> Dict[str, Any]: + options = { + "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant"), + "azure-arm": self._autorestapi.get_boolean_value("azure-arm"), + "models-mode": self._autorestapi.get_value("models-mode"), + } + return {k: v for k, v in options.items() if v is not None} diff --git a/packages/pygen/README.md b/packages/pygen/README.md new file mode 100644 index 00000000000..3d8a65a8919 --- /dev/null +++ b/packages/pygen/README.md @@ -0,0 +1 @@ +# Core Library for Python Generation diff --git a/packages/pygen/core/__init__.py b/packages/pygen/core/__init__.py new file mode 100644 index 00000000000..280c4c97c65 --- /dev/null +++ b/packages/pygen/core/__init__.py @@ -0,0 +1,108 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from pathlib import Path +import json +from abc import ABC, abstractmethod +from typing import Any, Dict, Union, List + +import yaml + +from ._version import VERSION + + +__version__ = VERSION +_LOGGER = logging.getLogger(__name__) + + +class ReaderAndWriter: + def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None: + self.output_folder = Path(output_folder) + self._list_file: List[str] = [] + try: + with open( + Path(self.output_folder) / Path("..") / Path("python.json"), + "r", + encoding="utf-8-sig", + ) as fd: + python_json = json.load(fd) + except Exception: # pylint: disable=broad-except + python_json = {} + self.options = kwargs + if python_json: + _LOGGER.warning("Loading python.json file. This behavior will be depreacted") + self.options.update(python_json) + + def read_file(self, path: Union[str, Path]) -> str: + """Directly reading from disk""" + # make path relative to output folder + try: + with open(self.output_folder / Path(path), "r", encoding="utf-8-sig") as fd: + return fd.read() + except FileNotFoundError: + return "" + + def write_file(self, filename: Union[str, Path], file_content: str) -> None: + """Directly writing to disk""" + file_folder = Path(filename).parent + if not Path.is_dir(self.output_folder / file_folder): + Path.mkdir(self.output_folder / file_folder, parents=True) + with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd: + fd.write(file_content) + + def list_file(self) -> List[str]: + return [str(f) for f in self.output_folder.glob("**/*") if f.is_file()] + + +class Plugin(ReaderAndWriter, ABC): + """A base class for autorest plugin. + + :param autorestapi: An autorest API instance + """ + + @abstractmethod + def process(self) -> bool: + """The plugin process. + + :rtype: bool + :returns: True if everything's ok, False optherwise + :raises Exception: Could raise any exception, stacktrace will be sent to autorest API + """ + raise NotImplementedError() + + +class YamlUpdatePlugin(Plugin): + """A plugin that update the YAML as input.""" + + def get_yaml(self) -> Dict[str, Any]: + # cadl file doesn't have to be relative to output folder + with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd: + return yaml.safe_load(fd.read()) + + def write_yaml(self, yaml_string: str) -> None: + with open(self.options["cadl_file"], "w", encoding="utf-8-sig") as fd: + fd.write(yaml_string) + + def process(self) -> bool: + # List the input file, should be only one + yaml_data = self.get_yaml() + + self.update_yaml(yaml_data) + + yaml_string = yaml.safe_dump(yaml_data) + + self.write_yaml(yaml_string) + return True + + @abstractmethod + def update_yaml(self, yaml_data: Dict[str, Any]) -> None: + """The code-model-v4-no-tags yaml model tree. + + :rtype: updated yaml + :raises Exception: Could raise any exception, stacktrace will be sent to autorest API + """ + raise NotImplementedError() + diff --git a/packages/pygen/core/_utils.py b/packages/pygen/core/_utils.py new file mode 100644 index 00000000000..95bf493bde8 --- /dev/null +++ b/packages/pygen/core/_utils.py @@ -0,0 +1,149 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict, Tuple, List +import re +import argparse + + +def update_enum_value(name: str, value: Any, description: str, enum_type: Dict[str, Any]) -> Dict[str, Any]: + return { + "name": name, + "type": "enumvalue", + "value": value, + "description": description, + "enumType": enum_type, + "valueType": enum_type["valueType"], + } + + +def to_snake_case(name: str) -> str: + def replace_upper_characters(m) -> str: + match_str = m.group().lower() + if m.start() > 0 and name[m.start() - 1] == "_": + # we are good if a '_' already exists + return match_str + # the first letter should not have _ + prefix = "_" if m.start() > 0 else "" + + # we will add an extra _ if there are multiple upper case chars together + next_non_upper_case_char_location = m.start() + len(match_str) + if ( + len(match_str) > 2 + and len(name) - next_non_upper_case_char_location > 1 + and name[next_non_upper_case_char_location].isalpha() + ): + return prefix + match_str[: len(match_str) - 1] + "_" + match_str[len(match_str) - 1] + + return prefix + match_str + + result = re.sub("[A-Z]+", replace_upper_characters, name) + return result.replace(" ", "_").replace("__", "_").replace("-", "") + + +def parse_args( + need_cadl_file: bool = True, +) -> Tuple[argparse.Namespace, Dict[str, Any]]: + parser = argparse.ArgumentParser( + description="Run mypy against target folder. Add a local custom plugin to the path prior to execution. " + ) + parser.add_argument( + "--output-folder", + dest="output_folder", + help="Output folder for generated SDK", + required=True, + ) + parser.add_argument( + "--cadl-file", + dest="cadl_file", + help="Serialized cadl file", + required=need_cadl_file, + ) + parser.add_argument( + "--debug", + dest="debug", + help="Debug mode", + required=False, + action="store", + ) + args, unknown_args = parser.parse_known_args() + + def _get_value(value: Any) -> Any: + if value == "true": + return True + if value == "false": + return False + try: + return int(value) + except ValueError: + pass + return value + + unknown_args_ret = { + ua.strip("--").split("=", maxsplit=1)[0]: _get_value( # pylint: disable=bad-str-strip-call + ua.strip("--").split("=", maxsplit=1)[1] # pylint: disable=bad-str-strip-call + ) + for ua in unknown_args + } + return args, unknown_args_ret + + +def get_body_type_for_description(body_parameter: Dict[str, Any]) -> str: + if body_parameter["type"]["type"] == "binary": + return "binary" + if body_parameter["type"]["type"] == "string": + return "string" + return "JSON" + + +# used if we want to get a string / binary type etc +KNOWN_TYPES: Dict[str, Dict[str, Any]] = { + "string": {"type": "string"}, + "binary": {"type": "binary"}, + "anydict": {"type": "dict", "elementType": {"type": "any"}}, + "any-object": {"type": "any-object"}, +} + +JSON_REGEXP = re.compile(r"^(application|text)/(.+\+)?json$") + + +def build_policies( + is_arm: bool, + async_mode: bool, + *, + is_azure_flavor: bool = False, + tracing: bool = True, +) -> List[str]: + if is_azure_flavor: + # for Azure + async_prefix = "Async" if async_mode else "" + policies = [ + "policies.RequestIdPolicy(**kwargs)", + "self._config.headers_policy", + "self._config.user_agent_policy", + "self._config.proxy_policy", + "policies.ContentDecodePolicy(**kwargs)", + (f"{async_prefix}ARMAutoResourceProviderRegistrationPolicy()" if is_arm else None), + "self._config.redirect_policy", + "self._config.retry_policy", + "self._config.authentication_policy", + "self._config.custom_hook_policy", + "self._config.logging_policy", + "policies.DistributedTracingPolicy(**kwargs)" if tracing else None, + "policies.SensitiveHeaderCleanupPolicy(**kwargs) if self._config.redirect_policy else None", + "self._config.http_logging_policy", + ] + else: + # for non-Azure + policies = [ + "self._config.headers_policy", + "self._config.user_agent_policy", + "self._config.proxy_policy", + "policies.ContentDecodePolicy(**kwargs)", + "self._config.retry_policy", + "self._config.authentication_policy", + "self._config.logging_policy", + ] + return [p for p in policies if p] diff --git a/packages/pygen/core/_version.py b/packages/pygen/core/_version.py new file mode 100644 index 00000000000..aadc0f31ff8 --- /dev/null +++ b/packages/pygen/core/_version.py @@ -0,0 +1,7 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +VERSION = "0.1.0" diff --git a/packages/autorest.python/autorest/black/__init__.py b/packages/pygen/core/black/__init__.py similarity index 91% rename from packages/autorest.python/autorest/black/__init__.py rename to packages/pygen/core/black/__init__.py index 8df1c79241c..a17a536485a 100644 --- a/packages/autorest.python/autorest/black/__init__.py +++ b/packages/pygen/core/black/__init__.py @@ -10,7 +10,7 @@ import black from black.report import NothingChanged -from .. import Plugin, PluginAutorest +from .. import Plugin from .._utils import parse_args _LOGGER = logging.getLogger("blib2to3") @@ -65,11 +65,6 @@ def format_file(self, file: Path) -> None: self.write_file(file, file_content) -class BlackScriptPluginAutorest(BlackScriptPlugin, PluginAutorest): - def get_options(self) -> Dict[str, Any]: - return {"output_folder": self._autorestapi.get_value("outputFolderUri")} - - if __name__ == "__main__": # CADL pipeline will call this args, unknown_args = parse_args(need_cadl_file=False) diff --git a/packages/autorest.python/autorest/codegen/__init__.py b/packages/pygen/core/codegen/__init__.py similarity index 72% rename from packages/autorest.python/autorest/codegen/__init__.py rename to packages/pygen/core/codegen/__init__.py index ecfb70f5752..c5a30203843 100644 --- a/packages/autorest.python/autorest/codegen/__init__.py +++ b/packages/pygen/core/codegen/__init__.py @@ -9,10 +9,10 @@ import yaml -from .. import Plugin, PluginAutorest +from .. import Plugin from .._utils import parse_args from .models.code_model import CodeModel -from .serializers import JinjaSerializer, JinjaSerializerAutorest +from .serializers import JinjaSerializer from ._utils import DEFAULT_HEADER_TEXT, VALID_PACKAGE_MODE, TYPESPEC_PACKAGE_MODE @@ -324,86 +324,6 @@ def process(self) -> bool: return True -class CodeGeneratorAutorest(CodeGenerator, PluginAutorest): - def get_options(self) -> Dict[str, Any]: - if self._autorestapi.get_boolean_value("python3-only") is False: - _LOGGER.warning("You have passed in --python3-only=False. We have force overriden this to True.") - if self._autorestapi.get_boolean_value("add-python3-operation-files"): - _LOGGER.warning( - "You have passed in --add-python3-operation-files. " - "This flag no longer has an effect bc all SDKs are now Python3 only." - ) - if self._autorestapi.get_boolean_value("reformat-next-link"): - _LOGGER.warning( - "You have passed in --reformat-next-link. We have force overriden " - "this to False because we no longer reformat initial query parameters into next " - "calls unless explicitly defined in the service definition." - ) - options = { - "azure-arm": self._autorestapi.get_boolean_value("azure-arm"), - "header-text": self._autorestapi.get_value("header-text"), - "low-level-client": self._autorestapi.get_boolean_value("low-level-client", False), - "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant", True), - "show-operations": self._autorestapi.get_boolean_value("show-operations"), - "python3-only": self._autorestapi.get_boolean_value("python3-only"), - "head-as-boolean": self._autorestapi.get_boolean_value("head-as-boolean", False), - "keep-version-file": self._autorestapi.get_boolean_value("keep-version-file"), - "no-async": self._autorestapi.get_boolean_value("no-async"), - "no-namespace-folders": self._autorestapi.get_boolean_value("no-namespace-folders"), - "basic-setup-py": self._autorestapi.get_boolean_value("basic-setup-py"), - "package-name": self._autorestapi.get_value("package-name"), - "package-version": self._autorestapi.get_value("package-version"), - "client-side-validation": self._autorestapi.get_boolean_value("client-side-validation"), - "tracing": self._autorestapi.get_boolean_value("trace"), - "multiapi": self._autorestapi.get_boolean_value("multiapi", False), - "polymorphic-examples": self._autorestapi.get_value("polymorphic-examples"), - "models-mode": self._autorestapi.get_value("models-mode"), - "builders-visibility": self._autorestapi.get_value("builders-visibility"), - "show-send-request": self._autorestapi.get_boolean_value("show-send-request"), - "only-path-and-body-params-positional": self._autorestapi.get_boolean_value( - "only-path-and-body-params-positional" - ), - "combine-operation-files": self._autorestapi.get_boolean_value("combine-operation-files"), - "package-mode": self._autorestapi.get_value("package-mode"), - "package-pprint-name": self._autorestapi.get_value("package-pprint-name"), - "packaging-files-config": self._autorestapi.get_value("package-configuration"), - "default-optional-constants-to-none": self._autorestapi.get_boolean_value( - "default-optional-constants-to-none" - ), - "generate-sample": self._autorestapi.get_boolean_value("generate-sample"), - "generate-test": self._autorestapi.get_boolean_value("generate-test"), - "default-api-version": self._autorestapi.get_value("default-api-version"), - } - return {k: v for k, v in options.items() if v is not None} - - def get_yaml(self) -> Dict[str, Any]: - inputs = self._autorestapi.list_inputs() - _LOGGER.debug("Possible Inputs: %s", inputs) - if "code-model-v4-no-tags.yaml" not in inputs: - raise ValueError("code-model-v4-no-tags.yaml must be a possible input") - - if self._autorestapi.get_value("input-yaml"): - input_yaml = self._autorestapi.get_value("input-yaml") - file_content = self._autorestapi.read_file(input_yaml) - else: - inputs = self._autorestapi.list_inputs() - _LOGGER.debug("Possible Inputs: %s", inputs) - if "code-model-v4-no-tags.yaml" not in inputs: - raise ValueError("code-model-v4-no-tags.yaml must be a possible input") - - file_content = self._autorestapi.read_file("code-model-v4-no-tags.yaml") - - # Parse the received YAML - return yaml.safe_load(file_content) - - def get_serializer(self, code_model: CodeModel): - return JinjaSerializerAutorest( - self._autorestapi, - code_model, - output_folder=self.output_folder, - ) - - if __name__ == "__main__": # CADL pipeline will call this parsed_args, unknown_args = parse_args() diff --git a/packages/autorest.python/autorest/codegen/_utils.py b/packages/pygen/core/codegen/_utils.py similarity index 100% rename from packages/autorest.python/autorest/codegen/_utils.py rename to packages/pygen/core/codegen/_utils.py diff --git a/packages/autorest.python/autorest/codegen/models/__init__.py b/packages/pygen/core/codegen/models/__init__.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/__init__.py rename to packages/pygen/core/codegen/models/__init__.py diff --git a/packages/autorest.python/autorest/codegen/models/base.py b/packages/pygen/core/codegen/models/base.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/base.py rename to packages/pygen/core/codegen/models/base.py diff --git a/packages/autorest.python/autorest/codegen/models/base_builder.py b/packages/pygen/core/codegen/models/base_builder.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/base_builder.py rename to packages/pygen/core/codegen/models/base_builder.py diff --git a/packages/autorest.python/autorest/codegen/models/client.py b/packages/pygen/core/codegen/models/client.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/client.py rename to packages/pygen/core/codegen/models/client.py diff --git a/packages/autorest.python/autorest/codegen/models/code_model.py b/packages/pygen/core/codegen/models/code_model.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/code_model.py rename to packages/pygen/core/codegen/models/code_model.py diff --git a/packages/autorest.python/autorest/codegen/models/combined_type.py b/packages/pygen/core/codegen/models/combined_type.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/combined_type.py rename to packages/pygen/core/codegen/models/combined_type.py diff --git a/packages/autorest.python/autorest/codegen/models/constant_type.py b/packages/pygen/core/codegen/models/constant_type.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/constant_type.py rename to packages/pygen/core/codegen/models/constant_type.py diff --git a/packages/autorest.python/autorest/codegen/models/credential_types.py b/packages/pygen/core/codegen/models/credential_types.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/credential_types.py rename to packages/pygen/core/codegen/models/credential_types.py diff --git a/packages/autorest.python/autorest/codegen/models/dictionary_type.py b/packages/pygen/core/codegen/models/dictionary_type.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/dictionary_type.py rename to packages/pygen/core/codegen/models/dictionary_type.py diff --git a/packages/autorest.python/autorest/codegen/models/enum_type.py b/packages/pygen/core/codegen/models/enum_type.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/enum_type.py rename to packages/pygen/core/codegen/models/enum_type.py diff --git a/packages/autorest.python/autorest/codegen/models/imports.py b/packages/pygen/core/codegen/models/imports.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/imports.py rename to packages/pygen/core/codegen/models/imports.py diff --git a/packages/autorest.python/autorest/codegen/models/list_type.py b/packages/pygen/core/codegen/models/list_type.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/list_type.py rename to packages/pygen/core/codegen/models/list_type.py diff --git a/packages/autorest.python/autorest/codegen/models/lro_operation.py b/packages/pygen/core/codegen/models/lro_operation.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/lro_operation.py rename to packages/pygen/core/codegen/models/lro_operation.py diff --git a/packages/autorest.python/autorest/codegen/models/lro_paging_operation.py b/packages/pygen/core/codegen/models/lro_paging_operation.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/lro_paging_operation.py rename to packages/pygen/core/codegen/models/lro_paging_operation.py diff --git a/packages/autorest.python/autorest/codegen/models/model_type.py b/packages/pygen/core/codegen/models/model_type.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/model_type.py rename to packages/pygen/core/codegen/models/model_type.py diff --git a/packages/autorest.python/autorest/codegen/models/operation.py b/packages/pygen/core/codegen/models/operation.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/operation.py rename to packages/pygen/core/codegen/models/operation.py diff --git a/packages/autorest.python/autorest/codegen/models/operation_group.py b/packages/pygen/core/codegen/models/operation_group.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/operation_group.py rename to packages/pygen/core/codegen/models/operation_group.py diff --git a/packages/autorest.python/autorest/codegen/models/paging_operation.py b/packages/pygen/core/codegen/models/paging_operation.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/paging_operation.py rename to packages/pygen/core/codegen/models/paging_operation.py diff --git a/packages/autorest.python/autorest/codegen/models/parameter.py b/packages/pygen/core/codegen/models/parameter.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/parameter.py rename to packages/pygen/core/codegen/models/parameter.py diff --git a/packages/autorest.python/autorest/codegen/models/parameter_list.py b/packages/pygen/core/codegen/models/parameter_list.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/parameter_list.py rename to packages/pygen/core/codegen/models/parameter_list.py diff --git a/packages/autorest.python/autorest/codegen/models/primitive_types.py b/packages/pygen/core/codegen/models/primitive_types.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/primitive_types.py rename to packages/pygen/core/codegen/models/primitive_types.py diff --git a/packages/autorest.python/autorest/codegen/models/property.py b/packages/pygen/core/codegen/models/property.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/property.py rename to packages/pygen/core/codegen/models/property.py diff --git a/packages/autorest.python/autorest/codegen/models/request_builder.py b/packages/pygen/core/codegen/models/request_builder.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/request_builder.py rename to packages/pygen/core/codegen/models/request_builder.py diff --git a/packages/autorest.python/autorest/codegen/models/request_builder_parameter.py b/packages/pygen/core/codegen/models/request_builder_parameter.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/request_builder_parameter.py rename to packages/pygen/core/codegen/models/request_builder_parameter.py diff --git a/packages/autorest.python/autorest/codegen/models/response.py b/packages/pygen/core/codegen/models/response.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/response.py rename to packages/pygen/core/codegen/models/response.py diff --git a/packages/autorest.python/autorest/codegen/models/utils.py b/packages/pygen/core/codegen/models/utils.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/utils.py rename to packages/pygen/core/codegen/models/utils.py diff --git a/packages/autorest.python/autorest/codegen/serializers/__init__.py b/packages/pygen/core/codegen/serializers/__init__.py similarity index 97% rename from packages/autorest.python/autorest/codegen/serializers/__init__.py rename to packages/pygen/core/codegen/serializers/__init__.py index 377a6b20ad1..4db3a0a07e5 100644 --- a/packages/autorest.python/autorest/codegen/serializers/__init__.py +++ b/packages/pygen/core/codegen/serializers/__init__.py @@ -8,8 +8,7 @@ from pathlib import Path from jinja2 import PackageLoader, Environment, FileSystemLoader, StrictUndefined -from ... import ReaderAndWriter, ReaderAndWriterAutorest -from ...jsonrpc import AutorestAPI +from ... import ReaderAndWriter from ..models import ( OperationGroup, RequestBuilder, @@ -138,7 +137,7 @@ def _serialize_namespace_level(self, env: Environment, namespace_path: Path, cli def serialize(self) -> None: env = Environment( - loader=PackageLoader("autorest.codegen", "templates"), + loader=PackageLoader("core.codegen", "templates"), keep_trailing_newline=True, line_statement_prefix="##", line_comment_prefix="###", @@ -196,7 +195,7 @@ def _serialize_and_write_package_files(self, namespace_path: Path) -> None: root_of_sdk = self._package_root_folder(namespace_path) if self.code_model.options["package_mode"] in VALID_PACKAGE_MODE: env = Environment( - loader=PackageLoader("autorest.codegen", "templates/packaging_templates"), + loader=PackageLoader("core.codegen", "templates/packaging_templates"), undefined=StrictUndefined, ) @@ -569,20 +568,3 @@ def _serialize_and_write_test(self, env: Environment, namespace_path: Path): log_error = f"error happens in test generation for operation group {og.class_name}: {e}" _LOGGER.error(log_error) self.code_model.for_test = False - - -class JinjaSerializerAutorest(JinjaSerializer, ReaderAndWriterAutorest): - def __init__( - self, - autorestapi: AutorestAPI, - code_model: CodeModel, - *, - output_folder: Union[str, Path], - **kwargs: Any, - ) -> None: - super().__init__( - autorestapi=autorestapi, - code_model=code_model, - output_folder=output_folder, - **kwargs, - ) diff --git a/packages/autorest.python/autorest/codegen/serializers/base_serializer.py b/packages/pygen/core/codegen/serializers/base_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/base_serializer.py rename to packages/pygen/core/codegen/serializers/base_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/builder_serializer.py b/packages/pygen/core/codegen/serializers/builder_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/builder_serializer.py rename to packages/pygen/core/codegen/serializers/builder_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/client_serializer.py b/packages/pygen/core/codegen/serializers/client_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/client_serializer.py rename to packages/pygen/core/codegen/serializers/client_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/enum_serializer.py b/packages/pygen/core/codegen/serializers/enum_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/enum_serializer.py rename to packages/pygen/core/codegen/serializers/enum_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/general_serializer.py b/packages/pygen/core/codegen/serializers/general_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/general_serializer.py rename to packages/pygen/core/codegen/serializers/general_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/import_serializer.py b/packages/pygen/core/codegen/serializers/import_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/import_serializer.py rename to packages/pygen/core/codegen/serializers/import_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/metadata_serializer.py b/packages/pygen/core/codegen/serializers/metadata_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/metadata_serializer.py rename to packages/pygen/core/codegen/serializers/metadata_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/model_init_serializer.py b/packages/pygen/core/codegen/serializers/model_init_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/model_init_serializer.py rename to packages/pygen/core/codegen/serializers/model_init_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/model_serializer.py b/packages/pygen/core/codegen/serializers/model_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/model_serializer.py rename to packages/pygen/core/codegen/serializers/model_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/operation_groups_serializer.py b/packages/pygen/core/codegen/serializers/operation_groups_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/operation_groups_serializer.py rename to packages/pygen/core/codegen/serializers/operation_groups_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/operations_init_serializer.py b/packages/pygen/core/codegen/serializers/operations_init_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/operations_init_serializer.py rename to packages/pygen/core/codegen/serializers/operations_init_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/parameter_serializer.py b/packages/pygen/core/codegen/serializers/parameter_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/parameter_serializer.py rename to packages/pygen/core/codegen/serializers/parameter_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/patch_serializer.py b/packages/pygen/core/codegen/serializers/patch_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/patch_serializer.py rename to packages/pygen/core/codegen/serializers/patch_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/request_builders_serializer.py b/packages/pygen/core/codegen/serializers/request_builders_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/request_builders_serializer.py rename to packages/pygen/core/codegen/serializers/request_builders_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/sample_serializer.py b/packages/pygen/core/codegen/serializers/sample_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/sample_serializer.py rename to packages/pygen/core/codegen/serializers/sample_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/test_serializer.py b/packages/pygen/core/codegen/serializers/test_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/test_serializer.py rename to packages/pygen/core/codegen/serializers/test_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/types_serializer.py b/packages/pygen/core/codegen/serializers/types_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/types_serializer.py rename to packages/pygen/core/codegen/serializers/types_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/utils.py b/packages/pygen/core/codegen/serializers/utils.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/utils.py rename to packages/pygen/core/codegen/serializers/utils.py diff --git a/packages/autorest.python/autorest/codegen/templates/client.py.jinja2 b/packages/pygen/core/codegen/templates/client.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/client.py.jinja2 rename to packages/pygen/core/codegen/templates/client.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/client_container.py.jinja2 b/packages/pygen/core/codegen/templates/client_container.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/client_container.py.jinja2 rename to packages/pygen/core/codegen/templates/client_container.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/config.py.jinja2 b/packages/pygen/core/codegen/templates/config.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/config.py.jinja2 rename to packages/pygen/core/codegen/templates/config.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/config_container.py.jinja2 b/packages/pygen/core/codegen/templates/config_container.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/config_container.py.jinja2 rename to packages/pygen/core/codegen/templates/config_container.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/conftest.py.jinja2 b/packages/pygen/core/codegen/templates/conftest.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/conftest.py.jinja2 rename to packages/pygen/core/codegen/templates/conftest.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/enum.py.jinja2 b/packages/pygen/core/codegen/templates/enum.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/enum.py.jinja2 rename to packages/pygen/core/codegen/templates/enum.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/enum_container.py.jinja2 b/packages/pygen/core/codegen/templates/enum_container.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/enum_container.py.jinja2 rename to packages/pygen/core/codegen/templates/enum_container.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/init.py.jinja2 b/packages/pygen/core/codegen/templates/init.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/init.py.jinja2 rename to packages/pygen/core/codegen/templates/init.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/keywords.jinja2 b/packages/pygen/core/codegen/templates/keywords.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/keywords.jinja2 rename to packages/pygen/core/codegen/templates/keywords.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/lro_operation.py.jinja2 b/packages/pygen/core/codegen/templates/lro_operation.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/lro_operation.py.jinja2 rename to packages/pygen/core/codegen/templates/lro_operation.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/lro_paging_operation.py.jinja2 b/packages/pygen/core/codegen/templates/lro_paging_operation.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/lro_paging_operation.py.jinja2 rename to packages/pygen/core/codegen/templates/lro_paging_operation.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/macros.jinja2 b/packages/pygen/core/codegen/templates/macros.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/macros.jinja2 rename to packages/pygen/core/codegen/templates/macros.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/metadata.json.jinja2 b/packages/pygen/core/codegen/templates/metadata.json.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/metadata.json.jinja2 rename to packages/pygen/core/codegen/templates/metadata.json.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/model_base.py.jinja2 b/packages/pygen/core/codegen/templates/model_base.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/model_base.py.jinja2 rename to packages/pygen/core/codegen/templates/model_base.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/model_container.py.jinja2 b/packages/pygen/core/codegen/templates/model_container.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/model_container.py.jinja2 rename to packages/pygen/core/codegen/templates/model_container.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/model_dpg.py.jinja2 b/packages/pygen/core/codegen/templates/model_dpg.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/model_dpg.py.jinja2 rename to packages/pygen/core/codegen/templates/model_dpg.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/model_init.py.jinja2 b/packages/pygen/core/codegen/templates/model_init.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/model_init.py.jinja2 rename to packages/pygen/core/codegen/templates/model_init.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/model_msrest.py.jinja2 b/packages/pygen/core/codegen/templates/model_msrest.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/model_msrest.py.jinja2 rename to packages/pygen/core/codegen/templates/model_msrest.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/operation.py.jinja2 b/packages/pygen/core/codegen/templates/operation.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/operation.py.jinja2 rename to packages/pygen/core/codegen/templates/operation.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/operation_group.py.jinja2 b/packages/pygen/core/codegen/templates/operation_group.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/operation_group.py.jinja2 rename to packages/pygen/core/codegen/templates/operation_group.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/operation_groups_container.py.jinja2 b/packages/pygen/core/codegen/templates/operation_groups_container.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/operation_groups_container.py.jinja2 rename to packages/pygen/core/codegen/templates/operation_groups_container.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/operation_tools.jinja2 b/packages/pygen/core/codegen/templates/operation_tools.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/operation_tools.jinja2 rename to packages/pygen/core/codegen/templates/operation_tools.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/operations_folder_init.py.jinja2 b/packages/pygen/core/codegen/templates/operations_folder_init.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/operations_folder_init.py.jinja2 rename to packages/pygen/core/codegen/templates/operations_folder_init.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 b/packages/pygen/core/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 rename to packages/pygen/core/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/LICENSE.jinja2 b/packages/pygen/core/codegen/templates/packaging_templates/LICENSE.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/LICENSE.jinja2 rename to packages/pygen/core/codegen/templates/packaging_templates/LICENSE.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/MANIFEST.in.jinja2 b/packages/pygen/core/codegen/templates/packaging_templates/MANIFEST.in.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/MANIFEST.in.jinja2 rename to packages/pygen/core/codegen/templates/packaging_templates/MANIFEST.in.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/README.md.jinja2 b/packages/pygen/core/codegen/templates/packaging_templates/README.md.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/README.md.jinja2 rename to packages/pygen/core/codegen/templates/packaging_templates/README.md.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 b/packages/pygen/core/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 rename to packages/pygen/core/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/setup.py.jinja2 b/packages/pygen/core/codegen/templates/packaging_templates/setup.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/setup.py.jinja2 rename to packages/pygen/core/codegen/templates/packaging_templates/setup.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/paging_operation.py.jinja2 b/packages/pygen/core/codegen/templates/paging_operation.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/paging_operation.py.jinja2 rename to packages/pygen/core/codegen/templates/paging_operation.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/patch.py.jinja2 b/packages/pygen/core/codegen/templates/patch.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/patch.py.jinja2 rename to packages/pygen/core/codegen/templates/patch.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/pkgutil_init.py.jinja2 b/packages/pygen/core/codegen/templates/pkgutil_init.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/pkgutil_init.py.jinja2 rename to packages/pygen/core/codegen/templates/pkgutil_init.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/request_builder.py.jinja2 b/packages/pygen/core/codegen/templates/request_builder.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/request_builder.py.jinja2 rename to packages/pygen/core/codegen/templates/request_builder.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/request_builders.py.jinja2 b/packages/pygen/core/codegen/templates/request_builders.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/request_builders.py.jinja2 rename to packages/pygen/core/codegen/templates/request_builders.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/rest_init.py.jinja2 b/packages/pygen/core/codegen/templates/rest_init.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/rest_init.py.jinja2 rename to packages/pygen/core/codegen/templates/rest_init.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/sample.py.jinja2 b/packages/pygen/core/codegen/templates/sample.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/sample.py.jinja2 rename to packages/pygen/core/codegen/templates/sample.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/serialization.py.jinja2 b/packages/pygen/core/codegen/templates/serialization.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/serialization.py.jinja2 rename to packages/pygen/core/codegen/templates/serialization.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/test.py.jinja2 b/packages/pygen/core/codegen/templates/test.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/test.py.jinja2 rename to packages/pygen/core/codegen/templates/test.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/testpreparer.py.jinja2 b/packages/pygen/core/codegen/templates/testpreparer.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/testpreparer.py.jinja2 rename to packages/pygen/core/codegen/templates/testpreparer.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/types.py.jinja2 b/packages/pygen/core/codegen/templates/types.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/types.py.jinja2 rename to packages/pygen/core/codegen/templates/types.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/validation.py.jinja2 b/packages/pygen/core/codegen/templates/validation.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/validation.py.jinja2 rename to packages/pygen/core/codegen/templates/validation.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/vendor.py.jinja2 b/packages/pygen/core/codegen/templates/vendor.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/vendor.py.jinja2 rename to packages/pygen/core/codegen/templates/vendor.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/version.py.jinja2 b/packages/pygen/core/codegen/templates/version.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/version.py.jinja2 rename to packages/pygen/core/codegen/templates/version.py.jinja2 diff --git a/packages/autorest.python/autorest/m2r/__init__.py b/packages/pygen/core/m2r/__init__.py similarity index 86% rename from packages/autorest.python/autorest/m2r/__init__.py rename to packages/pygen/core/m2r/__init__.py index 88ea00c7f16..450a0fadf63 100644 --- a/packages/autorest.python/autorest/m2r/__init__.py +++ b/packages/pygen/core/m2r/__init__.py @@ -3,21 +3,21 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -"""An autorest MD to RST plugin. +"""An MD to RST plugin. """ import logging from typing import Any, Dict, Set, Union import m2r2 -from .. import YamlUpdatePluginAutorest, YamlUpdatePlugin +from .. import YamlUpdatePlugin from .._utils import parse_args _LOGGER = logging.getLogger(__name__) -class AutorestRender(m2r2.RestRenderer): +class GeneratorRenderer(m2r2.RestRenderer): """Redefine the concept of inline HTML in the renderer, we don't want to define a new format in the description/summary. """ @@ -54,16 +54,11 @@ def _convert_docstring_no_cycles(self, yaml_data: Union[Dict[str, Any], str], no def convert_to_rst(string_to_convert: str) -> str: """Convert that string from MD to RST.""" try: - return m2r2.convert(string_to_convert, renderer=AutorestRender()).strip() + return m2r2.convert(string_to_convert, renderer=GeneratorRenderer()).strip() except Exception: # pylint: disable=broad-except return string_to_convert -class M2RAutorest(YamlUpdatePluginAutorest, M2R): - def get_options(self) -> Dict[str, Any]: - return {} - - if __name__ == "__main__": # CADL pipeline will call this args, unknown_args = parse_args() diff --git a/packages/autorest.python/autorest/postprocess/__init__.py b/packages/pygen/core/postprocess/__init__.py similarity index 96% rename from packages/autorest.python/autorest/postprocess/__init__.py rename to packages/pygen/core/postprocess/__init__.py index a0f16378cee..771177261cb 100644 --- a/packages/autorest.python/autorest/postprocess/__init__.py +++ b/packages/pygen/core/postprocess/__init__.py @@ -12,7 +12,7 @@ from black.report import NothingChanged from .venvtools import ExtendedEnvBuilder, python_run -from .. import Plugin, PluginAutorest +from .. import Plugin _BLACK_MODE = black.Mode() # pyright: ignore [reportPrivateImportUsage] _BLACK_MODE.line_length = 120 @@ -181,8 +181,3 @@ def fix_imports_in_init(self, generated_file_name: str, folder_path: Path, names file_content = file_content.replace("__all__ = [", f"__all__ = [\n{added_objs_all}", 1) formatted_file = format_file(file, file_content) self.write_file(file, formatted_file) - - -class PostProcessPluginAutorest(PostProcessPlugin, PluginAutorest): - def get_options(self) -> Dict[str, Any]: - return {"outputFolderUri": self._autorestapi.get_value("outputFolderUri")} diff --git a/packages/autorest.python/autorest/postprocess/get_all.py b/packages/pygen/core/postprocess/get_all.py similarity index 100% rename from packages/autorest.python/autorest/postprocess/get_all.py rename to packages/pygen/core/postprocess/get_all.py diff --git a/packages/autorest.python/autorest/postprocess/venvtools.py b/packages/pygen/core/postprocess/venvtools.py similarity index 100% rename from packages/autorest.python/autorest/postprocess/venvtools.py rename to packages/pygen/core/postprocess/venvtools.py diff --git a/packages/autorest.python/autorest/preprocess/__init__.py b/packages/pygen/core/preprocess/__init__.py similarity index 97% rename from packages/autorest.python/autorest/preprocess/__init__.py rename to packages/pygen/core/preprocess/__init__.py index 70c18c7f566..6cea93cc121 100644 --- a/packages/autorest.python/autorest/preprocess/__init__.py +++ b/packages/pygen/core/preprocess/__init__.py @@ -16,7 +16,7 @@ ) from .python_mappings import CADL_RESERVED_WORDS, RESERVED_WORDS, PadType -from .. import YamlUpdatePlugin, YamlUpdatePluginAutorest +from .. import YamlUpdatePlugin from .._utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES @@ -477,16 +477,6 @@ def update_yaml(self, yaml_data: Dict[str, Any]) -> None: yaml_data["namespace"] = pad_builtin_namespaces(yaml_data["namespace"]) -class PreProcessPluginAutorest(YamlUpdatePluginAutorest, PreProcessPlugin): - def get_options(self) -> Dict[str, Any]: - options = { - "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant"), - "azure-arm": self._autorestapi.get_boolean_value("azure-arm"), - "models-mode": self._autorestapi.get_value("models-mode"), - } - return {k: v for k, v in options.items() if v is not None} - - if __name__ == "__main__": # CADL pipeline will call this args, unknown_args = parse_args() diff --git a/packages/autorest.python/autorest/preprocess/helpers.py b/packages/pygen/core/preprocess/helpers.py similarity index 100% rename from packages/autorest.python/autorest/preprocess/helpers.py rename to packages/pygen/core/preprocess/helpers.py diff --git a/packages/autorest.python/autorest/preprocess/python_mappings.py b/packages/pygen/core/preprocess/python_mappings.py similarity index 98% rename from packages/autorest.python/autorest/preprocess/python_mappings.py rename to packages/pygen/core/preprocess/python_mappings.py index d83df4793be..4edebd7120b 100644 --- a/packages/autorest.python/autorest/preprocess/python_mappings.py +++ b/packages/pygen/core/preprocess/python_mappings.py @@ -112,7 +112,7 @@ class PadType(str, Enum): PadType.METHOD: [*_always_reserved], PadType.PARAMETER: [ "self", - # these are kwargs we've reserved for our autorest generated operations + # these are kwargs we've reserved for our generated operations "content_type", "accept", "cls", diff --git a/packages/pygen/setup.py b/packages/pygen/setup.py new file mode 100644 index 00000000000..c74fbc0c88d --- /dev/null +++ b/packages/pygen/setup.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + + +import os +import re + +from setuptools import setup, find_packages + + +# Version extraction inspired from 'requests' +with open(os.path.join("core", "_version.py"), "r") as fd: + version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) + +if not version: + raise RuntimeError("Cannot find version information") + +setup( + name="pygencore", + version=version, + description="Core Library for Python Generation", + long_description=open("README.md", "r").read(), + long_description_content_type="text/markdown", + license="MIT License", + author="Microsoft Corporation", + author_email="azpysdkhelp@microsoft.com", + url="https://github.com/Azure/autorest.python/packages/core", + classifiers=[ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", + ], + packages=find_packages( + exclude=[ + "test", + ] + ), + install_requires=[ + "json-rpc", + "Jinja2 >= 2.11", # I need "include" and auto-context + blank line are not indented by default + "pyyaml", + "m2r2", + "black", + ], +) From f899041ac566cf6adcdc3f38bd8a05d7b20693b8 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 12:32:32 -0400 Subject: [PATCH 02/75] split package into pygen/core and autorest.python --- packages/autorest.python/autorest/__init__.py | 2 +- packages/autorest.python/autorest/black.py | 2 +- packages/autorest.python/autorest/codegen.py | 6 +++--- packages/autorest.python/autorest/m2r.py | 2 +- packages/autorest.python/autorest/postprocess.py | 2 +- packages/autorest.python/autorest/preprocess.py | 2 +- packages/pygen/{core => pygen}/__init__.py | 0 packages/pygen/{core => pygen}/_utils.py | 0 packages/pygen/{core => pygen}/_version.py | 0 packages/pygen/{core => pygen}/black/__init__.py | 0 packages/pygen/{core => pygen}/codegen/__init__.py | 0 packages/pygen/{core => pygen}/codegen/_utils.py | 0 packages/pygen/{core => pygen}/codegen/models/__init__.py | 0 packages/pygen/{core => pygen}/codegen/models/base.py | 0 .../pygen/{core => pygen}/codegen/models/base_builder.py | 0 packages/pygen/{core => pygen}/codegen/models/client.py | 0 packages/pygen/{core => pygen}/codegen/models/code_model.py | 0 .../pygen/{core => pygen}/codegen/models/combined_type.py | 2 +- .../pygen/{core => pygen}/codegen/models/constant_type.py | 0 .../{core => pygen}/codegen/models/credential_types.py | 0 .../pygen/{core => pygen}/codegen/models/dictionary_type.py | 0 packages/pygen/{core => pygen}/codegen/models/enum_type.py | 0 packages/pygen/{core => pygen}/codegen/models/imports.py | 0 packages/pygen/{core => pygen}/codegen/models/list_type.py | 0 .../pygen/{core => pygen}/codegen/models/lro_operation.py | 0 .../{core => pygen}/codegen/models/lro_paging_operation.py | 0 packages/pygen/{core => pygen}/codegen/models/model_type.py | 2 +- packages/pygen/{core => pygen}/codegen/models/operation.py | 0 .../pygen/{core => pygen}/codegen/models/operation_group.py | 2 +- .../{core => pygen}/codegen/models/paging_operation.py | 0 packages/pygen/{core => pygen}/codegen/models/parameter.py | 0 .../pygen/{core => pygen}/codegen/models/parameter_list.py | 0 .../pygen/{core => pygen}/codegen/models/primitive_types.py | 0 packages/pygen/{core => pygen}/codegen/models/property.py | 0 .../pygen/{core => pygen}/codegen/models/request_builder.py | 0 .../codegen/models/request_builder_parameter.py | 0 packages/pygen/{core => pygen}/codegen/models/response.py | 0 packages/pygen/{core => pygen}/codegen/models/utils.py | 0 .../pygen/{core => pygen}/codegen/serializers/__init__.py | 4 ++-- .../{core => pygen}/codegen/serializers/base_serializer.py | 0 .../codegen/serializers/builder_serializer.py | 0 .../codegen/serializers/client_serializer.py | 0 .../{core => pygen}/codegen/serializers/enum_serializer.py | 0 .../codegen/serializers/general_serializer.py | 0 .../codegen/serializers/import_serializer.py | 0 .../codegen/serializers/metadata_serializer.py | 0 .../codegen/serializers/model_init_serializer.py | 0 .../{core => pygen}/codegen/serializers/model_serializer.py | 0 .../codegen/serializers/operation_groups_serializer.py | 0 .../codegen/serializers/operations_init_serializer.py | 2 +- .../codegen/serializers/parameter_serializer.py | 0 .../{core => pygen}/codegen/serializers/patch_serializer.py | 0 .../codegen/serializers/request_builders_serializer.py | 0 .../codegen/serializers/sample_serializer.py | 2 +- .../{core => pygen}/codegen/serializers/test_serializer.py | 0 .../{core => pygen}/codegen/serializers/types_serializer.py | 0 packages/pygen/{core => pygen}/codegen/serializers/utils.py | 0 .../{core => pygen}/codegen/templates/client.py.jinja2 | 0 .../codegen/templates/client_container.py.jinja2 | 0 .../{core => pygen}/codegen/templates/config.py.jinja2 | 0 .../codegen/templates/config_container.py.jinja2 | 0 .../{core => pygen}/codegen/templates/conftest.py.jinja2 | 0 .../pygen/{core => pygen}/codegen/templates/enum.py.jinja2 | 0 .../codegen/templates/enum_container.py.jinja2 | 0 .../pygen/{core => pygen}/codegen/templates/init.py.jinja2 | 0 .../pygen/{core => pygen}/codegen/templates/keywords.jinja2 | 0 .../codegen/templates/lro_operation.py.jinja2 | 0 .../codegen/templates/lro_paging_operation.py.jinja2 | 0 .../pygen/{core => pygen}/codegen/templates/macros.jinja2 | 0 .../{core => pygen}/codegen/templates/metadata.json.jinja2 | 0 .../{core => pygen}/codegen/templates/model_base.py.jinja2 | 0 .../codegen/templates/model_container.py.jinja2 | 0 .../{core => pygen}/codegen/templates/model_dpg.py.jinja2 | 0 .../{core => pygen}/codegen/templates/model_init.py.jinja2 | 0 .../codegen/templates/model_msrest.py.jinja2 | 0 .../{core => pygen}/codegen/templates/operation.py.jinja2 | 0 .../codegen/templates/operation_group.py.jinja2 | 0 .../codegen/templates/operation_groups_container.py.jinja2 | 0 .../codegen/templates/operation_tools.jinja2 | 0 .../codegen/templates/operations_folder_init.py.jinja2 | 0 .../templates/packaging_templates/CHANGELOG.md.jinja2 | 0 .../codegen/templates/packaging_templates/LICENSE.jinja2 | 0 .../templates/packaging_templates/MANIFEST.in.jinja2 | 0 .../codegen/templates/packaging_templates/README.md.jinja2 | 0 .../packaging_templates/dev_requirements.txt.jinja2 | 0 .../codegen/templates/packaging_templates/setup.py.jinja2 | 0 .../codegen/templates/paging_operation.py.jinja2 | 0 .../pygen/{core => pygen}/codegen/templates/patch.py.jinja2 | 0 .../codegen/templates/pkgutil_init.py.jinja2 | 0 .../codegen/templates/request_builder.py.jinja2 | 0 .../codegen/templates/request_builders.py.jinja2 | 0 .../{core => pygen}/codegen/templates/rest_init.py.jinja2 | 0 .../{core => pygen}/codegen/templates/sample.py.jinja2 | 0 .../codegen/templates/serialization.py.jinja2 | 0 .../pygen/{core => pygen}/codegen/templates/test.py.jinja2 | 0 .../codegen/templates/testpreparer.py.jinja2 | 0 .../pygen/{core => pygen}/codegen/templates/types.py.jinja2 | 0 .../{core => pygen}/codegen/templates/validation.py.jinja2 | 0 .../{core => pygen}/codegen/templates/vendor.py.jinja2 | 0 .../{core => pygen}/codegen/templates/version.py.jinja2 | 0 packages/pygen/{core => pygen}/m2r/__init__.py | 0 packages/pygen/{core => pygen}/postprocess/__init__.py | 0 packages/pygen/{core => pygen}/postprocess/get_all.py | 0 packages/pygen/{core => pygen}/postprocess/venvtools.py | 0 packages/pygen/{core => pygen}/preprocess/__init__.py | 0 packages/pygen/{core => pygen}/preprocess/helpers.py | 0 .../pygen/{core => pygen}/preprocess/python_mappings.py | 0 packages/pygen/setup.py | 4 ++-- 108 files changed, 17 insertions(+), 17 deletions(-) rename packages/pygen/{core => pygen}/__init__.py (100%) rename packages/pygen/{core => pygen}/_utils.py (100%) rename packages/pygen/{core => pygen}/_version.py (100%) rename packages/pygen/{core => pygen}/black/__init__.py (100%) rename packages/pygen/{core => pygen}/codegen/__init__.py (100%) rename packages/pygen/{core => pygen}/codegen/_utils.py (100%) rename packages/pygen/{core => pygen}/codegen/models/__init__.py (100%) rename packages/pygen/{core => pygen}/codegen/models/base.py (100%) rename packages/pygen/{core => pygen}/codegen/models/base_builder.py (100%) rename packages/pygen/{core => pygen}/codegen/models/client.py (100%) rename packages/pygen/{core => pygen}/codegen/models/code_model.py (100%) rename packages/pygen/{core => pygen}/codegen/models/combined_type.py (98%) rename packages/pygen/{core => pygen}/codegen/models/constant_type.py (100%) rename packages/pygen/{core => pygen}/codegen/models/credential_types.py (100%) rename packages/pygen/{core => pygen}/codegen/models/dictionary_type.py (100%) rename packages/pygen/{core => pygen}/codegen/models/enum_type.py (100%) rename packages/pygen/{core => pygen}/codegen/models/imports.py (100%) rename packages/pygen/{core => pygen}/codegen/models/list_type.py (100%) rename packages/pygen/{core => pygen}/codegen/models/lro_operation.py (100%) rename packages/pygen/{core => pygen}/codegen/models/lro_paging_operation.py (100%) rename packages/pygen/{core => pygen}/codegen/models/model_type.py (99%) rename packages/pygen/{core => pygen}/codegen/models/operation.py (100%) rename packages/pygen/{core => pygen}/codegen/models/operation_group.py (99%) rename packages/pygen/{core => pygen}/codegen/models/paging_operation.py (100%) rename packages/pygen/{core => pygen}/codegen/models/parameter.py (100%) rename packages/pygen/{core => pygen}/codegen/models/parameter_list.py (100%) rename packages/pygen/{core => pygen}/codegen/models/primitive_types.py (100%) rename packages/pygen/{core => pygen}/codegen/models/property.py (100%) rename packages/pygen/{core => pygen}/codegen/models/request_builder.py (100%) rename packages/pygen/{core => pygen}/codegen/models/request_builder_parameter.py (100%) rename packages/pygen/{core => pygen}/codegen/models/response.py (100%) rename packages/pygen/{core => pygen}/codegen/models/utils.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/__init__.py (99%) rename packages/pygen/{core => pygen}/codegen/serializers/base_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/builder_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/client_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/enum_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/general_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/import_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/metadata_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/model_init_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/model_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/operation_groups_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/operations_init_serializer.py (95%) rename packages/pygen/{core => pygen}/codegen/serializers/parameter_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/patch_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/request_builders_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/sample_serializer.py (99%) rename packages/pygen/{core => pygen}/codegen/serializers/test_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/types_serializer.py (100%) rename packages/pygen/{core => pygen}/codegen/serializers/utils.py (100%) rename packages/pygen/{core => pygen}/codegen/templates/client.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/client_container.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/config.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/config_container.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/conftest.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/enum.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/enum_container.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/init.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/keywords.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/lro_operation.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/lro_paging_operation.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/macros.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/metadata.json.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/model_base.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/model_container.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/model_dpg.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/model_init.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/model_msrest.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/operation.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/operation_group.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/operation_groups_container.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/operation_tools.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/operations_folder_init.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/packaging_templates/LICENSE.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/packaging_templates/MANIFEST.in.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/packaging_templates/README.md.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/packaging_templates/setup.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/paging_operation.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/patch.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/pkgutil_init.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/request_builder.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/request_builders.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/rest_init.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/sample.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/serialization.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/test.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/testpreparer.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/types.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/validation.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/vendor.py.jinja2 (100%) rename packages/pygen/{core => pygen}/codegen/templates/version.py.jinja2 (100%) rename packages/pygen/{core => pygen}/m2r/__init__.py (100%) rename packages/pygen/{core => pygen}/postprocess/__init__.py (100%) rename packages/pygen/{core => pygen}/postprocess/get_all.py (100%) rename packages/pygen/{core => pygen}/postprocess/venvtools.py (100%) rename packages/pygen/{core => pygen}/preprocess/__init__.py (100%) rename packages/pygen/{core => pygen}/preprocess/helpers.py (100%) rename packages/pygen/{core => pygen}/preprocess/python_mappings.py (100%) diff --git a/packages/autorest.python/autorest/__init__.py b/packages/autorest.python/autorest/__init__.py index bb2e1883f73..32bda3bb515 100644 --- a/packages/autorest.python/autorest/__init__.py +++ b/packages/autorest.python/autorest/__init__.py @@ -11,7 +11,7 @@ import yaml from .jsonrpc import AutorestAPI -from pygen.core import ReaderAndWriter, Plugin, YamlUpdatePlugin +from pygen import ReaderAndWriter, Plugin, YamlUpdatePlugin _LOGGER = logging.getLogger(__name__) diff --git a/packages/autorest.python/autorest/black.py b/packages/autorest.python/autorest/black.py index 8dc22c034fb..cc7d8e24d7b 100644 --- a/packages/autorest.python/autorest/black.py +++ b/packages/autorest.python/autorest/black.py @@ -4,7 +4,7 @@ # license information. # -------------------------------------------------------------------------- from typing import Dict, Any -from pygen.core.black import BlackScriptPlugin +from pygen.black import BlackScriptPlugin from . import PluginAutorest class BlackScriptPluginAutorest(BlackScriptPlugin, PluginAutorest): diff --git a/packages/autorest.python/autorest/codegen.py b/packages/autorest.python/autorest/codegen.py index 116d480d820..27dc846afa5 100644 --- a/packages/autorest.python/autorest/codegen.py +++ b/packages/autorest.python/autorest/codegen.py @@ -9,12 +9,12 @@ import yaml from . import ReaderAndWriterAutorest from .jsonrpc import AutorestAPI -from pygen.core.codegen.models import CodeModel -from pygen.core.codegen.serializers import JinjaSerializer +from pygen.codegen.models import CodeModel +from pygen.codegen.serializers import JinjaSerializer from . import PluginAutorest -from pygen.core.codegen import CodeGenerator +from pygen.codegen import CodeGenerator _LOGGER = logging.getLogger(__name__) diff --git a/packages/autorest.python/autorest/m2r.py b/packages/autorest.python/autorest/m2r.py index 82e415c1d42..3613ea3922f 100644 --- a/packages/autorest.python/autorest/m2r.py +++ b/packages/autorest.python/autorest/m2r.py @@ -7,7 +7,7 @@ """ from typing import Any, Dict -from pygen.core.m2r import M2R +from pygen.m2r import M2R from . import YamlUpdatePluginAutorest class M2RAutorest(YamlUpdatePluginAutorest, M2R): diff --git a/packages/autorest.python/autorest/postprocess.py b/packages/autorest.python/autorest/postprocess.py index 78436b200a0..6f70432c435 100644 --- a/packages/autorest.python/autorest/postprocess.py +++ b/packages/autorest.python/autorest/postprocess.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- from typing import Any, Dict -from pygen.core.postprocess import PostProcessPlugin +from pygen.postprocess import PostProcessPlugin from . import PluginAutorest class PostProcessPluginAutorest(PostProcessPlugin, PluginAutorest): diff --git a/packages/autorest.python/autorest/preprocess.py b/packages/autorest.python/autorest/preprocess.py index 3eb4dd725a8..43a9c26ae1b 100644 --- a/packages/autorest.python/autorest/preprocess.py +++ b/packages/autorest.python/autorest/preprocess.py @@ -7,7 +7,7 @@ """ from typing import Dict, Any from . import YamlUpdatePluginAutorest -from pygen.core.preprocess import PreProcessPlugin +from pygen.preprocess import PreProcessPlugin class PreProcessPluginAutorest(YamlUpdatePluginAutorest, PreProcessPlugin): diff --git a/packages/pygen/core/__init__.py b/packages/pygen/pygen/__init__.py similarity index 100% rename from packages/pygen/core/__init__.py rename to packages/pygen/pygen/__init__.py diff --git a/packages/pygen/core/_utils.py b/packages/pygen/pygen/_utils.py similarity index 100% rename from packages/pygen/core/_utils.py rename to packages/pygen/pygen/_utils.py diff --git a/packages/pygen/core/_version.py b/packages/pygen/pygen/_version.py similarity index 100% rename from packages/pygen/core/_version.py rename to packages/pygen/pygen/_version.py diff --git a/packages/pygen/core/black/__init__.py b/packages/pygen/pygen/black/__init__.py similarity index 100% rename from packages/pygen/core/black/__init__.py rename to packages/pygen/pygen/black/__init__.py diff --git a/packages/pygen/core/codegen/__init__.py b/packages/pygen/pygen/codegen/__init__.py similarity index 100% rename from packages/pygen/core/codegen/__init__.py rename to packages/pygen/pygen/codegen/__init__.py diff --git a/packages/pygen/core/codegen/_utils.py b/packages/pygen/pygen/codegen/_utils.py similarity index 100% rename from packages/pygen/core/codegen/_utils.py rename to packages/pygen/pygen/codegen/_utils.py diff --git a/packages/pygen/core/codegen/models/__init__.py b/packages/pygen/pygen/codegen/models/__init__.py similarity index 100% rename from packages/pygen/core/codegen/models/__init__.py rename to packages/pygen/pygen/codegen/models/__init__.py diff --git a/packages/pygen/core/codegen/models/base.py b/packages/pygen/pygen/codegen/models/base.py similarity index 100% rename from packages/pygen/core/codegen/models/base.py rename to packages/pygen/pygen/codegen/models/base.py diff --git a/packages/pygen/core/codegen/models/base_builder.py b/packages/pygen/pygen/codegen/models/base_builder.py similarity index 100% rename from packages/pygen/core/codegen/models/base_builder.py rename to packages/pygen/pygen/codegen/models/base_builder.py diff --git a/packages/pygen/core/codegen/models/client.py b/packages/pygen/pygen/codegen/models/client.py similarity index 100% rename from packages/pygen/core/codegen/models/client.py rename to packages/pygen/pygen/codegen/models/client.py diff --git a/packages/pygen/core/codegen/models/code_model.py b/packages/pygen/pygen/codegen/models/code_model.py similarity index 100% rename from packages/pygen/core/codegen/models/code_model.py rename to packages/pygen/pygen/codegen/models/code_model.py diff --git a/packages/pygen/core/codegen/models/combined_type.py b/packages/pygen/pygen/codegen/models/combined_type.py similarity index 98% rename from packages/pygen/core/codegen/models/combined_type.py rename to packages/pygen/pygen/codegen/models/combined_type.py index a865a69c863..8b6fc261c26 100644 --- a/packages/pygen/core/codegen/models/combined_type.py +++ b/packages/pygen/pygen/codegen/models/combined_type.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- from typing import Any, Dict, List, Optional, TYPE_CHECKING, Type, Tuple, Union import re -from autorest.codegen.models.imports import FileImport, ImportType, TypingSection +from .imports import FileImport, ImportType, TypingSection from .base import BaseType from .model_type import ModelType diff --git a/packages/pygen/core/codegen/models/constant_type.py b/packages/pygen/pygen/codegen/models/constant_type.py similarity index 100% rename from packages/pygen/core/codegen/models/constant_type.py rename to packages/pygen/pygen/codegen/models/constant_type.py diff --git a/packages/pygen/core/codegen/models/credential_types.py b/packages/pygen/pygen/codegen/models/credential_types.py similarity index 100% rename from packages/pygen/core/codegen/models/credential_types.py rename to packages/pygen/pygen/codegen/models/credential_types.py diff --git a/packages/pygen/core/codegen/models/dictionary_type.py b/packages/pygen/pygen/codegen/models/dictionary_type.py similarity index 100% rename from packages/pygen/core/codegen/models/dictionary_type.py rename to packages/pygen/pygen/codegen/models/dictionary_type.py diff --git a/packages/pygen/core/codegen/models/enum_type.py b/packages/pygen/pygen/codegen/models/enum_type.py similarity index 100% rename from packages/pygen/core/codegen/models/enum_type.py rename to packages/pygen/pygen/codegen/models/enum_type.py diff --git a/packages/pygen/core/codegen/models/imports.py b/packages/pygen/pygen/codegen/models/imports.py similarity index 100% rename from packages/pygen/core/codegen/models/imports.py rename to packages/pygen/pygen/codegen/models/imports.py diff --git a/packages/pygen/core/codegen/models/list_type.py b/packages/pygen/pygen/codegen/models/list_type.py similarity index 100% rename from packages/pygen/core/codegen/models/list_type.py rename to packages/pygen/pygen/codegen/models/list_type.py diff --git a/packages/pygen/core/codegen/models/lro_operation.py b/packages/pygen/pygen/codegen/models/lro_operation.py similarity index 100% rename from packages/pygen/core/codegen/models/lro_operation.py rename to packages/pygen/pygen/codegen/models/lro_operation.py diff --git a/packages/pygen/core/codegen/models/lro_paging_operation.py b/packages/pygen/pygen/codegen/models/lro_paging_operation.py similarity index 100% rename from packages/pygen/core/codegen/models/lro_paging_operation.py rename to packages/pygen/pygen/codegen/models/lro_paging_operation.py diff --git a/packages/pygen/core/codegen/models/model_type.py b/packages/pygen/pygen/codegen/models/model_type.py similarity index 99% rename from packages/pygen/core/codegen/models/model_type.py rename to packages/pygen/pygen/codegen/models/model_type.py index f113ac7604b..84bae17609a 100644 --- a/packages/pygen/core/codegen/models/model_type.py +++ b/packages/pygen/pygen/codegen/models/model_type.py @@ -6,7 +6,7 @@ from collections import OrderedDict from typing import Any, Dict, List, Optional, TYPE_CHECKING, cast import sys -from autorest.codegen.models.utils import ( +from .utils import ( add_to_pylint_disable, NAME_LENGTH_LIMIT, ) diff --git a/packages/pygen/core/codegen/models/operation.py b/packages/pygen/pygen/codegen/models/operation.py similarity index 100% rename from packages/pygen/core/codegen/models/operation.py rename to packages/pygen/pygen/codegen/models/operation.py diff --git a/packages/pygen/core/codegen/models/operation_group.py b/packages/pygen/pygen/codegen/models/operation_group.py similarity index 99% rename from packages/pygen/core/codegen/models/operation_group.py rename to packages/pygen/pygen/codegen/models/operation_group.py index bf890b6565e..23b6f4daaba 100644 --- a/packages/pygen/core/codegen/models/operation_group.py +++ b/packages/pygen/pygen/codegen/models/operation_group.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- from typing import Dict, List, Any, TYPE_CHECKING -from autorest.codegen.models.utils import OrderedSet +from .utils import OrderedSet from .base import BaseModel from .operation import get_operation diff --git a/packages/pygen/core/codegen/models/paging_operation.py b/packages/pygen/pygen/codegen/models/paging_operation.py similarity index 100% rename from packages/pygen/core/codegen/models/paging_operation.py rename to packages/pygen/pygen/codegen/models/paging_operation.py diff --git a/packages/pygen/core/codegen/models/parameter.py b/packages/pygen/pygen/codegen/models/parameter.py similarity index 100% rename from packages/pygen/core/codegen/models/parameter.py rename to packages/pygen/pygen/codegen/models/parameter.py diff --git a/packages/pygen/core/codegen/models/parameter_list.py b/packages/pygen/pygen/codegen/models/parameter_list.py similarity index 100% rename from packages/pygen/core/codegen/models/parameter_list.py rename to packages/pygen/pygen/codegen/models/parameter_list.py diff --git a/packages/pygen/core/codegen/models/primitive_types.py b/packages/pygen/pygen/codegen/models/primitive_types.py similarity index 100% rename from packages/pygen/core/codegen/models/primitive_types.py rename to packages/pygen/pygen/codegen/models/primitive_types.py diff --git a/packages/pygen/core/codegen/models/property.py b/packages/pygen/pygen/codegen/models/property.py similarity index 100% rename from packages/pygen/core/codegen/models/property.py rename to packages/pygen/pygen/codegen/models/property.py diff --git a/packages/pygen/core/codegen/models/request_builder.py b/packages/pygen/pygen/codegen/models/request_builder.py similarity index 100% rename from packages/pygen/core/codegen/models/request_builder.py rename to packages/pygen/pygen/codegen/models/request_builder.py diff --git a/packages/pygen/core/codegen/models/request_builder_parameter.py b/packages/pygen/pygen/codegen/models/request_builder_parameter.py similarity index 100% rename from packages/pygen/core/codegen/models/request_builder_parameter.py rename to packages/pygen/pygen/codegen/models/request_builder_parameter.py diff --git a/packages/pygen/core/codegen/models/response.py b/packages/pygen/pygen/codegen/models/response.py similarity index 100% rename from packages/pygen/core/codegen/models/response.py rename to packages/pygen/pygen/codegen/models/response.py diff --git a/packages/pygen/core/codegen/models/utils.py b/packages/pygen/pygen/codegen/models/utils.py similarity index 100% rename from packages/pygen/core/codegen/models/utils.py rename to packages/pygen/pygen/codegen/models/utils.py diff --git a/packages/pygen/core/codegen/serializers/__init__.py b/packages/pygen/pygen/codegen/serializers/__init__.py similarity index 99% rename from packages/pygen/core/codegen/serializers/__init__.py rename to packages/pygen/pygen/codegen/serializers/__init__.py index 4db3a0a07e5..09bc8ecde2d 100644 --- a/packages/pygen/core/codegen/serializers/__init__.py +++ b/packages/pygen/pygen/codegen/serializers/__init__.py @@ -137,7 +137,7 @@ def _serialize_namespace_level(self, env: Environment, namespace_path: Path, cli def serialize(self) -> None: env = Environment( - loader=PackageLoader("core.codegen", "templates"), + loader=PackageLoader("pygen.codegen", "templates"), keep_trailing_newline=True, line_statement_prefix="##", line_comment_prefix="###", @@ -195,7 +195,7 @@ def _serialize_and_write_package_files(self, namespace_path: Path) -> None: root_of_sdk = self._package_root_folder(namespace_path) if self.code_model.options["package_mode"] in VALID_PACKAGE_MODE: env = Environment( - loader=PackageLoader("core.codegen", "templates/packaging_templates"), + loader=PackageLoader("pygen.codegen", "templates/packaging_templates"), undefined=StrictUndefined, ) diff --git a/packages/pygen/core/codegen/serializers/base_serializer.py b/packages/pygen/pygen/codegen/serializers/base_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/base_serializer.py rename to packages/pygen/pygen/codegen/serializers/base_serializer.py diff --git a/packages/pygen/core/codegen/serializers/builder_serializer.py b/packages/pygen/pygen/codegen/serializers/builder_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/builder_serializer.py rename to packages/pygen/pygen/codegen/serializers/builder_serializer.py diff --git a/packages/pygen/core/codegen/serializers/client_serializer.py b/packages/pygen/pygen/codegen/serializers/client_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/client_serializer.py rename to packages/pygen/pygen/codegen/serializers/client_serializer.py diff --git a/packages/pygen/core/codegen/serializers/enum_serializer.py b/packages/pygen/pygen/codegen/serializers/enum_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/enum_serializer.py rename to packages/pygen/pygen/codegen/serializers/enum_serializer.py diff --git a/packages/pygen/core/codegen/serializers/general_serializer.py b/packages/pygen/pygen/codegen/serializers/general_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/general_serializer.py rename to packages/pygen/pygen/codegen/serializers/general_serializer.py diff --git a/packages/pygen/core/codegen/serializers/import_serializer.py b/packages/pygen/pygen/codegen/serializers/import_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/import_serializer.py rename to packages/pygen/pygen/codegen/serializers/import_serializer.py diff --git a/packages/pygen/core/codegen/serializers/metadata_serializer.py b/packages/pygen/pygen/codegen/serializers/metadata_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/metadata_serializer.py rename to packages/pygen/pygen/codegen/serializers/metadata_serializer.py diff --git a/packages/pygen/core/codegen/serializers/model_init_serializer.py b/packages/pygen/pygen/codegen/serializers/model_init_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/model_init_serializer.py rename to packages/pygen/pygen/codegen/serializers/model_init_serializer.py diff --git a/packages/pygen/core/codegen/serializers/model_serializer.py b/packages/pygen/pygen/codegen/serializers/model_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/model_serializer.py rename to packages/pygen/pygen/codegen/serializers/model_serializer.py diff --git a/packages/pygen/core/codegen/serializers/operation_groups_serializer.py b/packages/pygen/pygen/codegen/serializers/operation_groups_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/operation_groups_serializer.py rename to packages/pygen/pygen/codegen/serializers/operation_groups_serializer.py diff --git a/packages/pygen/core/codegen/serializers/operations_init_serializer.py b/packages/pygen/pygen/codegen/serializers/operations_init_serializer.py similarity index 95% rename from packages/pygen/core/codegen/serializers/operations_init_serializer.py rename to packages/pygen/pygen/codegen/serializers/operations_init_serializer.py index 83966ffa899..02232c527c5 100644 --- a/packages/pygen/core/codegen/serializers/operations_init_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/operations_init_serializer.py @@ -6,7 +6,7 @@ from typing import List from jinja2 import Environment -from autorest.codegen.models.operation_group import OperationGroup +from ..models.operation_group import OperationGroup from ..models import CodeModel, Client diff --git a/packages/pygen/core/codegen/serializers/parameter_serializer.py b/packages/pygen/pygen/codegen/serializers/parameter_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/parameter_serializer.py rename to packages/pygen/pygen/codegen/serializers/parameter_serializer.py diff --git a/packages/pygen/core/codegen/serializers/patch_serializer.py b/packages/pygen/pygen/codegen/serializers/patch_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/patch_serializer.py rename to packages/pygen/pygen/codegen/serializers/patch_serializer.py diff --git a/packages/pygen/core/codegen/serializers/request_builders_serializer.py b/packages/pygen/pygen/codegen/serializers/request_builders_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/request_builders_serializer.py rename to packages/pygen/pygen/codegen/serializers/request_builders_serializer.py diff --git a/packages/pygen/core/codegen/serializers/sample_serializer.py b/packages/pygen/pygen/codegen/serializers/sample_serializer.py similarity index 99% rename from packages/pygen/core/codegen/serializers/sample_serializer.py rename to packages/pygen/pygen/codegen/serializers/sample_serializer.py index ccf1e9dfe7a..215471efaf9 100644 --- a/packages/pygen/core/codegen/serializers/sample_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/sample_serializer.py @@ -8,7 +8,7 @@ from typing import Dict, Any, Union, Tuple from jinja2 import Environment -from autorest.codegen.models.operation import OperationBase +from ..models.operation import OperationBase from .import_serializer import FileImportSerializer from .base_serializer import BaseSerializer from ..models import ( diff --git a/packages/pygen/core/codegen/serializers/test_serializer.py b/packages/pygen/pygen/codegen/serializers/test_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/test_serializer.py rename to packages/pygen/pygen/codegen/serializers/test_serializer.py diff --git a/packages/pygen/core/codegen/serializers/types_serializer.py b/packages/pygen/pygen/codegen/serializers/types_serializer.py similarity index 100% rename from packages/pygen/core/codegen/serializers/types_serializer.py rename to packages/pygen/pygen/codegen/serializers/types_serializer.py diff --git a/packages/pygen/core/codegen/serializers/utils.py b/packages/pygen/pygen/codegen/serializers/utils.py similarity index 100% rename from packages/pygen/core/codegen/serializers/utils.py rename to packages/pygen/pygen/codegen/serializers/utils.py diff --git a/packages/pygen/core/codegen/templates/client.py.jinja2 b/packages/pygen/pygen/codegen/templates/client.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/client.py.jinja2 rename to packages/pygen/pygen/codegen/templates/client.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/client_container.py.jinja2 b/packages/pygen/pygen/codegen/templates/client_container.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/client_container.py.jinja2 rename to packages/pygen/pygen/codegen/templates/client_container.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/config.py.jinja2 b/packages/pygen/pygen/codegen/templates/config.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/config.py.jinja2 rename to packages/pygen/pygen/codegen/templates/config.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/config_container.py.jinja2 b/packages/pygen/pygen/codegen/templates/config_container.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/config_container.py.jinja2 rename to packages/pygen/pygen/codegen/templates/config_container.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/conftest.py.jinja2 b/packages/pygen/pygen/codegen/templates/conftest.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/conftest.py.jinja2 rename to packages/pygen/pygen/codegen/templates/conftest.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/enum.py.jinja2 b/packages/pygen/pygen/codegen/templates/enum.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/enum.py.jinja2 rename to packages/pygen/pygen/codegen/templates/enum.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/enum_container.py.jinja2 b/packages/pygen/pygen/codegen/templates/enum_container.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/enum_container.py.jinja2 rename to packages/pygen/pygen/codegen/templates/enum_container.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/init.py.jinja2 b/packages/pygen/pygen/codegen/templates/init.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/init.py.jinja2 rename to packages/pygen/pygen/codegen/templates/init.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/keywords.jinja2 b/packages/pygen/pygen/codegen/templates/keywords.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/keywords.jinja2 rename to packages/pygen/pygen/codegen/templates/keywords.jinja2 diff --git a/packages/pygen/core/codegen/templates/lro_operation.py.jinja2 b/packages/pygen/pygen/codegen/templates/lro_operation.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/lro_operation.py.jinja2 rename to packages/pygen/pygen/codegen/templates/lro_operation.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/lro_paging_operation.py.jinja2 b/packages/pygen/pygen/codegen/templates/lro_paging_operation.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/lro_paging_operation.py.jinja2 rename to packages/pygen/pygen/codegen/templates/lro_paging_operation.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/macros.jinja2 b/packages/pygen/pygen/codegen/templates/macros.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/macros.jinja2 rename to packages/pygen/pygen/codegen/templates/macros.jinja2 diff --git a/packages/pygen/core/codegen/templates/metadata.json.jinja2 b/packages/pygen/pygen/codegen/templates/metadata.json.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/metadata.json.jinja2 rename to packages/pygen/pygen/codegen/templates/metadata.json.jinja2 diff --git a/packages/pygen/core/codegen/templates/model_base.py.jinja2 b/packages/pygen/pygen/codegen/templates/model_base.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/model_base.py.jinja2 rename to packages/pygen/pygen/codegen/templates/model_base.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/model_container.py.jinja2 b/packages/pygen/pygen/codegen/templates/model_container.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/model_container.py.jinja2 rename to packages/pygen/pygen/codegen/templates/model_container.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/model_dpg.py.jinja2 b/packages/pygen/pygen/codegen/templates/model_dpg.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/model_dpg.py.jinja2 rename to packages/pygen/pygen/codegen/templates/model_dpg.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/model_init.py.jinja2 b/packages/pygen/pygen/codegen/templates/model_init.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/model_init.py.jinja2 rename to packages/pygen/pygen/codegen/templates/model_init.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/model_msrest.py.jinja2 b/packages/pygen/pygen/codegen/templates/model_msrest.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/model_msrest.py.jinja2 rename to packages/pygen/pygen/codegen/templates/model_msrest.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/operation.py.jinja2 b/packages/pygen/pygen/codegen/templates/operation.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/operation.py.jinja2 rename to packages/pygen/pygen/codegen/templates/operation.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/operation_group.py.jinja2 b/packages/pygen/pygen/codegen/templates/operation_group.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/operation_group.py.jinja2 rename to packages/pygen/pygen/codegen/templates/operation_group.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/operation_groups_container.py.jinja2 b/packages/pygen/pygen/codegen/templates/operation_groups_container.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/operation_groups_container.py.jinja2 rename to packages/pygen/pygen/codegen/templates/operation_groups_container.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/operation_tools.jinja2 b/packages/pygen/pygen/codegen/templates/operation_tools.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/operation_tools.jinja2 rename to packages/pygen/pygen/codegen/templates/operation_tools.jinja2 diff --git a/packages/pygen/core/codegen/templates/operations_folder_init.py.jinja2 b/packages/pygen/pygen/codegen/templates/operations_folder_init.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/operations_folder_init.py.jinja2 rename to packages/pygen/pygen/codegen/templates/operations_folder_init.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 b/packages/pygen/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 rename to packages/pygen/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 diff --git a/packages/pygen/core/codegen/templates/packaging_templates/LICENSE.jinja2 b/packages/pygen/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/packaging_templates/LICENSE.jinja2 rename to packages/pygen/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 diff --git a/packages/pygen/core/codegen/templates/packaging_templates/MANIFEST.in.jinja2 b/packages/pygen/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/packaging_templates/MANIFEST.in.jinja2 rename to packages/pygen/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 diff --git a/packages/pygen/core/codegen/templates/packaging_templates/README.md.jinja2 b/packages/pygen/pygen/codegen/templates/packaging_templates/README.md.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/packaging_templates/README.md.jinja2 rename to packages/pygen/pygen/codegen/templates/packaging_templates/README.md.jinja2 diff --git a/packages/pygen/core/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 b/packages/pygen/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 rename to packages/pygen/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 diff --git a/packages/pygen/core/codegen/templates/packaging_templates/setup.py.jinja2 b/packages/pygen/pygen/codegen/templates/packaging_templates/setup.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/packaging_templates/setup.py.jinja2 rename to packages/pygen/pygen/codegen/templates/packaging_templates/setup.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/paging_operation.py.jinja2 b/packages/pygen/pygen/codegen/templates/paging_operation.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/paging_operation.py.jinja2 rename to packages/pygen/pygen/codegen/templates/paging_operation.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/patch.py.jinja2 b/packages/pygen/pygen/codegen/templates/patch.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/patch.py.jinja2 rename to packages/pygen/pygen/codegen/templates/patch.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/pkgutil_init.py.jinja2 b/packages/pygen/pygen/codegen/templates/pkgutil_init.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/pkgutil_init.py.jinja2 rename to packages/pygen/pygen/codegen/templates/pkgutil_init.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/request_builder.py.jinja2 b/packages/pygen/pygen/codegen/templates/request_builder.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/request_builder.py.jinja2 rename to packages/pygen/pygen/codegen/templates/request_builder.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/request_builders.py.jinja2 b/packages/pygen/pygen/codegen/templates/request_builders.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/request_builders.py.jinja2 rename to packages/pygen/pygen/codegen/templates/request_builders.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/rest_init.py.jinja2 b/packages/pygen/pygen/codegen/templates/rest_init.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/rest_init.py.jinja2 rename to packages/pygen/pygen/codegen/templates/rest_init.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/sample.py.jinja2 b/packages/pygen/pygen/codegen/templates/sample.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/sample.py.jinja2 rename to packages/pygen/pygen/codegen/templates/sample.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/serialization.py.jinja2 b/packages/pygen/pygen/codegen/templates/serialization.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/serialization.py.jinja2 rename to packages/pygen/pygen/codegen/templates/serialization.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/test.py.jinja2 b/packages/pygen/pygen/codegen/templates/test.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/test.py.jinja2 rename to packages/pygen/pygen/codegen/templates/test.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/testpreparer.py.jinja2 b/packages/pygen/pygen/codegen/templates/testpreparer.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/testpreparer.py.jinja2 rename to packages/pygen/pygen/codegen/templates/testpreparer.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/types.py.jinja2 b/packages/pygen/pygen/codegen/templates/types.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/types.py.jinja2 rename to packages/pygen/pygen/codegen/templates/types.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/validation.py.jinja2 b/packages/pygen/pygen/codegen/templates/validation.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/validation.py.jinja2 rename to packages/pygen/pygen/codegen/templates/validation.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/vendor.py.jinja2 b/packages/pygen/pygen/codegen/templates/vendor.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/vendor.py.jinja2 rename to packages/pygen/pygen/codegen/templates/vendor.py.jinja2 diff --git a/packages/pygen/core/codegen/templates/version.py.jinja2 b/packages/pygen/pygen/codegen/templates/version.py.jinja2 similarity index 100% rename from packages/pygen/core/codegen/templates/version.py.jinja2 rename to packages/pygen/pygen/codegen/templates/version.py.jinja2 diff --git a/packages/pygen/core/m2r/__init__.py b/packages/pygen/pygen/m2r/__init__.py similarity index 100% rename from packages/pygen/core/m2r/__init__.py rename to packages/pygen/pygen/m2r/__init__.py diff --git a/packages/pygen/core/postprocess/__init__.py b/packages/pygen/pygen/postprocess/__init__.py similarity index 100% rename from packages/pygen/core/postprocess/__init__.py rename to packages/pygen/pygen/postprocess/__init__.py diff --git a/packages/pygen/core/postprocess/get_all.py b/packages/pygen/pygen/postprocess/get_all.py similarity index 100% rename from packages/pygen/core/postprocess/get_all.py rename to packages/pygen/pygen/postprocess/get_all.py diff --git a/packages/pygen/core/postprocess/venvtools.py b/packages/pygen/pygen/postprocess/venvtools.py similarity index 100% rename from packages/pygen/core/postprocess/venvtools.py rename to packages/pygen/pygen/postprocess/venvtools.py diff --git a/packages/pygen/core/preprocess/__init__.py b/packages/pygen/pygen/preprocess/__init__.py similarity index 100% rename from packages/pygen/core/preprocess/__init__.py rename to packages/pygen/pygen/preprocess/__init__.py diff --git a/packages/pygen/core/preprocess/helpers.py b/packages/pygen/pygen/preprocess/helpers.py similarity index 100% rename from packages/pygen/core/preprocess/helpers.py rename to packages/pygen/pygen/preprocess/helpers.py diff --git a/packages/pygen/core/preprocess/python_mappings.py b/packages/pygen/pygen/preprocess/python_mappings.py similarity index 100% rename from packages/pygen/core/preprocess/python_mappings.py rename to packages/pygen/pygen/preprocess/python_mappings.py diff --git a/packages/pygen/setup.py b/packages/pygen/setup.py index c74fbc0c88d..79bba224efe 100644 --- a/packages/pygen/setup.py +++ b/packages/pygen/setup.py @@ -14,14 +14,14 @@ # Version extraction inspired from 'requests' -with open(os.path.join("core", "_version.py"), "r") as fd: +with open(os.path.join("pygen", "_version.py"), "r") as fd: version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) if not version: raise RuntimeError("Cannot find version information") setup( - name="pygencore", + name="pygen", version=version, description="Core Library for Python Generation", long_description=open("README.md", "r").read(), From 213bdd7386fbb5bee716e96c7f098cfeae8cdde9 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 12:53:10 -0400 Subject: [PATCH 03/75] black --- packages/autorest.python/autorest/__init__.py | 5 ++++- packages/autorest.python/autorest/black.py | 1 + packages/autorest.python/autorest/codegen.py | 10 +++++----- packages/autorest.python/autorest/m2r.py | 1 + packages/autorest.python/autorest/postprocess.py | 1 + packages/autorest.python/autorest/preprocess.py | 2 +- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/autorest.python/autorest/__init__.py b/packages/autorest.python/autorest/__init__.py index 32bda3bb515..e5682d9aad0 100644 --- a/packages/autorest.python/autorest/__init__.py +++ b/packages/autorest.python/autorest/__init__.py @@ -10,11 +10,13 @@ import yaml -from .jsonrpc import AutorestAPI from pygen import ReaderAndWriter, Plugin, YamlUpdatePlugin +from .jsonrpc import AutorestAPI + _LOGGER = logging.getLogger(__name__) + class ReaderAndWriterAutorest(ReaderAndWriter): def __init__(self, *, output_folder: Union[str, Path], autorestapi: AutorestAPI) -> None: super().__init__(output_folder=output_folder) @@ -41,6 +43,7 @@ def __init__(self, autorestapi: AutorestAPI, *, output_folder: Union[str, Path]) def get_options(self) -> Dict[str, Any]: """Get the options bag using the AutorestAPI that we send to the parent plugin""" + class YamlUpdatePluginAutorest(YamlUpdatePlugin, PluginAutorest): # pylint: disable=abstract-method def get_yaml(self) -> Dict[str, Any]: return yaml.safe_load(self.read_file("code-model-v4-no-tags.yaml")) diff --git a/packages/autorest.python/autorest/black.py b/packages/autorest.python/autorest/black.py index cc7d8e24d7b..c77cfcf0513 100644 --- a/packages/autorest.python/autorest/black.py +++ b/packages/autorest.python/autorest/black.py @@ -7,6 +7,7 @@ from pygen.black import BlackScriptPlugin from . import PluginAutorest + class BlackScriptPluginAutorest(BlackScriptPlugin, PluginAutorest): def get_options(self) -> Dict[str, Any]: return {"output_folder": self._autorestapi.get_value("outputFolderUri")} diff --git a/packages/autorest.python/autorest/codegen.py b/packages/autorest.python/autorest/codegen.py index 27dc846afa5..fdbf4bd1030 100644 --- a/packages/autorest.python/autorest/codegen.py +++ b/packages/autorest.python/autorest/codegen.py @@ -7,18 +7,19 @@ from typing import Dict, Any, Union from pathlib import Path import yaml -from . import ReaderAndWriterAutorest -from .jsonrpc import AutorestAPI + +from pygen.codegen import CodeGenerator from pygen.codegen.models import CodeModel from pygen.codegen.serializers import JinjaSerializer - +from . import ReaderAndWriterAutorest +from .jsonrpc import AutorestAPI from . import PluginAutorest -from pygen.codegen import CodeGenerator _LOGGER = logging.getLogger(__name__) + class JinjaSerializerAutorest(JinjaSerializer, ReaderAndWriterAutorest): def __init__( self, @@ -36,7 +37,6 @@ def __init__( ) - class CodeGeneratorAutorest(CodeGenerator, PluginAutorest): def get_options(self) -> Dict[str, Any]: if self._autorestapi.get_boolean_value("python3-only") is False: diff --git a/packages/autorest.python/autorest/m2r.py b/packages/autorest.python/autorest/m2r.py index 3613ea3922f..418a9ca2899 100644 --- a/packages/autorest.python/autorest/m2r.py +++ b/packages/autorest.python/autorest/m2r.py @@ -10,6 +10,7 @@ from pygen.m2r import M2R from . import YamlUpdatePluginAutorest + class M2RAutorest(YamlUpdatePluginAutorest, M2R): def get_options(self) -> Dict[str, Any]: return {} diff --git a/packages/autorest.python/autorest/postprocess.py b/packages/autorest.python/autorest/postprocess.py index 6f70432c435..5aa9f951de3 100644 --- a/packages/autorest.python/autorest/postprocess.py +++ b/packages/autorest.python/autorest/postprocess.py @@ -8,6 +8,7 @@ from pygen.postprocess import PostProcessPlugin from . import PluginAutorest + class PostProcessPluginAutorest(PostProcessPlugin, PluginAutorest): def get_options(self) -> Dict[str, Any]: return {"outputFolderUri": self._autorestapi.get_value("outputFolderUri")} diff --git a/packages/autorest.python/autorest/preprocess.py b/packages/autorest.python/autorest/preprocess.py index 43a9c26ae1b..ebe66c56e3e 100644 --- a/packages/autorest.python/autorest/preprocess.py +++ b/packages/autorest.python/autorest/preprocess.py @@ -6,8 +6,8 @@ """The preprocessing autorest plugin. """ from typing import Dict, Any -from . import YamlUpdatePluginAutorest from pygen.preprocess import PreProcessPlugin +from . import YamlUpdatePluginAutorest class PreProcessPluginAutorest(YamlUpdatePluginAutorest, PreProcessPlugin): From 13db1bbe093f220bfa5402fd684c7767dda50090 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 13:02:25 -0400 Subject: [PATCH 04/75] pylint and mypy --- packages/autorest.python/autorest/codegen.py | 2 +- packages/autorest.python/mypy.ini | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/autorest.python/autorest/codegen.py b/packages/autorest.python/autorest/codegen.py index fdbf4bd1030..3240f7e7d49 100644 --- a/packages/autorest.python/autorest/codegen.py +++ b/packages/autorest.python/autorest/codegen.py @@ -29,7 +29,7 @@ def __init__( output_folder: Union[str, Path], **kwargs: Any, ) -> None: - super().__init__( + super().__init__( # type: ignore autorestapi=autorestapi, code_model=code_model, output_folder=output_folder, diff --git a/packages/autorest.python/mypy.ini b/packages/autorest.python/mypy.ini index 611cc27aa2a..76618ab1a44 100644 --- a/packages/autorest.python/mypy.ini +++ b/packages/autorest.python/mypy.ini @@ -30,3 +30,6 @@ ignore_missing_imports = True [mypy-*._patch] ignore_missing_imports = True + +[mypy-pygen.*] +ignore_missing_imports = True From fa03ccd5dc5e75514d8e0729a5e9827e61cc7390 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 13:12:48 -0400 Subject: [PATCH 05/75] add editable install of pygen into autorest.python --- packages/autorest.python/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index 17ae43bada8..d7e96763277 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -11,3 +11,4 @@ platformdirs==3.2.0 PyYAML==6.0.1 tomli==2.0.1 setuptools==69.2.0 +-e ../pygen From c0367d45df2b0915a4d42c102459bb49d051898d Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 14:04:41 -0400 Subject: [PATCH 06/75] move files over to pygen --- packages/autorest.python/package.json | 11 ++--- packages/pygen/LICENSE | 21 +++++++++ packages/pygen/dev_requirements.txt | 5 +++ .../{autorest.python => pygen}/install.py | 0 packages/pygen/package.json | 44 +++++++++++++++++++ .../{autorest.python => pygen}/prepare.py | 0 packages/pygen/requirements.txt | 13 ++++++ packages/pygen/run-python3.js | 22 ++++++++++ .../run_cadl.py => pygen/run_tsp.py} | 0 packages/{autorest.python => pygen}/start.py | 0 .../{autorest.python => pygen}/venvtools.py | 0 pnpm-lock.yaml | 25 ++++++++--- 12 files changed, 130 insertions(+), 11 deletions(-) create mode 100644 packages/pygen/LICENSE create mode 100644 packages/pygen/dev_requirements.txt rename packages/{autorest.python => pygen}/install.py (100%) create mode 100644 packages/pygen/package.json rename packages/{autorest.python => pygen}/prepare.py (100%) create mode 100644 packages/pygen/requirements.txt create mode 100644 packages/pygen/run-python3.js rename packages/{autorest.python/run_cadl.py => pygen/run_tsp.py} (100%) rename packages/{autorest.python => pygen}/start.py (100%) rename packages/{autorest.python => pygen}/venvtools.py (100%) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 330f4e1a86e..309a29a4cd9 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -3,10 +3,10 @@ "version": "6.13.17", "description": "The Python extension for generators in AutoRest.", "scripts": { - "prepare": "node run-python3.js prepare.py", - "start": "node run-python3.js start.py", - "install": "node run-python3.js install.py", - "debug": "node run-python3.js start.py --debug" + "prepare": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/prepare.py", + "start": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/start.py", + "install": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/install.py", + "debug": "node r./node_modules/@azure-tools/python-client-generator-core/un-python3.js ./node_modules/@azure-tools/python-client-generator-core/start.py --debug" }, "main": "index.js", "repository": { @@ -25,7 +25,8 @@ }, "homepage": "https://github.com/Azure/autorest.python/blob/main/README.md", "dependencies": { - "@autorest/system-requirements": "~1.0.2" + "@autorest/system-requirements": "~1.0.2", + "@azure-tools/python-client-generator-core": "workspace:^" }, "devDependencies": { "@microsoft.azure/autorest.testserver": "^3.3.46", diff --git a/packages/pygen/LICENSE b/packages/pygen/LICENSE new file mode 100644 index 00000000000..21071075c24 --- /dev/null +++ b/packages/pygen/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/packages/pygen/dev_requirements.txt b/packages/pygen/dev_requirements.txt new file mode 100644 index 00000000000..d458756b7d8 --- /dev/null +++ b/packages/pygen/dev_requirements.txt @@ -0,0 +1,5 @@ +-e . +-r ../../eng/requirements.txt +-r ../../eng/dev_requirements.txt +ptvsd==4.3.2 +types-PyYAML==6.0.12.8 diff --git a/packages/autorest.python/install.py b/packages/pygen/install.py similarity index 100% rename from packages/autorest.python/install.py rename to packages/pygen/install.py diff --git a/packages/pygen/package.json b/packages/pygen/package.json new file mode 100644 index 00000000000..ca5016b3898 --- /dev/null +++ b/packages/pygen/package.json @@ -0,0 +1,44 @@ +{ + "name": "@azure-tools/python-client-generator-core", + "version": "0.1.0", + "description": "Core generation library for Python clients", + "scripts": { + "prepare": "node run-python3.js prepare.py", + "start": "node run-python3.js start.py", + "install": "node run-python3.js install.py", + "debug": "node run-python3.js start.py --debug" + }, + "main": "index.js", + "repository": { + "type": "git", + "url": "https://github.com/Azure/autorest.python/tree/main" + }, + "readme": "https://github.com/Azure/autorest.python/blob/main/README.md", + "keywords": [ + "python" + ], + "author": "Microsoft Corporation", + "license": "MIT", + "bugs": { + "url": "https://github.com/Azure/autorest.python/issues" + }, + "homepage": "https://github.com/Azure/autorest.python/blob/main/README.md", + "dependencies": { + "@autorest/system-requirements": "~1.0.2" + }, + "devDependencies": { + "typescript": "~5.1.3" + }, + "files": [ + "pygen/**/*.py", + "pygen/**/*.jinja2", + "setup.py", + "install.py", + "prepare.py", + "start.py", + "venvtools.py", + "run-python3.js", + "requirements.txt", + "run_tsp.py" + ] +} diff --git a/packages/autorest.python/prepare.py b/packages/pygen/prepare.py similarity index 100% rename from packages/autorest.python/prepare.py rename to packages/pygen/prepare.py diff --git a/packages/pygen/requirements.txt b/packages/pygen/requirements.txt new file mode 100644 index 00000000000..17ae43bada8 --- /dev/null +++ b/packages/pygen/requirements.txt @@ -0,0 +1,13 @@ +black==24.4.0 +click==8.1.3 +docutils==0.19 +Jinja2==3.1.4 +json-rpc==1.14.0 +m2r2==0.3.3 +MarkupSafe==2.1.2 +mistune==0.8.4 +pathspec==0.11.1 +platformdirs==3.2.0 +PyYAML==6.0.1 +tomli==2.0.1 +setuptools==69.2.0 diff --git a/packages/pygen/run-python3.js b/packages/pygen/run-python3.js new file mode 100644 index 00000000000..020cbce0a0f --- /dev/null +++ b/packages/pygen/run-python3.js @@ -0,0 +1,22 @@ +// This script wraps logic in @azure-tools/extension to resolve +// the path to Python 3 so that a Python script file can be run +// from an npm script in package.json. It uses the same Python 3 +// path resolution algorithm as AutoRest so that the behavior +// is fully consistent (and also supports AUTOREST_PYTHON_EXE). +// +// Invoke it like so: "node run-python3.js script.py" + +const cp = require("child_process"); +const extension = require("@autorest/system-requirements"); + +async function runPython3(scriptName, ...args) { + const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE" }); + cp.execSync(command.join(" "), { + stdio: [0, 1, 2] + }); +} + +runPython3(...process.argv.slice(2)).catch(err => { + console.error(err.toString()); + process.exit(1); +}); diff --git a/packages/autorest.python/run_cadl.py b/packages/pygen/run_tsp.py similarity index 100% rename from packages/autorest.python/run_cadl.py rename to packages/pygen/run_tsp.py diff --git a/packages/autorest.python/start.py b/packages/pygen/start.py similarity index 100% rename from packages/autorest.python/start.py rename to packages/pygen/start.py diff --git a/packages/autorest.python/venvtools.py b/packages/pygen/venvtools.py similarity index 100% rename from packages/autorest.python/venvtools.py rename to packages/pygen/venvtools.py diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fad4c90a4c3..03b9d39c9be 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,5 +1,9 @@ lockfileVersion: '6.0' +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + importers: .: @@ -43,6 +47,9 @@ importers: '@autorest/system-requirements': specifier: ~1.0.2 version: 1.0.2 + '@azure-tools/python-client-generator-core': + specifier: workspace:^ + version: link:../pygen devDependencies: '@microsoft.azure/autorest.testserver': specifier: ^3.3.46 @@ -51,6 +58,16 @@ importers: specifier: ~5.1.3 version: 5.1.3 + packages/pygen: + dependencies: + '@autorest/system-requirements': + specifier: ~1.0.2 + version: 1.0.2 + devDependencies: + typescript: + specifier: ~5.1.3 + version: 5.1.6 + packages/typespec-python: dependencies: '@autorest/python': @@ -448,7 +465,7 @@ packages: '@azure/core-auth': 1.5.0 '@azure/core-tracing': 1.0.0-preview.13 '@azure/core-util': 1.4.0 - '@azure/logger': 1.0.4 + '@azure/logger': 1.1.1 '@types/node-fetch': 2.6.4 '@types/tunnel': 0.0.3 form-data: 4.0.0 @@ -490,7 +507,7 @@ packages: dependencies: '@azure/abort-controller': 1.1.0 '@azure/core-util': 1.4.0 - '@azure/logger': 1.0.4 + '@azure/logger': 1.1.1 tslib: 2.6.2 dev: true @@ -6733,7 +6750,3 @@ packages: /zod@3.22.4: resolution: {integrity: sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==} dev: true - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false From 93ea4937b4de8d331878228cec9703a429644fa3 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 14:09:07 -0400 Subject: [PATCH 07/75] basic generaiton possible from autorest with script files moved to pygen --- packages/autorest.python/package.json | 2 +- packages/{pygen => autorest.python}/start.py | 2 +- packages/pygen/install.py | 2 +- packages/pygen/package.json | 1 - packages/pygen/prepare.py | 2 +- packages/pygen/pygen/black/__init__.py | 2 +- packages/pygen/pygen/codegen/__init__.py | 2 +- packages/pygen/pygen/codegen/serializers/__init__.py | 2 +- .../pygen/pygen/codegen/serializers/builder_serializer.py | 2 +- packages/pygen/pygen/codegen/serializers/client_serializer.py | 2 +- packages/pygen/pygen/codegen/serializers/sample_serializer.py | 2 +- packages/pygen/pygen/m2r/__init__.py | 2 +- packages/pygen/pygen/preprocess/__init__.py | 4 ++-- packages/pygen/pygen/{_utils.py => utils/__init__.py} | 0 packages/pygen/{ => pygen/utils}/venvtools.py | 0 packages/pygen/run_tsp.py | 2 +- 16 files changed, 14 insertions(+), 15 deletions(-) rename packages/{pygen => autorest.python}/start.py (95%) rename packages/pygen/pygen/{_utils.py => utils/__init__.py} (100%) rename packages/pygen/{ => pygen/utils}/venvtools.py (100%) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 309a29a4cd9..c651b28476e 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -4,7 +4,7 @@ "description": "The Python extension for generators in AutoRest.", "scripts": { "prepare": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/prepare.py", - "start": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/start.py", + "start": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js start.py", "install": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/install.py", "debug": "node r./node_modules/@azure-tools/python-client-generator-core/un-python3.js ./node_modules/@azure-tools/python-client-generator-core/start.py --debug" }, diff --git a/packages/pygen/start.py b/packages/autorest.python/start.py similarity index 95% rename from packages/pygen/start.py rename to packages/autorest.python/start.py index fdf57593653..fd57a95b087 100644 --- a/packages/pygen/start.py +++ b/packages/autorest.python/start.py @@ -13,7 +13,7 @@ from pathlib import Path import venv -from venvtools import python_run +from pygen.utils.venvtools import python_run _ROOT_DIR = Path(__file__).parent diff --git a/packages/pygen/install.py b/packages/pygen/install.py index 14ae38dd278..9b7009c3d66 100644 --- a/packages/pygen/install.py +++ b/packages/pygen/install.py @@ -26,7 +26,7 @@ import subprocess from pathlib import Path -from venvtools import ExtendedEnvBuilder, python_run +from pygen.utils.venvtools import ExtendedEnvBuilder, python_run _ROOT_DIR = Path(__file__).parent diff --git a/packages/pygen/package.json b/packages/pygen/package.json index ca5016b3898..f49bd5706b4 100644 --- a/packages/pygen/package.json +++ b/packages/pygen/package.json @@ -35,7 +35,6 @@ "setup.py", "install.py", "prepare.py", - "start.py", "venvtools.py", "run-python3.js", "requirements.txt", diff --git a/packages/pygen/prepare.py b/packages/pygen/prepare.py index 84b592e30ee..b1b554bedfe 100644 --- a/packages/pygen/prepare.py +++ b/packages/pygen/prepare.py @@ -13,7 +13,7 @@ from pathlib import Path import venv -from venvtools import python_run +from pygen.utils.venvtools import python_run _ROOT_DIR = Path(__file__).parent diff --git a/packages/pygen/pygen/black/__init__.py b/packages/pygen/pygen/black/__init__.py index a17a536485a..82033d53b17 100644 --- a/packages/pygen/pygen/black/__init__.py +++ b/packages/pygen/pygen/black/__init__.py @@ -11,7 +11,7 @@ from black.report import NothingChanged from .. import Plugin -from .._utils import parse_args +from ..utils import parse_args _LOGGER = logging.getLogger("blib2to3") diff --git a/packages/pygen/pygen/codegen/__init__.py b/packages/pygen/pygen/codegen/__init__.py index c5a30203843..ea4716f86e6 100644 --- a/packages/pygen/pygen/codegen/__init__.py +++ b/packages/pygen/pygen/codegen/__init__.py @@ -10,7 +10,7 @@ from .. import Plugin -from .._utils import parse_args +from ..utils import parse_args from .models.code_model import CodeModel from .serializers import JinjaSerializer from ._utils import DEFAULT_HEADER_TEXT, VALID_PACKAGE_MODE, TYPESPEC_PACKAGE_MODE diff --git a/packages/pygen/pygen/codegen/serializers/__init__.py b/packages/pygen/pygen/codegen/serializers/__init__.py index 09bc8ecde2d..ee3d27ff8ad 100644 --- a/packages/pygen/pygen/codegen/serializers/__init__.py +++ b/packages/pygen/pygen/codegen/serializers/__init__.py @@ -28,7 +28,7 @@ from .sample_serializer import SampleSerializer from .test_serializer import TestSerializer, TestGeneralSerializer from .types_serializer import TypesSerializer -from ..._utils import to_snake_case +from ...utils import to_snake_case from .._utils import VALID_PACKAGE_MODE from .utils import ( extract_sample_name, diff --git a/packages/pygen/pygen/codegen/serializers/builder_serializer.py b/packages/pygen/pygen/codegen/serializers/builder_serializer.py index 42df4ae0ff9..6207714e7d9 100644 --- a/packages/pygen/pygen/codegen/serializers/builder_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/builder_serializer.py @@ -36,7 +36,7 @@ from .parameter_serializer import ParameterSerializer, PopKwargType from ..models.parameter_list import ParameterType from . import utils -from ..._utils import JSON_REGEXP +from ...utils import JSON_REGEXP T = TypeVar("T") OrderedSet = Dict[T, None] diff --git a/packages/pygen/pygen/codegen/serializers/client_serializer.py b/packages/pygen/pygen/codegen/serializers/client_serializer.py index ce8dfd87964..28041370efb 100644 --- a/packages/pygen/pygen/codegen/serializers/client_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/client_serializer.py @@ -8,7 +8,7 @@ from . import utils from ..models import Client, ParameterMethodLocation from .parameter_serializer import ParameterSerializer, PopKwargType -from ..._utils import build_policies +from ...utils import build_policies class ClientSerializer: diff --git a/packages/pygen/pygen/codegen/serializers/sample_serializer.py b/packages/pygen/pygen/codegen/serializers/sample_serializer.py index 215471efaf9..5694804c687 100644 --- a/packages/pygen/pygen/codegen/serializers/sample_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/sample_serializer.py @@ -22,7 +22,7 @@ FileImport, ) from .utils import get_namespace_config, get_namespace_from_package_name -from ..._utils import to_snake_case +from ...utils import to_snake_case _LOGGER = logging.getLogger(__name__) diff --git a/packages/pygen/pygen/m2r/__init__.py b/packages/pygen/pygen/m2r/__init__.py index 450a0fadf63..d6e78871787 100644 --- a/packages/pygen/pygen/m2r/__init__.py +++ b/packages/pygen/pygen/m2r/__init__.py @@ -11,7 +11,7 @@ import m2r2 from .. import YamlUpdatePlugin -from .._utils import parse_args +from ..utils import parse_args _LOGGER = logging.getLogger(__name__) diff --git a/packages/pygen/pygen/preprocess/__init__.py b/packages/pygen/pygen/preprocess/__init__.py index 6cea93cc121..81cc9134aac 100644 --- a/packages/pygen/pygen/preprocess/__init__.py +++ b/packages/pygen/pygen/preprocess/__init__.py @@ -8,7 +8,7 @@ import copy from typing import Callable, Dict, Any, List, Optional -from .._utils import to_snake_case +from ..utils import to_snake_case from .helpers import ( add_redefined_builtin_info, pad_builtin_namespaces, @@ -17,7 +17,7 @@ from .python_mappings import CADL_RESERVED_WORDS, RESERVED_WORDS, PadType from .. import YamlUpdatePlugin -from .._utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES +from ..utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES def update_overload_section( diff --git a/packages/pygen/pygen/_utils.py b/packages/pygen/pygen/utils/__init__.py similarity index 100% rename from packages/pygen/pygen/_utils.py rename to packages/pygen/pygen/utils/__init__.py diff --git a/packages/pygen/venvtools.py b/packages/pygen/pygen/utils/venvtools.py similarity index 100% rename from packages/pygen/venvtools.py rename to packages/pygen/pygen/utils/venvtools.py diff --git a/packages/pygen/run_tsp.py b/packages/pygen/run_tsp.py index acc4404a90d..eb75ca2d021 100644 --- a/packages/pygen/run_tsp.py +++ b/packages/pygen/run_tsp.py @@ -7,7 +7,7 @@ import venv import logging from pathlib import Path -from venvtools import python_run +from pygen.utils.venvtools import python_run _ROOT_DIR = Path(__file__).parent From d2a84cace75249ce281ad0f732d79d7865c493d5 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 14:29:32 -0400 Subject: [PATCH 08/75] autorest generation working from scripts in pygen file --- packages/autorest.python/start.py | 4 +- .../utils => autorest.python}/venvtools.py | 0 packages/pygen/install.py | 2 +- packages/pygen/prepare.py | 8 +- .../pygen/{utils/__init__.py => _utils.py} | 0 packages/pygen/pygen/black/__init__.py | 2 +- packages/pygen/pygen/codegen/__init__.py | 2 +- .../pygen/codegen/serializers/__init__.py | 2 +- .../codegen/serializers/builder_serializer.py | 2 +- .../codegen/serializers/client_serializer.py | 2 +- .../codegen/serializers/sample_serializer.py | 2 +- packages/pygen/pygen/m2r/__init__.py | 2 +- packages/pygen/pygen/preprocess/__init__.py | 4 +- packages/pygen/run_tsp.py | 2 +- packages/pygen/venvtools.py | 81 +++++++++++++++++++ 15 files changed, 99 insertions(+), 16 deletions(-) rename packages/{pygen/pygen/utils => autorest.python}/venvtools.py (100%) rename packages/pygen/pygen/{utils/__init__.py => _utils.py} (100%) create mode 100644 packages/pygen/venvtools.py diff --git a/packages/autorest.python/start.py b/packages/autorest.python/start.py index fd57a95b087..fca22e73286 100644 --- a/packages/autorest.python/start.py +++ b/packages/autorest.python/start.py @@ -13,13 +13,13 @@ from pathlib import Path import venv -from pygen.utils.venvtools import python_run +from venvtools import python_run _ROOT_DIR = Path(__file__).parent def main(): - venv_path = _ROOT_DIR / "venv" + venv_path = _ROOT_DIR / "node_modules/@azure-tools/python-client-generator-core/venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/packages/pygen/pygen/utils/venvtools.py b/packages/autorest.python/venvtools.py similarity index 100% rename from packages/pygen/pygen/utils/venvtools.py rename to packages/autorest.python/venvtools.py diff --git a/packages/pygen/install.py b/packages/pygen/install.py index 9b7009c3d66..14ae38dd278 100644 --- a/packages/pygen/install.py +++ b/packages/pygen/install.py @@ -26,7 +26,7 @@ import subprocess from pathlib import Path -from pygen.utils.venvtools import ExtendedEnvBuilder, python_run +from venvtools import ExtendedEnvBuilder, python_run _ROOT_DIR = Path(__file__).parent diff --git a/packages/pygen/prepare.py b/packages/pygen/prepare.py index b1b554bedfe..e407cc04c46 100644 --- a/packages/pygen/prepare.py +++ b/packages/pygen/prepare.py @@ -13,7 +13,7 @@ from pathlib import Path import venv -from pygen.utils.venvtools import python_run +from venvtools import python_run _ROOT_DIR = Path(__file__).parent @@ -27,8 +27,10 @@ def main(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) requirements_path = _ROOT_DIR / "dev_requirements.txt" - - python_run(venv_context, "pip", ["install", "-r", str(requirements_path)]) + try: + python_run(venv_context, "pip", ["install", "-r", str(requirements_path)]) + except FileNotFoundError as e: + raise ValueError(e.filename) if __name__ == "__main__": diff --git a/packages/pygen/pygen/utils/__init__.py b/packages/pygen/pygen/_utils.py similarity index 100% rename from packages/pygen/pygen/utils/__init__.py rename to packages/pygen/pygen/_utils.py diff --git a/packages/pygen/pygen/black/__init__.py b/packages/pygen/pygen/black/__init__.py index 82033d53b17..a17a536485a 100644 --- a/packages/pygen/pygen/black/__init__.py +++ b/packages/pygen/pygen/black/__init__.py @@ -11,7 +11,7 @@ from black.report import NothingChanged from .. import Plugin -from ..utils import parse_args +from .._utils import parse_args _LOGGER = logging.getLogger("blib2to3") diff --git a/packages/pygen/pygen/codegen/__init__.py b/packages/pygen/pygen/codegen/__init__.py index ea4716f86e6..c5a30203843 100644 --- a/packages/pygen/pygen/codegen/__init__.py +++ b/packages/pygen/pygen/codegen/__init__.py @@ -10,7 +10,7 @@ from .. import Plugin -from ..utils import parse_args +from .._utils import parse_args from .models.code_model import CodeModel from .serializers import JinjaSerializer from ._utils import DEFAULT_HEADER_TEXT, VALID_PACKAGE_MODE, TYPESPEC_PACKAGE_MODE diff --git a/packages/pygen/pygen/codegen/serializers/__init__.py b/packages/pygen/pygen/codegen/serializers/__init__.py index ee3d27ff8ad..09bc8ecde2d 100644 --- a/packages/pygen/pygen/codegen/serializers/__init__.py +++ b/packages/pygen/pygen/codegen/serializers/__init__.py @@ -28,7 +28,7 @@ from .sample_serializer import SampleSerializer from .test_serializer import TestSerializer, TestGeneralSerializer from .types_serializer import TypesSerializer -from ...utils import to_snake_case +from ..._utils import to_snake_case from .._utils import VALID_PACKAGE_MODE from .utils import ( extract_sample_name, diff --git a/packages/pygen/pygen/codegen/serializers/builder_serializer.py b/packages/pygen/pygen/codegen/serializers/builder_serializer.py index 6207714e7d9..42df4ae0ff9 100644 --- a/packages/pygen/pygen/codegen/serializers/builder_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/builder_serializer.py @@ -36,7 +36,7 @@ from .parameter_serializer import ParameterSerializer, PopKwargType from ..models.parameter_list import ParameterType from . import utils -from ...utils import JSON_REGEXP +from ..._utils import JSON_REGEXP T = TypeVar("T") OrderedSet = Dict[T, None] diff --git a/packages/pygen/pygen/codegen/serializers/client_serializer.py b/packages/pygen/pygen/codegen/serializers/client_serializer.py index 28041370efb..ce8dfd87964 100644 --- a/packages/pygen/pygen/codegen/serializers/client_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/client_serializer.py @@ -8,7 +8,7 @@ from . import utils from ..models import Client, ParameterMethodLocation from .parameter_serializer import ParameterSerializer, PopKwargType -from ...utils import build_policies +from ..._utils import build_policies class ClientSerializer: diff --git a/packages/pygen/pygen/codegen/serializers/sample_serializer.py b/packages/pygen/pygen/codegen/serializers/sample_serializer.py index 5694804c687..215471efaf9 100644 --- a/packages/pygen/pygen/codegen/serializers/sample_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/sample_serializer.py @@ -22,7 +22,7 @@ FileImport, ) from .utils import get_namespace_config, get_namespace_from_package_name -from ...utils import to_snake_case +from ..._utils import to_snake_case _LOGGER = logging.getLogger(__name__) diff --git a/packages/pygen/pygen/m2r/__init__.py b/packages/pygen/pygen/m2r/__init__.py index d6e78871787..450a0fadf63 100644 --- a/packages/pygen/pygen/m2r/__init__.py +++ b/packages/pygen/pygen/m2r/__init__.py @@ -11,7 +11,7 @@ import m2r2 from .. import YamlUpdatePlugin -from ..utils import parse_args +from .._utils import parse_args _LOGGER = logging.getLogger(__name__) diff --git a/packages/pygen/pygen/preprocess/__init__.py b/packages/pygen/pygen/preprocess/__init__.py index 81cc9134aac..6cea93cc121 100644 --- a/packages/pygen/pygen/preprocess/__init__.py +++ b/packages/pygen/pygen/preprocess/__init__.py @@ -8,7 +8,7 @@ import copy from typing import Callable, Dict, Any, List, Optional -from ..utils import to_snake_case +from .._utils import to_snake_case from .helpers import ( add_redefined_builtin_info, pad_builtin_namespaces, @@ -17,7 +17,7 @@ from .python_mappings import CADL_RESERVED_WORDS, RESERVED_WORDS, PadType from .. import YamlUpdatePlugin -from ..utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES +from .._utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES def update_overload_section( diff --git a/packages/pygen/run_tsp.py b/packages/pygen/run_tsp.py index eb75ca2d021..acc4404a90d 100644 --- a/packages/pygen/run_tsp.py +++ b/packages/pygen/run_tsp.py @@ -7,7 +7,7 @@ import venv import logging from pathlib import Path -from pygen.utils.venvtools import python_run +from venvtools import python_run _ROOT_DIR = Path(__file__).parent diff --git a/packages/pygen/venvtools.py b/packages/pygen/venvtools.py new file mode 100644 index 00000000000..01e1300b1c8 --- /dev/null +++ b/packages/pygen/venvtools.py @@ -0,0 +1,81 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from contextlib import contextmanager +import tempfile +import subprocess +import venv +import sys +from pathlib import Path + + +_ROOT_DIR = Path(__file__).parent + + +class ExtendedEnvBuilder(venv.EnvBuilder): + """An extended env builder which saves the context, to have access + easily to bin path and such. + """ + + def __init__(self, *args, **kwargs): + self.context = None + if sys.version_info < (3, 9, 0): + # Not supported on Python 3.8, and we don't need it + kwargs.pop("upgrade_deps", None) + super().__init__(*args, **kwargs) + + def ensure_directories(self, env_dir): + self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir) + return self.context + + +def create( + env_dir, system_site_packages=False, clear=False, symlinks=False, with_pip=False, prompt=None, upgrade_deps=False +): + """Create a virtual environment in a directory.""" + builder = ExtendedEnvBuilder( + system_site_packages=system_site_packages, + clear=clear, + symlinks=symlinks, + with_pip=with_pip, + prompt=prompt, + upgrade_deps=upgrade_deps, + ) + builder.create(env_dir) + return builder.context + + +@contextmanager +def create_venv_with_package(packages): + """Create a venv with these packages in a temp dir and yielf the env. + + packages should be an iterable of pip version instructio (e.g. package~=1.2.3) + """ + with tempfile.TemporaryDirectory() as tempdir: + myenv = create(tempdir, with_pip=True, upgrade_deps=True) + pip_call = [ + myenv.env_exe, + "-m", + "pip", + "install", + ] + subprocess.check_call(pip_call + ["-U", "pip"]) + if packages: + subprocess.check_call(pip_call + packages) + yield myenv + + +def python_run(venv_context, module, command=None, *, additional_dir="."): + try: + cmd_line = [venv_context.env_exe, "-m", module] + (command if command else []) + print("Executing: {}".format(" ".join(cmd_line))) + subprocess.run( + cmd_line, + cwd=_ROOT_DIR / additional_dir, + check=True, + ) + except subprocess.CalledProcessError as err: + print(err) + sys.exit(1) From ac5126d7c90c56cf3653cc1423453eb9013daec9 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 14:37:57 -0400 Subject: [PATCH 09/75] require changelog check --- eng/pipelines/ci-template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index 77216c0fd5f..f0a04d607b5 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -59,7 +59,6 @@ steps: displayName: Check changelog workingDirectory: $(Build.SourcesDirectory)/autorest.python/ condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/heads/publish/')), not(startsWith(variables['Build.SourceBranch'], 'refs/heads/dependabot/'))) - continueOnError: true - script: pnpm list displayName: Pnpm list From 5e9f02302be9239422001e770d99e4d6e28ae7a4 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 14:39:07 -0400 Subject: [PATCH 10/75] add changeset --- .chronus/changes/link_emitters-2024-4-13-14-39-2.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .chronus/changes/link_emitters-2024-4-13-14-39-2.md diff --git a/.chronus/changes/link_emitters-2024-4-13-14-39-2.md b/.chronus/changes/link_emitters-2024-4-13-14-39-2.md new file mode 100644 index 00000000000..ceda1c8c4ca --- /dev/null +++ b/.chronus/changes/link_emitters-2024-4-13-14-39-2.md @@ -0,0 +1,8 @@ +--- +changeKind: feature +packages: + - "@autorest/python" + - "@azure-tools/python-client-generator-core" +--- + +add package pygen that both autorest.python and typespec-python will rely on \ No newline at end of file From 721a467e0ec32fdf1754d4bbe0597be5ee18287d Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 14:39:51 -0400 Subject: [PATCH 11/75] add pcgc to changelog --- .chronus/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.chronus/config.yaml b/.chronus/config.yaml index a16058fb0e3..8001a421ecd 100644 --- a/.chronus/config.yaml +++ b/.chronus/config.yaml @@ -38,6 +38,7 @@ versionPolicies: packages: - "@azure-tools/typespec-python" - "@autorest/python" + - "@azure-tools/python-client-generator-core" changelog: ["@chronus/github/changelog", { repo: "azure/autorest.python" }] From def03ac07d04ab63844c0545a1960bbef8d3d84c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 13 May 2024 14:45:40 -0400 Subject: [PATCH 12/75] comment out pylinting --- eng/pipelines/ci-template.yml | 48 +++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index f0a04d607b5..1fbd8b85def 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -72,30 +72,30 @@ steps: displayName: Pip install dev requirements workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - - script: pylint ${{parameters.pythonFolderName}} - displayName: Pylint - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - - - script: mypy ${{parameters.pythonFolderName}} - displayName: Mypy - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - - - script: pyright ${{parameters.pythonFolderName}} - displayName: Pyright - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - - - script: black $(Build.SourcesDirectory)/autorest.python - displayName: Black - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - - - script: node ./eng/scripts/check-for-changed-files.js - displayName: Fail on black autorest diff - workingDirectory: $(Build.SourcesDirectory)/autorest.python/ - condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) + # - script: pylint ${{parameters.pythonFolderName}} + # displayName: Pylint + # workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + # condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) + + # - script: mypy ${{parameters.pythonFolderName}} + # displayName: Mypy + # workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + # condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) + + # - script: pyright ${{parameters.pythonFolderName}} + # displayName: Pyright + # workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + # condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) + + # - script: black $(Build.SourcesDirectory)/autorest.python + # displayName: Black + # workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + # condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) + + # - script: node ./eng/scripts/check-for-changed-files.js + # displayName: Fail on black autorest diff + # workingDirectory: $(Build.SourcesDirectory)/autorest.python/ + # condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - script: | cd test/unittests From 35fee3cedd21c1e351c5ef9499763b9cf7a429a5 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 11:52:09 -0400 Subject: [PATCH 13/75] add back linting and check --- eng/pipelines/ci-template.yml | 48 +++---- packages/autorest.python/autorest/_utils.py | 149 -------------------- packages/autorest.python/package.json | 6 +- packages/autorest.python/start.py | 2 +- packages/pygen/install.py | 16 ++- packages/pygen/prepare.py | 14 +- 6 files changed, 45 insertions(+), 190 deletions(-) delete mode 100644 packages/autorest.python/autorest/_utils.py diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index 1fbd8b85def..f0a04d607b5 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -72,30 +72,30 @@ steps: displayName: Pip install dev requirements workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - # - script: pylint ${{parameters.pythonFolderName}} - # displayName: Pylint - # workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - # condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - - # - script: mypy ${{parameters.pythonFolderName}} - # displayName: Mypy - # workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - # condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - - # - script: pyright ${{parameters.pythonFolderName}} - # displayName: Pyright - # workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - # condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - - # - script: black $(Build.SourcesDirectory)/autorest.python - # displayName: Black - # workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - # condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - - # - script: node ./eng/scripts/check-for-changed-files.js - # displayName: Fail on black autorest diff - # workingDirectory: $(Build.SourcesDirectory)/autorest.python/ - # condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) + - script: pylint ${{parameters.pythonFolderName}} + displayName: Pylint + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) + + - script: mypy ${{parameters.pythonFolderName}} + displayName: Mypy + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) + + - script: pyright ${{parameters.pythonFolderName}} + displayName: Pyright + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) + + - script: black $(Build.SourcesDirectory)/autorest.python + displayName: Black + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) + + - script: node ./eng/scripts/check-for-changed-files.js + displayName: Fail on black autorest diff + workingDirectory: $(Build.SourcesDirectory)/autorest.python/ + condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - script: | cd test/unittests diff --git a/packages/autorest.python/autorest/_utils.py b/packages/autorest.python/autorest/_utils.py deleted file mode 100644 index 95bf493bde8..00000000000 --- a/packages/autorest.python/autorest/_utils.py +++ /dev/null @@ -1,149 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any, Dict, Tuple, List -import re -import argparse - - -def update_enum_value(name: str, value: Any, description: str, enum_type: Dict[str, Any]) -> Dict[str, Any]: - return { - "name": name, - "type": "enumvalue", - "value": value, - "description": description, - "enumType": enum_type, - "valueType": enum_type["valueType"], - } - - -def to_snake_case(name: str) -> str: - def replace_upper_characters(m) -> str: - match_str = m.group().lower() - if m.start() > 0 and name[m.start() - 1] == "_": - # we are good if a '_' already exists - return match_str - # the first letter should not have _ - prefix = "_" if m.start() > 0 else "" - - # we will add an extra _ if there are multiple upper case chars together - next_non_upper_case_char_location = m.start() + len(match_str) - if ( - len(match_str) > 2 - and len(name) - next_non_upper_case_char_location > 1 - and name[next_non_upper_case_char_location].isalpha() - ): - return prefix + match_str[: len(match_str) - 1] + "_" + match_str[len(match_str) - 1] - - return prefix + match_str - - result = re.sub("[A-Z]+", replace_upper_characters, name) - return result.replace(" ", "_").replace("__", "_").replace("-", "") - - -def parse_args( - need_cadl_file: bool = True, -) -> Tuple[argparse.Namespace, Dict[str, Any]]: - parser = argparse.ArgumentParser( - description="Run mypy against target folder. Add a local custom plugin to the path prior to execution. " - ) - parser.add_argument( - "--output-folder", - dest="output_folder", - help="Output folder for generated SDK", - required=True, - ) - parser.add_argument( - "--cadl-file", - dest="cadl_file", - help="Serialized cadl file", - required=need_cadl_file, - ) - parser.add_argument( - "--debug", - dest="debug", - help="Debug mode", - required=False, - action="store", - ) - args, unknown_args = parser.parse_known_args() - - def _get_value(value: Any) -> Any: - if value == "true": - return True - if value == "false": - return False - try: - return int(value) - except ValueError: - pass - return value - - unknown_args_ret = { - ua.strip("--").split("=", maxsplit=1)[0]: _get_value( # pylint: disable=bad-str-strip-call - ua.strip("--").split("=", maxsplit=1)[1] # pylint: disable=bad-str-strip-call - ) - for ua in unknown_args - } - return args, unknown_args_ret - - -def get_body_type_for_description(body_parameter: Dict[str, Any]) -> str: - if body_parameter["type"]["type"] == "binary": - return "binary" - if body_parameter["type"]["type"] == "string": - return "string" - return "JSON" - - -# used if we want to get a string / binary type etc -KNOWN_TYPES: Dict[str, Dict[str, Any]] = { - "string": {"type": "string"}, - "binary": {"type": "binary"}, - "anydict": {"type": "dict", "elementType": {"type": "any"}}, - "any-object": {"type": "any-object"}, -} - -JSON_REGEXP = re.compile(r"^(application|text)/(.+\+)?json$") - - -def build_policies( - is_arm: bool, - async_mode: bool, - *, - is_azure_flavor: bool = False, - tracing: bool = True, -) -> List[str]: - if is_azure_flavor: - # for Azure - async_prefix = "Async" if async_mode else "" - policies = [ - "policies.RequestIdPolicy(**kwargs)", - "self._config.headers_policy", - "self._config.user_agent_policy", - "self._config.proxy_policy", - "policies.ContentDecodePolicy(**kwargs)", - (f"{async_prefix}ARMAutoResourceProviderRegistrationPolicy()" if is_arm else None), - "self._config.redirect_policy", - "self._config.retry_policy", - "self._config.authentication_policy", - "self._config.custom_hook_policy", - "self._config.logging_policy", - "policies.DistributedTracingPolicy(**kwargs)" if tracing else None, - "policies.SensitiveHeaderCleanupPolicy(**kwargs) if self._config.redirect_policy else None", - "self._config.http_logging_policy", - ] - else: - # for non-Azure - policies = [ - "self._config.headers_policy", - "self._config.user_agent_policy", - "self._config.proxy_policy", - "policies.ContentDecodePolicy(**kwargs)", - "self._config.retry_policy", - "self._config.authentication_policy", - "self._config.logging_policy", - ] - return [p for p in policies if p] diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index c651b28476e..cfa53db8a79 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -3,10 +3,10 @@ "version": "6.13.17", "description": "The Python extension for generators in AutoRest.", "scripts": { - "prepare": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/prepare.py", + "prepare": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/prepare.py --root .", "start": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js start.py", - "install": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/install.py", - "debug": "node r./node_modules/@azure-tools/python-client-generator-core/un-python3.js ./node_modules/@azure-tools/python-client-generator-core/start.py --debug" + "install": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/install.py --root .", + "debug": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js start.py --debug" }, "main": "index.js", "repository": { diff --git a/packages/autorest.python/start.py b/packages/autorest.python/start.py index fca22e73286..fdf57593653 100644 --- a/packages/autorest.python/start.py +++ b/packages/autorest.python/start.py @@ -19,7 +19,7 @@ def main(): - venv_path = _ROOT_DIR / "node_modules/@azure-tools/python-client-generator-core/venv" + venv_path = _ROOT_DIR / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/packages/pygen/install.py b/packages/pygen/install.py index 14ae38dd278..2e0ff9361c2 100644 --- a/packages/pygen/install.py +++ b/packages/pygen/install.py @@ -6,6 +6,7 @@ # license information. # -------------------------------------------------------------------------- import sys +import argparse if not sys.version_info >= (3, 8, 0): raise Exception("Autorest for Python extension requires Python 3.8 at least") @@ -28,12 +29,9 @@ from venvtools import ExtendedEnvBuilder, python_run -_ROOT_DIR = Path(__file__).parent - - -def main(): - venv_path = _ROOT_DIR / "venv" +def main(root_dir): + venv_path = root_dir / "venv" if venv_path.exists(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) @@ -44,8 +42,12 @@ def main(): python_run(venv_context, "pip", ["install", "-U", "pip"]) python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) - python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) + python_run(venv_context, "pip", ["install", "-e", str(root_dir)]) if __name__ == "__main__": - main() + parser = argparse.ArgumentParser(description="Run emitter build commands.") + parser.add_argument('--root', help='Path to the virtual environment') + + args = parser.parse_args() + main(Path(args.root or Path(__file__).parent)) diff --git a/packages/pygen/prepare.py b/packages/pygen/prepare.py index e407cc04c46..f6e84951180 100644 --- a/packages/pygen/prepare.py +++ b/packages/pygen/prepare.py @@ -6,6 +6,7 @@ # license information. # -------------------------------------------------------------------------- import sys +import argparse if not sys.version_info >= (3, 8, 0): raise Exception("Autorest for Python extension requires Python 3.8 at least") @@ -15,18 +16,16 @@ from venvtools import python_run -_ROOT_DIR = Path(__file__).parent - -def main(): - venv_path = _ROOT_DIR / "venv" +def main(root_dir): + venv_path = root_dir / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) - requirements_path = _ROOT_DIR / "dev_requirements.txt" + requirements_path = root_dir / "dev_requirements.txt" try: python_run(venv_context, "pip", ["install", "-r", str(requirements_path)]) except FileNotFoundError as e: @@ -34,4 +33,7 @@ def main(): if __name__ == "__main__": - main() + parser = argparse.ArgumentParser(description="Run emitter build commands.") + parser.add_argument('--root', help='Path to the virtual environment') + args = parser.parse_args() + main(Path(args.root or Path(__file__).parent)) From 321f1e4bc7f24cedaad66eeb05fca7561dcb2ab7 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 12:06:12 -0400 Subject: [PATCH 14/75] list installed packages --- eng/pipelines/ci-template.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index f0a04d607b5..a0c3fa673f4 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -68,10 +68,18 @@ steps: displayName: Build project workingDirectory: $(Build.SourcesDirectory)/autorest.python/ + - script: pip list + displayname: List installed packages + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pip install -r dev_requirements.txt displayName: Pip install dev requirements workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pip list + displayname: List installed packages + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pylint ${{parameters.pythonFolderName}} displayName: Pylint workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} From a42baf1e19a02cc8b3b3f1429fe5b848d0f7e3f6 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 12:52:08 -0400 Subject: [PATCH 15/75] list installed packages --- eng/pipelines/ci-template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index a0c3fa673f4..2b49a88122e 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -69,7 +69,7 @@ steps: workingDirectory: $(Build.SourcesDirectory)/autorest.python/ - script: pip list - displayname: List installed packages + displayName: List installed packages workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - script: pip install -r dev_requirements.txt @@ -77,7 +77,7 @@ steps: workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - script: pip list - displayname: List installed packages + displayName: List installed packages workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - script: pylint ${{parameters.pythonFolderName}} From edc0faf11e6b3787bd466acfb882b0c7ad165a80 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 13:02:20 -0400 Subject: [PATCH 16/75] activate env in pipeline --- eng/pipelines/ci-template.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index 2b49a88122e..e8abbe087a7 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -68,6 +68,15 @@ steps: displayName: Build project workingDirectory: $(Build.SourcesDirectory)/autorest.python/ + - script: | + if [ "$(Agent.OS)" == "Windows_NT" ]; then + call venv\Scripts\activate + else + source venv/bin/activate + fi + displayName: 'Activate virtual environment' + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/autorest.python + - script: pip list displayName: List installed packages workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} From 94982aa502955ebc99c81098ee57ee7e9270777a Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 13:06:35 -0400 Subject: [PATCH 17/75] add pygen to common reqs --- eng/pipelines/ci-template.yml | 9 --------- eng/requirements.txt | 1 + 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index e8abbe087a7..2b49a88122e 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -68,15 +68,6 @@ steps: displayName: Build project workingDirectory: $(Build.SourcesDirectory)/autorest.python/ - - script: | - if [ "$(Agent.OS)" == "Windows_NT" ]; then - call venv\Scripts\activate - else - source venv/bin/activate - fi - displayName: 'Activate virtual environment' - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/autorest.python - - script: pip list displayName: List installed packages workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} diff --git a/eng/requirements.txt b/eng/requirements.txt index ec4654ea52f..091247e7713 100644 --- a/eng/requirements.txt +++ b/eng/requirements.txt @@ -3,3 +3,4 @@ pyright==1.1.362 pylint==3.1.0 tox==4.15.0 mypy==1.10.0 +-e ./packages/pygen From 5d8f9813e7b2a7143d89ea5cd04a99f7c766f647 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 13:13:44 -0400 Subject: [PATCH 18/75] change path to be relative from autorest.python --- eng/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/requirements.txt b/eng/requirements.txt index 091247e7713..01a0dd95c02 100644 --- a/eng/requirements.txt +++ b/eng/requirements.txt @@ -3,4 +3,4 @@ pyright==1.1.362 pylint==3.1.0 tox==4.15.0 mypy==1.10.0 --e ./packages/pygen +-e ../pygen From 99f1d25da986b9cc496486456f6c6f2d5dd28956 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 13:42:51 -0400 Subject: [PATCH 19/75] pylint --- packages/autorest.python/autorest/m4reformatter/__init__.py | 2 +- .../autorest.python/autorest/multiapi/serializers/__init__.py | 3 ++- packages/pygen/install.py | 2 +- packages/pygen/prepare.py | 2 +- packages/pygen/pygen/__init__.py | 1 - packages/pygen/pygen/black/__init__.py | 2 +- packages/pygen/pygen/codegen/__init__.py | 2 +- packages/pygen/pygen/codegen/serializers/__init__.py | 2 +- .../pygen/pygen/codegen/serializers/builder_serializer.py | 2 +- packages/pygen/pygen/codegen/serializers/client_serializer.py | 2 +- packages/pygen/pygen/codegen/serializers/sample_serializer.py | 2 +- packages/pygen/pygen/m2r/__init__.py | 2 +- packages/pygen/pygen/preprocess/__init__.py | 4 ++-- packages/pygen/pygen/{_utils.py => utils.py} | 0 14 files changed, 14 insertions(+), 14 deletions(-) rename packages/pygen/pygen/{_utils.py => utils.py} (100%) diff --git a/packages/autorest.python/autorest/m4reformatter/__init__.py b/packages/autorest.python/autorest/m4reformatter/__init__.py index ab4cfa8fd88..18364e575b2 100644 --- a/packages/autorest.python/autorest/m4reformatter/__init__.py +++ b/packages/autorest.python/autorest/m4reformatter/__init__.py @@ -11,7 +11,7 @@ import logging from typing import Callable, Dict, Any, Iterable, List, Optional, Set -from .._utils import ( +from pygen.utils import ( to_snake_case, KNOWN_TYPES, get_body_type_for_description, diff --git a/packages/autorest.python/autorest/multiapi/serializers/__init__.py b/packages/autorest.python/autorest/multiapi/serializers/__init__.py index 99c2be1547d..a84cad8a92a 100644 --- a/packages/autorest.python/autorest/multiapi/serializers/__init__.py +++ b/packages/autorest.python/autorest/multiapi/serializers/__init__.py @@ -6,13 +6,14 @@ from pathlib import Path from typing import Any, Optional, Union, List from jinja2 import PackageLoader, Environment +from pygen.utils import build_policies from .import_serializer import FileImportSerializer from ...jsonrpc import AutorestAPI from ..models import CodeModel, GlobalParameter from ... import ReaderAndWriter, ReaderAndWriterAutorest -from ..._utils import build_policies + __all__ = [ "MultiAPISerializer", diff --git a/packages/pygen/install.py b/packages/pygen/install.py index 2e0ff9361c2..dd177f67477 100644 --- a/packages/pygen/install.py +++ b/packages/pygen/install.py @@ -47,7 +47,7 @@ def main(root_dir): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run emitter build commands.") - parser.add_argument('--root', help='Path to the virtual environment') + parser.add_argument("--root", help="Path to the virtual environment") args = parser.parse_args() main(Path(args.root or Path(__file__).parent)) diff --git a/packages/pygen/prepare.py b/packages/pygen/prepare.py index f6e84951180..afbfaafe1e9 100644 --- a/packages/pygen/prepare.py +++ b/packages/pygen/prepare.py @@ -34,6 +34,6 @@ def main(root_dir): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run emitter build commands.") - parser.add_argument('--root', help='Path to the virtual environment') + parser.add_argument("--root", help="Path to the virtual environment") args = parser.parse_args() main(Path(args.root or Path(__file__).parent)) diff --git a/packages/pygen/pygen/__init__.py b/packages/pygen/pygen/__init__.py index 280c4c97c65..6050cbec00f 100644 --- a/packages/pygen/pygen/__init__.py +++ b/packages/pygen/pygen/__init__.py @@ -105,4 +105,3 @@ def update_yaml(self, yaml_data: Dict[str, Any]) -> None: :raises Exception: Could raise any exception, stacktrace will be sent to autorest API """ raise NotImplementedError() - diff --git a/packages/pygen/pygen/black/__init__.py b/packages/pygen/pygen/black/__init__.py index a17a536485a..82033d53b17 100644 --- a/packages/pygen/pygen/black/__init__.py +++ b/packages/pygen/pygen/black/__init__.py @@ -11,7 +11,7 @@ from black.report import NothingChanged from .. import Plugin -from .._utils import parse_args +from ..utils import parse_args _LOGGER = logging.getLogger("blib2to3") diff --git a/packages/pygen/pygen/codegen/__init__.py b/packages/pygen/pygen/codegen/__init__.py index c5a30203843..ea4716f86e6 100644 --- a/packages/pygen/pygen/codegen/__init__.py +++ b/packages/pygen/pygen/codegen/__init__.py @@ -10,7 +10,7 @@ from .. import Plugin -from .._utils import parse_args +from ..utils import parse_args from .models.code_model import CodeModel from .serializers import JinjaSerializer from ._utils import DEFAULT_HEADER_TEXT, VALID_PACKAGE_MODE, TYPESPEC_PACKAGE_MODE diff --git a/packages/pygen/pygen/codegen/serializers/__init__.py b/packages/pygen/pygen/codegen/serializers/__init__.py index 09bc8ecde2d..ee3d27ff8ad 100644 --- a/packages/pygen/pygen/codegen/serializers/__init__.py +++ b/packages/pygen/pygen/codegen/serializers/__init__.py @@ -28,7 +28,7 @@ from .sample_serializer import SampleSerializer from .test_serializer import TestSerializer, TestGeneralSerializer from .types_serializer import TypesSerializer -from ..._utils import to_snake_case +from ...utils import to_snake_case from .._utils import VALID_PACKAGE_MODE from .utils import ( extract_sample_name, diff --git a/packages/pygen/pygen/codegen/serializers/builder_serializer.py b/packages/pygen/pygen/codegen/serializers/builder_serializer.py index 42df4ae0ff9..6207714e7d9 100644 --- a/packages/pygen/pygen/codegen/serializers/builder_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/builder_serializer.py @@ -36,7 +36,7 @@ from .parameter_serializer import ParameterSerializer, PopKwargType from ..models.parameter_list import ParameterType from . import utils -from ..._utils import JSON_REGEXP +from ...utils import JSON_REGEXP T = TypeVar("T") OrderedSet = Dict[T, None] diff --git a/packages/pygen/pygen/codegen/serializers/client_serializer.py b/packages/pygen/pygen/codegen/serializers/client_serializer.py index ce8dfd87964..28041370efb 100644 --- a/packages/pygen/pygen/codegen/serializers/client_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/client_serializer.py @@ -8,7 +8,7 @@ from . import utils from ..models import Client, ParameterMethodLocation from .parameter_serializer import ParameterSerializer, PopKwargType -from ..._utils import build_policies +from ...utils import build_policies class ClientSerializer: diff --git a/packages/pygen/pygen/codegen/serializers/sample_serializer.py b/packages/pygen/pygen/codegen/serializers/sample_serializer.py index 215471efaf9..5694804c687 100644 --- a/packages/pygen/pygen/codegen/serializers/sample_serializer.py +++ b/packages/pygen/pygen/codegen/serializers/sample_serializer.py @@ -22,7 +22,7 @@ FileImport, ) from .utils import get_namespace_config, get_namespace_from_package_name -from ..._utils import to_snake_case +from ...utils import to_snake_case _LOGGER = logging.getLogger(__name__) diff --git a/packages/pygen/pygen/m2r/__init__.py b/packages/pygen/pygen/m2r/__init__.py index 450a0fadf63..d6e78871787 100644 --- a/packages/pygen/pygen/m2r/__init__.py +++ b/packages/pygen/pygen/m2r/__init__.py @@ -11,7 +11,7 @@ import m2r2 from .. import YamlUpdatePlugin -from .._utils import parse_args +from ..utils import parse_args _LOGGER = logging.getLogger(__name__) diff --git a/packages/pygen/pygen/preprocess/__init__.py b/packages/pygen/pygen/preprocess/__init__.py index 6cea93cc121..81cc9134aac 100644 --- a/packages/pygen/pygen/preprocess/__init__.py +++ b/packages/pygen/pygen/preprocess/__init__.py @@ -8,7 +8,7 @@ import copy from typing import Callable, Dict, Any, List, Optional -from .._utils import to_snake_case +from ..utils import to_snake_case from .helpers import ( add_redefined_builtin_info, pad_builtin_namespaces, @@ -17,7 +17,7 @@ from .python_mappings import CADL_RESERVED_WORDS, RESERVED_WORDS, PadType from .. import YamlUpdatePlugin -from .._utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES +from ..utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES def update_overload_section( diff --git a/packages/pygen/pygen/_utils.py b/packages/pygen/pygen/utils.py similarity index 100% rename from packages/pygen/pygen/_utils.py rename to packages/pygen/pygen/utils.py From b6e0faebb9bcff42edaf558f3f7961db4831803f Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 13:52:43 -0400 Subject: [PATCH 20/75] pyright --- packages/autorest.python/autorest/multiapi/__init__.py | 3 ++- .../autorest.python/autorest/multiapi/serializers/__init__.py | 3 ++- packages/autorest.python/autorest/multiclient/__init__.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/autorest.python/autorest/multiapi/__init__.py b/packages/autorest.python/autorest/multiapi/__init__.py index 4b321cc8d97..c1df35c21cf 100644 --- a/packages/autorest.python/autorest/multiapi/__init__.py +++ b/packages/autorest.python/autorest/multiapi/__init__.py @@ -10,12 +10,13 @@ from collections import defaultdict from pathlib import Path from typing import Dict, List, Optional, cast, Any +from pygen import Plugin, ReaderAndWriter from .serializers import MultiAPISerializer, MultiAPISerializerAutorest from .models import CodeModel from .utils import _get_default_api_version_from_list -from .. import Plugin, PluginAutorest, ReaderAndWriter, ReaderAndWriterAutorest +from .. import PluginAutorest, ReaderAndWriterAutorest _LOGGER = logging.getLogger(__name__) diff --git a/packages/autorest.python/autorest/multiapi/serializers/__init__.py b/packages/autorest.python/autorest/multiapi/serializers/__init__.py index a84cad8a92a..41d998381ea 100644 --- a/packages/autorest.python/autorest/multiapi/serializers/__init__.py +++ b/packages/autorest.python/autorest/multiapi/serializers/__init__.py @@ -6,13 +6,14 @@ from pathlib import Path from typing import Any, Optional, Union, List from jinja2 import PackageLoader, Environment +from pygen import ReaderAndWriter from pygen.utils import build_policies from .import_serializer import FileImportSerializer from ...jsonrpc import AutorestAPI from ..models import CodeModel, GlobalParameter -from ... import ReaderAndWriter, ReaderAndWriterAutorest +from ... import ReaderAndWriterAutorest __all__ = [ diff --git a/packages/autorest.python/autorest/multiclient/__init__.py b/packages/autorest.python/autorest/multiclient/__init__.py index 3d397f6b086..f0e0c32bba9 100644 --- a/packages/autorest.python/autorest/multiclient/__init__.py +++ b/packages/autorest.python/autorest/multiclient/__init__.py @@ -7,7 +7,8 @@ from typing import Any, Dict from pathlib import Path from jinja2 import Environment, PackageLoader -from .. import Plugin, PluginAutorest +from pygen import Plugin +from .. import PluginAutorest _LOGGER = logging.getLogger(__name__) From ea73b14924ac2fd49fcd204ba4366ca576502083 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 14:55:05 -0400 Subject: [PATCH 21/75] add pygen to setup.py, remove from requirements.txt --- eng/pipelines/ci-template.yml | 12 ++---------- eng/requirements.txt | 1 - packages/autorest.python/requirements.txt | 1 - packages/autorest.python/setup.py | 3 +++ 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index 2b49a88122e..22fdb007180 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -68,18 +68,10 @@ steps: displayName: Build project workingDirectory: $(Build.SourcesDirectory)/autorest.python/ - - script: pip list - displayName: List installed packages - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - - script: pip install -r dev_requirements.txt displayName: Pip install dev requirements workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - - script: pip list - displayName: List installed packages - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - - script: pylint ${{parameters.pythonFolderName}} displayName: Pylint workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} @@ -106,8 +98,8 @@ steps: condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - script: | - cd test/unittests - tox run -e ci + cd test/unittests + tox run -e ci displayName: Unit tests workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/ condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) diff --git a/eng/requirements.txt b/eng/requirements.txt index 01a0dd95c02..ec4654ea52f 100644 --- a/eng/requirements.txt +++ b/eng/requirements.txt @@ -3,4 +3,3 @@ pyright==1.1.362 pylint==3.1.0 tox==4.15.0 mypy==1.10.0 --e ../pygen diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index d7e96763277..17ae43bada8 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -11,4 +11,3 @@ platformdirs==3.2.0 PyYAML==6.0.1 tomli==2.0.1 setuptools==69.2.0 --e ../pygen diff --git a/packages/autorest.python/setup.py b/packages/autorest.python/setup.py index b22fa2172ca..b648365e464 100644 --- a/packages/autorest.python/setup.py +++ b/packages/autorest.python/setup.py @@ -53,4 +53,7 @@ "m2r2", "black", ], + dependency_links=[ + 'file:../pygen#egg=pygen', + ] ) From 9f59d8501f3e3c278690f88a1f178954c5afc2d5 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 15:02:15 -0400 Subject: [PATCH 22/75] Revert "add pygen to setup.py, remove from requirements.txt" This reverts commit ea73b14924ac2fd49fcd204ba4366ca576502083. --- eng/pipelines/ci-template.yml | 12 ++++++++++-- eng/requirements.txt | 1 + packages/autorest.python/requirements.txt | 1 + packages/autorest.python/setup.py | 3 --- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index 22fdb007180..2b49a88122e 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -68,10 +68,18 @@ steps: displayName: Build project workingDirectory: $(Build.SourcesDirectory)/autorest.python/ + - script: pip list + displayName: List installed packages + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pip install -r dev_requirements.txt displayName: Pip install dev requirements workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pip list + displayName: List installed packages + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pylint ${{parameters.pythonFolderName}} displayName: Pylint workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} @@ -98,8 +106,8 @@ steps: condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) - script: | - cd test/unittests - tox run -e ci + cd test/unittests + tox run -e ci displayName: Unit tests workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/ condition: and(succeeded(), ${{ parameters.pythonCodeChecks }}) diff --git a/eng/requirements.txt b/eng/requirements.txt index ec4654ea52f..01a0dd95c02 100644 --- a/eng/requirements.txt +++ b/eng/requirements.txt @@ -3,3 +3,4 @@ pyright==1.1.362 pylint==3.1.0 tox==4.15.0 mypy==1.10.0 +-e ../pygen diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index 17ae43bada8..d7e96763277 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -11,3 +11,4 @@ platformdirs==3.2.0 PyYAML==6.0.1 tomli==2.0.1 setuptools==69.2.0 +-e ../pygen diff --git a/packages/autorest.python/setup.py b/packages/autorest.python/setup.py index b648365e464..b22fa2172ca 100644 --- a/packages/autorest.python/setup.py +++ b/packages/autorest.python/setup.py @@ -53,7 +53,4 @@ "m2r2", "black", ], - dependency_links=[ - 'file:../pygen#egg=pygen', - ] ) From d67326954edd4d7e31856f4919179880ee8fe711 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 15:06:48 -0400 Subject: [PATCH 23/75] add pygen to unittests requirements.txt --- packages/autorest.python/test/unittests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/autorest.python/test/unittests/requirements.txt b/packages/autorest.python/test/unittests/requirements.txt index e68ca624cb2..26bf5c043ae 100644 --- a/packages/autorest.python/test/unittests/requirements.txt +++ b/packages/autorest.python/test/unittests/requirements.txt @@ -4,3 +4,4 @@ pytest pytest-cov azure-core==1.30.0 -e ../../. +-e ../../../pygen From 9b21837314a8c61211ad114e14ef66576cae785f Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 15:15:32 -0400 Subject: [PATCH 24/75] update unittests --- packages/autorest.python/test/unittests/test_m2r.py | 2 +- .../autorest.python/test/unittests/test_name_converter.py | 4 ++-- .../test/unittests/test_optional_return_type.py | 6 +++--- .../test/unittests/test_parameter_ordering.py | 4 ++-- packages/autorest.python/test/unittests/test_sort_schema.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/autorest.python/test/unittests/test_m2r.py b/packages/autorest.python/test/unittests/test_m2r.py index 8ac1287dda3..f61941a3b7e 100644 --- a/packages/autorest.python/test/unittests/test_m2r.py +++ b/packages/autorest.python/test/unittests/test_m2r.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from autorest.m2r import M2R +from pygen.m2r import M2R _MD_LINK = "[inline link](https://github.com/Azure/autorest.python)" diff --git a/packages/autorest.python/test/unittests/test_name_converter.py b/packages/autorest.python/test/unittests/test_name_converter.py index 1e650033e9b..930eb0fc60d 100644 --- a/packages/autorest.python/test/unittests/test_name_converter.py +++ b/packages/autorest.python/test/unittests/test_name_converter.py @@ -3,8 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from autorest.preprocess import PreProcessPlugin -from autorest.preprocess.python_mappings import PadType +from pygen.preprocess import PreProcessPlugin +from pygen.preprocess.python_mappings import PadType def pad_reserved_words(name: str, pad_type: PadType) -> str: diff --git a/packages/autorest.python/test/unittests/test_optional_return_type.py b/packages/autorest.python/test/unittests/test_optional_return_type.py index 1a4cfae46e5..26f413714cf 100644 --- a/packages/autorest.python/test/unittests/test_optional_return_type.py +++ b/packages/autorest.python/test/unittests/test_optional_return_type.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest -from autorest.codegen.models import ( +from pygen.codegen.models import ( Operation, LROOperation, PagingOperation, @@ -15,8 +15,8 @@ RequestBuilder, Client, ) -from autorest.codegen.models.parameter_list import RequestBuilderParameterList -from autorest.codegen.models.primitive_types import StringType +from pygen.codegen.models.parameter_list import RequestBuilderParameterList +from pygen.codegen.models.primitive_types import StringType @pytest.fixture diff --git a/packages/autorest.python/test/unittests/test_parameter_ordering.py b/packages/autorest.python/test/unittests/test_parameter_ordering.py index 8c68fd14863..4f64c991adf 100644 --- a/packages/autorest.python/test/unittests/test_parameter_ordering.py +++ b/packages/autorest.python/test/unittests/test_parameter_ordering.py @@ -3,8 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from autorest.codegen.models import Parameter, AnyType, CodeModel, StringType -from autorest.codegen.models.parameter_list import ParameterList +from pygen.codegen.models import Parameter, AnyType, CodeModel, StringType +from pygen.codegen.models.parameter_list import ParameterList def get_code_model(): diff --git a/packages/autorest.python/test/unittests/test_sort_schema.py b/packages/autorest.python/test/unittests/test_sort_schema.py index 156b162598b..253b83f100e 100644 --- a/packages/autorest.python/test/unittests/test_sort_schema.py +++ b/packages/autorest.python/test/unittests/test_sort_schema.py @@ -4,7 +4,7 @@ # license information. # -------------------------------------------------------------------------- -from autorest.codegen.models import CodeModel, DPGModelType +from pygen.codegen.models import CodeModel, DPGModelType def get_code_model(): From e31a16f0ae294a3717bebdf9d7e66e2497278a58 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 14 May 2024 16:54:07 -0400 Subject: [PATCH 25/75] hopefully fix multiapi --- .../autorest.python/autorest/multiapi/serializers/__init__.py | 2 +- packages/autorest.python/mypy.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/autorest.python/autorest/multiapi/serializers/__init__.py b/packages/autorest.python/autorest/multiapi/serializers/__init__.py index 41d998381ea..9f197a7e2e0 100644 --- a/packages/autorest.python/autorest/multiapi/serializers/__init__.py +++ b/packages/autorest.python/autorest/multiapi/serializers/__init__.py @@ -125,7 +125,7 @@ def serialize(self, code_model: CodeModel, no_async: Optional[bool]) -> None: if not code_model.client.client_side_validation: codegen_env = Environment( - loader=PackageLoader("autorest.codegen", "templates"), + loader=PackageLoader("pygen.codegen", "templates"), keep_trailing_newline=True, line_statement_prefix="##", line_comment_prefix="###", diff --git a/packages/autorest.python/mypy.ini b/packages/autorest.python/mypy.ini index 76618ab1a44..7fbbd82b849 100644 --- a/packages/autorest.python/mypy.ini +++ b/packages/autorest.python/mypy.ini @@ -22,7 +22,7 @@ ignore_missing_imports = True [mypy-autorest.common.python_mappings.*] ignore_missing_imports = True -[mypy-autorest.codegen.models.*] +[mypy-pygen.codegen.models.*] ignore_missing_imports = True [mypy-setuptools] From 1ad22b43d228b6e799a2aed62d6f339a0b5b45db Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 15 May 2024 12:55:38 -0400 Subject: [PATCH 26/75] keep it as just one venv in pygen --- packages/autorest.python/dev_requirements.txt | 4 -- packages/autorest.python/install.py | 49 +++++++++++++++++++ packages/autorest.python/package.json | 4 +- packages/autorest.python/requirements.txt | 12 ----- packages/autorest.python/start.py | 3 +- packages/pygen/install.py | 16 +++--- packages/pygen/prepare.py | 13 +++-- packages/pygen/requirements.txt | 1 - 8 files changed, 65 insertions(+), 37 deletions(-) create mode 100644 packages/autorest.python/install.py diff --git a/packages/autorest.python/dev_requirements.txt b/packages/autorest.python/dev_requirements.txt index d458756b7d8..d6e1198b1ab 100644 --- a/packages/autorest.python/dev_requirements.txt +++ b/packages/autorest.python/dev_requirements.txt @@ -1,5 +1 @@ -e . --r ../../eng/requirements.txt --r ../../eng/dev_requirements.txt -ptvsd==4.3.2 -types-PyYAML==6.0.12.8 diff --git a/packages/autorest.python/install.py b/packages/autorest.python/install.py new file mode 100644 index 00000000000..288d9338e1f --- /dev/null +++ b/packages/autorest.python/install.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys + +if not sys.version_info >= (3, 8, 0): + raise Exception("Autorest for Python extension requires Python 3.8 at least") + +try: + import pip +except ImportError: + raise Exception("Your Python installation doesn't have pip available") + +try: + import venv +except ImportError: + raise Exception("Your Python installation doesn't have venv available") + + +# Now we have pip and Py >= 3.8, go to work + +from pathlib import Path + +from venvtools import ExtendedEnvBuilder, python_run + +_ROOT_DIR = Path(__file__).parent + + +def main(): + # we use pygen's venv + venv_path = _ROOT_DIR.parent / "pygen" / "venv" + if venv_path.exists(): + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + else: + env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True) + env_builder.create(venv_path) + venv_context = env_builder.context + + python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) + python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) + + +if __name__ == "__main__": + main() diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index cfa53db8a79..74a16e8b4e7 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -3,9 +3,9 @@ "version": "6.13.17", "description": "The Python extension for generators in AutoRest.", "scripts": { - "prepare": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/prepare.py --root .", + "prepare": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/prepare.py", "start": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js start.py", - "install": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/install.py --root .", + "install": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js install.py", "debug": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js start.py --debug" }, "main": "index.js", diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index d7e96763277..169d3121b06 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -1,14 +1,2 @@ -black==24.4.0 -click==8.1.3 -docutils==0.19 -Jinja2==3.1.4 json-rpc==1.14.0 -m2r2==0.3.3 -MarkupSafe==2.1.2 -mistune==0.8.4 -pathspec==0.11.1 -platformdirs==3.2.0 -PyYAML==6.0.1 -tomli==2.0.1 -setuptools==69.2.0 -e ../pygen diff --git a/packages/autorest.python/start.py b/packages/autorest.python/start.py index fdf57593653..85053e82ff7 100644 --- a/packages/autorest.python/start.py +++ b/packages/autorest.python/start.py @@ -19,7 +19,8 @@ def main(): - venv_path = _ROOT_DIR / "venv" + # we use pygen's venv + venv_path = _ROOT_DIR.parent / "pygen" / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/packages/pygen/install.py b/packages/pygen/install.py index dd177f67477..237a982b0c6 100644 --- a/packages/pygen/install.py +++ b/packages/pygen/install.py @@ -6,7 +6,6 @@ # license information. # -------------------------------------------------------------------------- import sys -import argparse if not sys.version_info >= (3, 8, 0): raise Exception("Autorest for Python extension requires Python 3.8 at least") @@ -24,14 +23,15 @@ # Now we have pip and Py >= 3.8, go to work -import subprocess from pathlib import Path from venvtools import ExtendedEnvBuilder, python_run +_ROOT_DIR = Path(__file__).parent -def main(root_dir): - venv_path = root_dir / "venv" + +def main(): + venv_path = _ROOT_DIR / "venv" if venv_path.exists(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) @@ -42,12 +42,8 @@ def main(root_dir): python_run(venv_context, "pip", ["install", "-U", "pip"]) python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) - python_run(venv_context, "pip", ["install", "-e", str(root_dir)]) + python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Run emitter build commands.") - parser.add_argument("--root", help="Path to the virtual environment") - - args = parser.parse_args() - main(Path(args.root or Path(__file__).parent)) + main() diff --git a/packages/pygen/prepare.py b/packages/pygen/prepare.py index afbfaafe1e9..b7e9b2dc781 100644 --- a/packages/pygen/prepare.py +++ b/packages/pygen/prepare.py @@ -16,16 +16,18 @@ from venvtools import python_run +_ROOT_DIR = Path(__file__).parent -def main(root_dir): - venv_path = root_dir / "venv" + +def main(): + venv_path = _ROOT_DIR / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) - requirements_path = root_dir / "dev_requirements.txt" + requirements_path = _ROOT_DIR / "dev_requirements.txt" try: python_run(venv_context, "pip", ["install", "-r", str(requirements_path)]) except FileNotFoundError as e: @@ -33,7 +35,4 @@ def main(root_dir): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Run emitter build commands.") - parser.add_argument("--root", help="Path to the virtual environment") - args = parser.parse_args() - main(Path(args.root or Path(__file__).parent)) + main() diff --git a/packages/pygen/requirements.txt b/packages/pygen/requirements.txt index 17ae43bada8..bebbbb5a645 100644 --- a/packages/pygen/requirements.txt +++ b/packages/pygen/requirements.txt @@ -2,7 +2,6 @@ black==24.4.0 click==8.1.3 docutils==0.19 Jinja2==3.1.4 -json-rpc==1.14.0 m2r2==0.3.3 MarkupSafe==2.1.2 mistune==0.8.4 From 904ba52567c6ea95416fe74a6b507ff3c83d598a Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 15 May 2024 14:52:07 -0400 Subject: [PATCH 27/75] typespec compiling with pygen inside dist --- packages/autorest.python/install.py | 2 +- packages/typespec-python/package.json | 7 +++---- packages/typespec-python/src/emitter.ts | 5 +++-- pnpm-lock.yaml | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/autorest.python/install.py b/packages/autorest.python/install.py index 288d9338e1f..fddd5e9fa6c 100644 --- a/packages/autorest.python/install.py +++ b/packages/autorest.python/install.py @@ -27,7 +27,7 @@ from venvtools import ExtendedEnvBuilder, python_run -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent def main(): diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index b526d6f244c..3859f9e5007 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -38,8 +38,7 @@ "files": [ "lib/*.cadl", "dist/**", - "!dist/test/**", - "get-autorest-python-path.cjs" + "!dist/test/**" ], "peerDependencies": { "@azure-tools/typespec-azure-core": ">=0.42.0 <1.0.0", @@ -58,9 +57,9 @@ } }, "dependencies": { - "@autorest/python": "workspace:^", "js-yaml": "~4.1.0", - "@typespec/openapi3": "~0.56.0" + "@typespec/openapi3": "~0.56.0", + "@autorest/system-requirements": "~1.0.2" }, "devDependencies": { "@azure-tools/typespec-azure-resource-manager": "~0.42.0", diff --git a/packages/typespec-python/src/emitter.ts b/packages/typespec-python/src/emitter.ts index cc376200ab9..63e35f41bbd 100644 --- a/packages/typespec-python/src/emitter.ts +++ b/packages/typespec-python/src/emitter.ts @@ -12,6 +12,7 @@ import { execFileSync } from "child_process"; import { PythonEmitterOptions, PythonSdkContext } from "./lib.js"; import { emitCodeModel } from "./code-model.js"; import { removeUnderscoresFromNamespace } from "./utils.js"; +import path from "path"; export function getModelsMode(context: SdkContext): "dpg" | "none" { const specifiedModelsMode = context.emitContext.options["models-mode"]; @@ -66,14 +67,14 @@ function createPythonSdkContext( export async function $onEmit(context: EmitContext) { const program = context.program; const sdkContext = createPythonSdkContext(context); - const root = await resolveModuleRoot(program, "@autorest/python", dirname(fileURLToPath(import.meta.url))); + const root = path.join(dirname(fileURLToPath(import.meta.url)), "pygen"); const outputDir = context.emitterOutputDir; const yamlMap = emitCodeModel(sdkContext); addDefaultOptions(sdkContext); const yamlPath = await saveCodeModelAsYaml("typespec-python-yaml-map", yamlMap); const commandArgs = [ `${root}/run-python3.js`, - `${root}/run_cadl.py`, + `${root}/run_tsp.py`, `--output-folder=${outputDir}`, `--cadl-file=${yamlPath}`, ]; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 03b9d39c9be..1529bd349b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -70,9 +70,9 @@ importers: packages/typespec-python: dependencies: - '@autorest/python': - specifier: workspace:^ - version: link:../autorest.python + '@autorest/system-requirements': + specifier: ~1.0.2 + version: 1.0.2 '@typespec/openapi3': specifier: ~0.56.0 version: 0.56.0(@typespec/compiler@0.56.0)(@typespec/http@0.56.0)(@typespec/openapi@0.56.0)(@typespec/versioning@0.56.0) From c03399259541b2fb3799e8c09b2cae05ee3f162b Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 15 May 2024 15:26:34 -0400 Subject: [PATCH 28/75] can copy pygen into dist when building and generate all typespec --- packages/pygen/run_tsp.py | 8 ++++---- packages/typespec-python/package.json | 5 +++-- packages/typespec-python/scripts/post-build.js | 12 ++++++++++++ pnpm-lock.yaml | 15 ++++++++++++--- 4 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 packages/typespec-python/scripts/post-build.js diff --git a/packages/pygen/run_tsp.py b/packages/pygen/run_tsp.py index acc4404a90d..d9bb0d20910 100644 --- a/packages/pygen/run_tsp.py +++ b/packages/pygen/run_tsp.py @@ -34,7 +34,7 @@ breakpoint() # pylint: disable=undefined-variable # run m2r - python_run(venv_context, "autorest.m2r.__init__", command=sys.argv[1:]) - python_run(venv_context, "autorest.preprocess.__init__", command=sys.argv[1:]) - python_run(venv_context, "autorest.codegen.__init__", command=sys.argv[1:]) - python_run(venv_context, "autorest.black.__init__", command=sys.argv[1:]) + python_run(venv_context, "pygen.m2r.__init__", command=sys.argv[1:]) + python_run(venv_context, "pygen.preprocess.__init__", command=sys.argv[1:]) + python_run(venv_context, "pygen.codegen.__init__", command=sys.argv[1:]) + python_run(venv_context, "pygen.black.__init__", command=sys.argv[1:]) diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 3859f9e5007..3b11006016c 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -28,7 +28,7 @@ }, "scripts": { "clean": "rimraf ./dist ./temp", - "build": "tsc -p .", + "build": "tsc -p . && node ./scripts/post-build.js", "watch": "tsc -p . --watch", "test": "mocha", "test-official": "c8 mocha --forbid-only", @@ -59,7 +59,8 @@ "dependencies": { "js-yaml": "~4.1.0", "@typespec/openapi3": "~0.56.0", - "@autorest/system-requirements": "~1.0.2" + "@autorest/system-requirements": "~1.0.2", + "fs-extra": "11.2.0" }, "devDependencies": { "@azure-tools/typespec-azure-resource-manager": "~0.42.0", diff --git a/packages/typespec-python/scripts/post-build.js b/packages/typespec-python/scripts/post-build.js new file mode 100644 index 00000000000..d39d3ad1057 --- /dev/null +++ b/packages/typespec-python/scripts/post-build.js @@ -0,0 +1,12 @@ +import fs from 'fs-extra'; +import {join, dirname} from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)) + +// Define the source and destination directories +const sourceDir = join(__dirname, "..", "..", 'pygen'); +const destDir = join(__dirname, "..", 'dist', "src", "pygen"); + +// Copy the source directory to the destination directory +fs.copySync(sourceDir, destDir); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1529bd349b4..ceb9889d14a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -76,6 +76,9 @@ importers: '@typespec/openapi3': specifier: ~0.56.0 version: 0.56.0(@typespec/compiler@0.56.0)(@typespec/http@0.56.0)(@typespec/openapi@0.56.0)(@typespec/versioning@0.56.0) + fs-extra: + specifier: 11.2.0 + version: 11.2.0 js-yaml: specifier: ~4.1.0 version: 4.1.0 @@ -3491,6 +3494,15 @@ packages: universalify: 2.0.1 dev: true + /fs-extra@11.2.0: + resolution: {integrity: sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==} + engines: {node: '>=14.14'} + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.1 + dev: false + /fs-minipass@2.1.0: resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} engines: {node: '>= 8'} @@ -3693,7 +3705,6 @@ packages: /graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} - dev: true /graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} @@ -4333,7 +4344,6 @@ packages: universalify: 2.0.1 optionalDependencies: graceful-fs: 4.2.11 - dev: true /jsonparse@1.3.1: resolution: {integrity: sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==} @@ -6389,7 +6399,6 @@ packages: /universalify@2.0.1: resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} engines: {node: '>= 10.0.0'} - dev: true /unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} From ead75336a846ccaf40e2bd2f580613e310a267f4 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 15 May 2024 17:09:22 -0400 Subject: [PATCH 29/75] trying to remove packaging files from pygen --- packages/autorest.python/package.json | 3 +- packages/pygen/package.json | 43 ----- packages/pygen/run-python3.js | 175 +++++++++++++++++- packages/typespec-python/package.json | 3 +- .../typespec-python/scripts/post-build.js | 7 +- pnpm-lock.yaml | 62 +++---- 6 files changed, 211 insertions(+), 82 deletions(-) delete mode 100644 packages/pygen/package.json diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 74a16e8b4e7..60265c00f44 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -25,8 +25,7 @@ }, "homepage": "https://github.com/Azure/autorest.python/blob/main/README.md", "dependencies": { - "@autorest/system-requirements": "~1.0.2", - "@azure-tools/python-client-generator-core": "workspace:^" + "@autorest/system-requirements": "~1.0.2" }, "devDependencies": { "@microsoft.azure/autorest.testserver": "^3.3.46", diff --git a/packages/pygen/package.json b/packages/pygen/package.json deleted file mode 100644 index f49bd5706b4..00000000000 --- a/packages/pygen/package.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "@azure-tools/python-client-generator-core", - "version": "0.1.0", - "description": "Core generation library for Python clients", - "scripts": { - "prepare": "node run-python3.js prepare.py", - "start": "node run-python3.js start.py", - "install": "node run-python3.js install.py", - "debug": "node run-python3.js start.py --debug" - }, - "main": "index.js", - "repository": { - "type": "git", - "url": "https://github.com/Azure/autorest.python/tree/main" - }, - "readme": "https://github.com/Azure/autorest.python/blob/main/README.md", - "keywords": [ - "python" - ], - "author": "Microsoft Corporation", - "license": "MIT", - "bugs": { - "url": "https://github.com/Azure/autorest.python/issues" - }, - "homepage": "https://github.com/Azure/autorest.python/blob/main/README.md", - "dependencies": { - "@autorest/system-requirements": "~1.0.2" - }, - "devDependencies": { - "typescript": "~5.1.3" - }, - "files": [ - "pygen/**/*.py", - "pygen/**/*.jinja2", - "setup.py", - "install.py", - "prepare.py", - "venvtools.py", - "run-python3.js", - "requirements.txt", - "run_tsp.py" - ] -} diff --git a/packages/pygen/run-python3.js b/packages/pygen/run-python3.js index 020cbce0a0f..7a309ff8052 100644 --- a/packages/pygen/run-python3.js +++ b/packages/pygen/run-python3.js @@ -7,10 +7,9 @@ // Invoke it like so: "node run-python3.js script.py" const cp = require("child_process"); -const extension = require("@autorest/system-requirements"); async function runPython3(scriptName, ...args) { - const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE" }); + const command = await patchPythonPath(["python", scriptName, ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE" }); cp.execSync(command.join(" "), { stdio: [0, 1, 2] }); @@ -20,3 +19,175 @@ runPython3(...process.argv.slice(2)).catch(err => { console.error(err.toString()); process.exit(1); }); + +/* + * Copied from @autorest/system-requirements + */ + +const PythonRequirement = "python"; +const PRINT_PYTHON_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))"; +const semver_1 = require("semver"); + +const execute = (command, cmdlineargs, options = {}) => { + return new Promise((resolve, reject) => { + const cp = (0, child_process_1.spawn)(command, cmdlineargs, { ...options, stdio: "pipe", shell: true }); + if (options.onCreate) { + options.onCreate(cp); + } + options.onStdOutData ? cp.stdout.on("data", options.onStdOutData) : cp; + options.onStdErrData ? cp.stderr.on("data", options.onStdErrData) : cp; + let err = ""; + let out = ""; + let all = ""; + cp.stderr.on("data", (chunk) => { + err += chunk; + all += chunk; + }); + cp.stdout.on("data", (chunk) => { + out += chunk; + all += chunk; + }); + cp.on("error", (err) => { + reject(err); + }); + cp.on("close", (code, signal) => resolve({ + stdout: out, + stderr: err, + log: all, + error: code ? new Error("Process Failed.") : null, + code, + })); + }); +}; + +const versionIsSatisfied = (version, requirement) => { + const cleanedVersion = semver_1.default.coerce(version); + if (!cleanedVersion) { + throw new Error(`Invalid version ${version}.`); + } + return semver_1.default.satisfies(cleanedVersion, requirement, true); +}; + +/** + * Validate the provided system requirement resolution is satisfying the version requirement if applicable. + * @param resolution Command resolution. + * @param actualVersion Version for that resolution. + * @param requirement Requirement. + * @returns the resolution if it is valid or an @see SystemRequirementError if not. + */ +const validateVersionRequirement = (resolution, actualVersion, requirement) => { + if (!requirement.version) { + return resolution; // No version requirement. + } + try { + if ((0, versionIsSatisfied)(actualVersion, requirement.version)) { + return resolution; + } + return { + ...resolution, + error: true, + message: `'${resolution.command}' version is '${actualVersion}' but doesn't satisfy requirement '${requirement.version}'. Please update.`, + actualVersion: actualVersion, + neededVersion: requirement.version, + }; + } + catch (_a) { + return { + ...resolution, + error: true, + message: `Couldn't parse the version ${actualVersion}. This is not a valid semver version.`, + actualVersion: actualVersion, + neededVersion: requirement.version, + }; + } +}; + +const tryPython = async (requirement, command, additionalArgs = []) => { + const resolution = { + name: PythonRequirement, + command, + additionalArgs: additionalArgs.length > 0 ? additionalArgs : undefined, + }; + try { + const result = await (0, execute)(command, [...additionalArgs, "-c", `"${PRINT_PYTHON_VERSION_SCRIPT}"`]); + return (0, validateVersionRequirement)(resolution, result.stdout.trim(), requirement); + } + catch (e) { + return { + error: true, + ...resolution, + message: `'${command}' command line is not found in the path. Make sure to have it installed.`, + }; + } +}; + +/** + * Returns the path to the executable as asked in the requirement. + * @param requirement System requirement definition. + * @returns If the requirement provide an environment variable for the path returns the value of that environment variable. undefined otherwise. + */ +const getExecutablePath = (requirement) => requirement.environmentVariable && process.env[requirement.environmentVariable]; + +const createPythonErrorMessage = (requirement, errors) => { + var _a; + const versionReq = (_a = requirement.version) !== null && _a !== void 0 ? _a : "*"; + const lines = [ + `Couldn't find a valid python interpreter satisfying the requirement (version: ${versionReq}). Tried:`, + ...errors.map((x) => ` - ${x.command} (${x.message})`), + ]; + return { + error: true, + name: "python", + command: "python", + message: lines.join("\n"), + }; +}; + +const resolvePythonRequirement = async (requirement) => { + var _a; + // Hardcoding AUTOREST_PYTHON_EXE is for backward compatibility + const path = (_a = (0, getExecutablePath)(requirement)) !== null && _a !== void 0 ? _a : process.env["AUTOREST_PYTHON_EXE"]; + if (path) { + return await tryPython(requirement, path); + } + const errors = []; + // On windows try `py` executable with `-3` flag. + if (process.platform === "win32") { + const pyResult = await tryPython(requirement, "py", ["-3"]); + if ("error" in pyResult) { + errors.push(pyResult); + } + else { + return pyResult; + } + } + const python3Result = await tryPython(requirement, "python3"); + if ("error" in python3Result) { + errors.push(python3Result); + } + else { + return python3Result; + } + const pythonResult = await tryPython(requirement, "python"); + if ("error" in pythonResult) { + errors.push(pythonResult); + } + else { + return pythonResult; + } + return createPythonErrorMessage(requirement, errors); +}; + +/** + * @param command list of the command and arguments. First item in array must be a python exe @see KnownPythonExe. (e.g. ["python", "mypythonfile.py"] + * @param requirement + */ +const patchPythonPath = async (command, requirement) => { + var _a; + const [_, ...args] = command; + const resolution = await (0, resolvePythonRequirement)(requirement); + if ("error" in resolution) { + throw new Error(`Failed to find compatible python version. ${resolution.message}`); + } + return [resolution.command, ...((_a = resolution.additionalArgs) !== null && _a !== void 0 ? _a : []), ...args]; +}; diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 3b11006016c..007fac31f70 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -60,7 +60,8 @@ "js-yaml": "~4.1.0", "@typespec/openapi3": "~0.56.0", "@autorest/system-requirements": "~1.0.2", - "fs-extra": "11.2.0" + "fs-extra": "11.2.0", + "semver": "7.6.2" }, "devDependencies": { "@azure-tools/typespec-azure-resource-manager": "~0.42.0", diff --git a/packages/typespec-python/scripts/post-build.js b/packages/typespec-python/scripts/post-build.js index d39d3ad1057..e6483f0d742 100644 --- a/packages/typespec-python/scripts/post-build.js +++ b/packages/typespec-python/scripts/post-build.js @@ -8,5 +8,10 @@ const __dirname = dirname(fileURLToPath(import.meta.url)) const sourceDir = join(__dirname, "..", "..", 'pygen'); const destDir = join(__dirname, "..", 'dist', "src", "pygen"); +// Define the filter function. Don't want to copy node_modules +const filterFunc = (src) => { + return src.indexOf('node_modules') === -1; + }; + // Copy the source directory to the destination directory -fs.copySync(sourceDir, destDir); +fs.copySync(sourceDir, destDir, { filter: filterFunc }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ceb9889d14a..b0fdbf7113e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,9 +47,6 @@ importers: '@autorest/system-requirements': specifier: ~1.0.2 version: 1.0.2 - '@azure-tools/python-client-generator-core': - specifier: workspace:^ - version: link:../pygen devDependencies: '@microsoft.azure/autorest.testserver': specifier: ^3.3.46 @@ -58,16 +55,6 @@ importers: specifier: ~5.1.3 version: 5.1.3 - packages/pygen: - dependencies: - '@autorest/system-requirements': - specifier: ~1.0.2 - version: 1.0.2 - devDependencies: - typescript: - specifier: ~5.1.3 - version: 5.1.6 - packages/typespec-python: dependencies: '@autorest/system-requirements': @@ -82,6 +69,9 @@ importers: js-yaml: specifier: ~4.1.0 version: 4.1.0 + semver: + specifier: 7.6.2 + version: 7.6.2 devDependencies: '@azure-tools/cadl-ranch-expect': specifier: ~0.13.4 @@ -917,7 +907,7 @@ packages: resolution: {integrity: sha512-7kZUAaLscfgbwBQRbvdMYaZOWyMEcPTH/tJjnyAWJ/dvvs9Ef+CERx/qJb9GExJpl1qipaDGn7KqHnFGGixd0w==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} dependencies: - semver: 7.6.0 + semver: 7.6.2 dev: true /@npmcli/git@5.0.6: @@ -930,7 +920,7 @@ packages: proc-log: 4.2.0 promise-inflight: 1.0.1 promise-retry: 2.0.1 - semver: 7.6.0 + semver: 7.6.2 which: 4.0.0 transitivePeerDependencies: - bluebird @@ -960,7 +950,7 @@ packages: json-parse-even-better-errors: 3.0.1 normalize-package-data: 6.0.0 proc-log: 4.2.0 - semver: 7.6.0 + semver: 7.6.2 transitivePeerDependencies: - bluebird dev: true @@ -1463,7 +1453,7 @@ packages: graphemer: 1.4.0 ignore: 5.3.1 natural-compare: 1.4.0 - semver: 7.6.0 + semver: 7.6.2 ts-api-utils: 1.3.0(typescript@5.4.5) typescript: 5.4.5 transitivePeerDependencies: @@ -1552,7 +1542,7 @@ packages: globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 - semver: 7.6.0 + semver: 7.6.2 ts-api-utils: 1.3.0(typescript@5.4.5) typescript: 5.4.5 transitivePeerDependencies: @@ -1574,7 +1564,7 @@ packages: globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.4 - semver: 7.6.0 + semver: 7.6.2 ts-api-utils: 1.3.0(typescript@5.1.3) typescript: 5.1.3 transitivePeerDependencies: @@ -1596,7 +1586,7 @@ packages: globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.4 - semver: 7.6.0 + semver: 7.6.2 ts-api-utils: 1.3.0(typescript@5.4.5) typescript: 5.4.5 transitivePeerDependencies: @@ -1616,7 +1606,7 @@ packages: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.4.5) eslint: 8.57.0 - semver: 7.6.0 + semver: 7.6.2 transitivePeerDependencies: - supports-color - typescript @@ -1635,7 +1625,7 @@ packages: '@typescript-eslint/types': 7.8.0 '@typescript-eslint/typescript-estree': 7.8.0(typescript@5.4.5) eslint: 8.57.0 - semver: 7.6.0 + semver: 7.6.2 transitivePeerDependencies: - supports-color - typescript @@ -1670,7 +1660,7 @@ packages: picocolors: 1.0.0 prettier: 3.2.5 prompts: 2.4.2 - semver: 7.6.0 + semver: 7.6.2 vscode-languageserver: 9.0.1 vscode-languageserver-textdocument: 1.0.11 yaml: 2.4.1 @@ -1690,7 +1680,7 @@ packages: picocolors: 1.0.0 prettier: 3.2.5 prompts: 2.4.2 - semver: 7.6.0 + semver: 7.6.2 vscode-languageserver: 9.0.1 vscode-languageserver-textdocument: 1.0.11 yaml: 2.4.1 @@ -2180,7 +2170,7 @@ packages: /builtins@5.1.0: resolution: {integrity: sha512-SW9lzGTLvWTP1AY8xeAMZimqDrIaSdLQUcVr9DMef51niJ022Ri87SwRRKYm4A6iHfkPaiVUu/Duw2Wc4J7kKg==} dependencies: - semver: 7.6.0 + semver: 7.6.2 dev: true /busboy@1.6.0: @@ -3045,7 +3035,7 @@ packages: read-pkg-up: 7.0.1 regexp-tree: 0.1.27 regjsparser: 0.10.0 - semver: 7.6.0 + semver: 7.6.2 strip-indent: 3.0.0 transitivePeerDependencies: - supports-color @@ -4363,7 +4353,7 @@ packages: lodash.isstring: 4.0.1 lodash.once: 4.1.1 ms: 2.1.3 - semver: 7.6.0 + semver: 7.6.2 dev: true /jsprim@1.4.2: @@ -4526,7 +4516,7 @@ packages: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} engines: {node: '>=10'} dependencies: - semver: 7.6.0 + semver: 7.6.2 dev: true /make-fetch-happen@13.0.0: @@ -4869,7 +4859,7 @@ packages: make-fetch-happen: 13.0.0 nopt: 7.2.0 proc-log: 3.0.0 - semver: 7.6.0 + semver: 7.6.2 tar: 6.2.1 which: 4.0.0 transitivePeerDependencies: @@ -4903,7 +4893,7 @@ packages: dependencies: hosted-git-info: 7.0.1 is-core-module: 2.13.1 - semver: 7.6.0 + semver: 7.6.2 validate-npm-package-license: 3.0.4 dev: true @@ -4923,7 +4913,7 @@ packages: resolution: {integrity: sha512-W29RiK/xtpCGqn6f3ixfRYGk+zRyr+Ew9F2E20BfXxT5/euLdA/Nm7fO7OeTGuAmTs30cpgInyJ0cYe708YTZw==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} dependencies: - semver: 7.6.0 + semver: 7.6.2 dev: true /npm-normalize-package-bin@3.0.1: @@ -4937,7 +4927,7 @@ packages: dependencies: hosted-git-info: 7.0.1 proc-log: 4.2.0 - semver: 7.6.0 + semver: 7.6.2 validate-npm-package-name: 5.0.0 dev: true @@ -4955,7 +4945,7 @@ packages: npm-install-checks: 6.3.0 npm-normalize-package-bin: 3.0.1 npm-package-arg: 11.0.2 - semver: 7.6.0 + semver: 7.6.2 dev: true /npm-registry-fetch@16.2.1: @@ -5709,6 +5699,12 @@ packages: hasBin: true dependencies: lru-cache: 6.0.0 + dev: true + + /semver@7.6.2: + resolution: {integrity: sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==} + engines: {node: '>=10'} + hasBin: true /send@0.18.0: resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==} From 2bbe3ed9ccb6e8802242cb7d79feb7a370e0c638 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 14:14:14 -0400 Subject: [PATCH 30/75] typespec and autorest.python working with vendored pygen --- packages/autorest.python/install.py | 49 ----- packages/autorest.python/package.json | 17 +- packages/autorest.python/run-python3.js | 2 +- .../autorest.python/scripts/copy-pygen.js | 15 ++ packages/autorest.python/scripts/install.py | 58 ++++++ .../autorest.python/{ => scripts}/start.py | 4 +- .../{ => scripts}/venvtools.py | 2 +- packages/pygen/dev_requirements.txt | 4 +- packages/pygen/run-python3.js | 193 ------------------ packages/pygen/{ => scripts}/install.py | 2 +- packages/pygen/{ => scripts}/prepare.py | 2 +- packages/pygen/scripts/run-python3.cjs | 22 ++ packages/pygen/{ => scripts}/run_tsp.py | 2 +- .../pygen/scripts/system-requirements.cjs | 180 ++++++++++++++++ packages/pygen/{ => scripts}/venvtools.py | 2 +- packages/typespec-python/package.json | 3 +- .../{post-build.js => post-install.js} | 2 +- packages/typespec-python/src/emitter.ts | 6 +- pnpm-lock.yaml | 27 ++- 19 files changed, 317 insertions(+), 275 deletions(-) delete mode 100644 packages/autorest.python/install.py create mode 100644 packages/autorest.python/scripts/copy-pygen.js create mode 100644 packages/autorest.python/scripts/install.py rename packages/autorest.python/{ => scripts}/start.py (90%) rename packages/autorest.python/{ => scripts}/venvtools.py (98%) delete mode 100644 packages/pygen/run-python3.js rename packages/pygen/{ => scripts}/install.py (97%) rename packages/pygen/{ => scripts}/prepare.py (96%) create mode 100644 packages/pygen/scripts/run-python3.cjs rename packages/pygen/{ => scripts}/run_tsp.py (97%) create mode 100644 packages/pygen/scripts/system-requirements.cjs rename packages/pygen/{ => scripts}/venvtools.py (98%) rename packages/typespec-python/scripts/{post-build.js => post-install.js} (89%) diff --git a/packages/autorest.python/install.py b/packages/autorest.python/install.py deleted file mode 100644 index fddd5e9fa6c..00000000000 --- a/packages/autorest.python/install.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import sys - -if not sys.version_info >= (3, 8, 0): - raise Exception("Autorest for Python extension requires Python 3.8 at least") - -try: - import pip -except ImportError: - raise Exception("Your Python installation doesn't have pip available") - -try: - import venv -except ImportError: - raise Exception("Your Python installation doesn't have venv available") - - -# Now we have pip and Py >= 3.8, go to work - -from pathlib import Path - -from venvtools import ExtendedEnvBuilder, python_run - -_ROOT_DIR = Path(__file__).parent - - -def main(): - # we use pygen's venv - venv_path = _ROOT_DIR.parent / "pygen" / "venv" - if venv_path.exists(): - env_builder = venv.EnvBuilder(with_pip=True) - venv_context = env_builder.ensure_directories(venv_path) - else: - env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True) - env_builder.create(venv_path) - venv_context = env_builder.context - - python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) - python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) - - -if __name__ == "__main__": - main() diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 60265c00f44..c821ca61ea0 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -3,10 +3,10 @@ "version": "6.13.17", "description": "The Python extension for generators in AutoRest.", "scripts": { - "prepare": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js ./node_modules/@azure-tools/python-client-generator-core/prepare.py", - "start": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js start.py", - "install": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js install.py", - "debug": "node ./node_modules/@azure-tools/python-client-generator-core/run-python3.js start.py --debug" + "prepare": "node ./node_modules/pygen/scripts/run-python3.cjs ./node_modules/pygen/scripts/prepare.py", + "start": "node ./node_modules/pygen/scripts/run-python3.cjs ./scripts/start.py", + "install": "node ./scripts/copy-pygen.js && node ./node_modules/pygen/scripts/run-python3.cjs ./node_modules/pygen/scripts/install.py && node ./node_modules/pygen/scripts/run-python3.cjs ./scripts/install.py", + "debug": "node ./node_modules/pygen/scripts/run-python3.cjs ./scripts/start.py --debug" }, "main": "index.js", "repository": { @@ -25,7 +25,9 @@ }, "homepage": "https://github.com/Azure/autorest.python/blob/main/README.md", "dependencies": { - "@autorest/system-requirements": "~1.0.2" + "@autorest/system-requirements": "~1.0.2", + "fs-extra": "^10.0.0", + "semver": "^7.3.5" }, "devDependencies": { "@microsoft.azure/autorest.testserver": "^3.3.46", @@ -34,12 +36,9 @@ "files": [ "autorest/**/*.py", "autorest/**/*.jinja2", + "scripts/", "setup.py", - "install.py", - "prepare.py", - "start.py", "venvtools.py", - "run-python3.js", "requirements.txt", "run_cadl.py" ] diff --git a/packages/autorest.python/run-python3.js b/packages/autorest.python/run-python3.js index 020cbce0a0f..70ec3bc1e6e 100644 --- a/packages/autorest.python/run-python3.js +++ b/packages/autorest.python/run-python3.js @@ -4,7 +4,7 @@ // path resolution algorithm as AutoRest so that the behavior // is fully consistent (and also supports AUTOREST_PYTHON_EXE). // -// Invoke it like so: "node run-python3.js script.py" +// Invoke it like so: "node run-python3.cjs script.py" const cp = require("child_process"); const extension = require("@autorest/system-requirements"); diff --git a/packages/autorest.python/scripts/copy-pygen.js b/packages/autorest.python/scripts/copy-pygen.js new file mode 100644 index 00000000000..800132d30c6 --- /dev/null +++ b/packages/autorest.python/scripts/copy-pygen.js @@ -0,0 +1,15 @@ +const fs = require('fs-extra'); +const path = require('path'); +const url = require('url'); + +// Define the source and destination directories +const sourceDir = path.join(__dirname, "..", "..", 'pygen'); +const destDir = path.join(__dirname, "..", "node_modules", "pygen"); + +// Define the filter function. Don't want to copy node_modules +const filterFunc = (src) => { + return src.indexOf('node_modules') === -1; + }; + +// Copy the source directory to the destination directory +fs.copySync(sourceDir, destDir, { filter: filterFunc }); diff --git a/packages/autorest.python/scripts/install.py b/packages/autorest.python/scripts/install.py new file mode 100644 index 00000000000..32a5e47f499 --- /dev/null +++ b/packages/autorest.python/scripts/install.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys + +if not sys.version_info >= (3, 8, 0): + raise Exception("Autorest for Python extension requires Python 3.8 at least") + +try: + import pip +except ImportError: + raise Exception("Your Python installation doesn't have pip available") + +try: + import venv +except ImportError: + raise Exception("Your Python installation doesn't have venv available") + + +# Now we have pip and Py >= 3.8, go to work + +from pathlib import Path +import shutil + +from venvtools import python_run + +_ROOT_DIR = Path(__file__).parent.parent + + +def ignore_node_modules(dirname, filenames): + return ["node_modules"] if "node_modules" in filenames else [] + + +def main(): + # Define the source and destination directories + source_dir = _ROOT_DIR.parent / "pygen" + dest_dir = _ROOT_DIR / "node_modules" / "pygen" + + # Copy the source directory to the destination directory + shutil.copytree(source_dir, dest_dir, dirs_exist_ok=True, ignore=ignore_node_modules) + + # we use pygen's venv + venv_path = dest_dir / "venv" + assert venv_path.exists() # Otherwise install was not done + + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + # install autorest.python specific stuff into it + python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) + python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) + + +if __name__ == "__main__": + main() diff --git a/packages/autorest.python/start.py b/packages/autorest.python/scripts/start.py similarity index 90% rename from packages/autorest.python/start.py rename to packages/autorest.python/scripts/start.py index 85053e82ff7..38fb337b59b 100644 --- a/packages/autorest.python/start.py +++ b/packages/autorest.python/scripts/start.py @@ -15,12 +15,12 @@ from venvtools import python_run -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent def main(): # we use pygen's venv - venv_path = _ROOT_DIR.parent / "pygen" / "venv" + venv_path = _ROOT_DIR / "node_modules" / "pygen" / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/packages/autorest.python/venvtools.py b/packages/autorest.python/scripts/venvtools.py similarity index 98% rename from packages/autorest.python/venvtools.py rename to packages/autorest.python/scripts/venvtools.py index 01e1300b1c8..944ff96e36b 100644 --- a/packages/autorest.python/venvtools.py +++ b/packages/autorest.python/scripts/venvtools.py @@ -11,7 +11,7 @@ from pathlib import Path -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent class ExtendedEnvBuilder(venv.EnvBuilder): diff --git a/packages/pygen/dev_requirements.txt b/packages/pygen/dev_requirements.txt index d458756b7d8..652362d3aa5 100644 --- a/packages/pygen/dev_requirements.txt +++ b/packages/pygen/dev_requirements.txt @@ -1,5 +1,5 @@ -e . --r ../../eng/requirements.txt --r ../../eng/dev_requirements.txt +-r ../../../../eng/requirements.txt +-r ../../../../eng/dev_requirements.txt ptvsd==4.3.2 types-PyYAML==6.0.12.8 diff --git a/packages/pygen/run-python3.js b/packages/pygen/run-python3.js deleted file mode 100644 index 7a309ff8052..00000000000 --- a/packages/pygen/run-python3.js +++ /dev/null @@ -1,193 +0,0 @@ -// This script wraps logic in @azure-tools/extension to resolve -// the path to Python 3 so that a Python script file can be run -// from an npm script in package.json. It uses the same Python 3 -// path resolution algorithm as AutoRest so that the behavior -// is fully consistent (and also supports AUTOREST_PYTHON_EXE). -// -// Invoke it like so: "node run-python3.js script.py" - -const cp = require("child_process"); - -async function runPython3(scriptName, ...args) { - const command = await patchPythonPath(["python", scriptName, ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE" }); - cp.execSync(command.join(" "), { - stdio: [0, 1, 2] - }); -} - -runPython3(...process.argv.slice(2)).catch(err => { - console.error(err.toString()); - process.exit(1); -}); - -/* - * Copied from @autorest/system-requirements - */ - -const PythonRequirement = "python"; -const PRINT_PYTHON_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))"; -const semver_1 = require("semver"); - -const execute = (command, cmdlineargs, options = {}) => { - return new Promise((resolve, reject) => { - const cp = (0, child_process_1.spawn)(command, cmdlineargs, { ...options, stdio: "pipe", shell: true }); - if (options.onCreate) { - options.onCreate(cp); - } - options.onStdOutData ? cp.stdout.on("data", options.onStdOutData) : cp; - options.onStdErrData ? cp.stderr.on("data", options.onStdErrData) : cp; - let err = ""; - let out = ""; - let all = ""; - cp.stderr.on("data", (chunk) => { - err += chunk; - all += chunk; - }); - cp.stdout.on("data", (chunk) => { - out += chunk; - all += chunk; - }); - cp.on("error", (err) => { - reject(err); - }); - cp.on("close", (code, signal) => resolve({ - stdout: out, - stderr: err, - log: all, - error: code ? new Error("Process Failed.") : null, - code, - })); - }); -}; - -const versionIsSatisfied = (version, requirement) => { - const cleanedVersion = semver_1.default.coerce(version); - if (!cleanedVersion) { - throw new Error(`Invalid version ${version}.`); - } - return semver_1.default.satisfies(cleanedVersion, requirement, true); -}; - -/** - * Validate the provided system requirement resolution is satisfying the version requirement if applicable. - * @param resolution Command resolution. - * @param actualVersion Version for that resolution. - * @param requirement Requirement. - * @returns the resolution if it is valid or an @see SystemRequirementError if not. - */ -const validateVersionRequirement = (resolution, actualVersion, requirement) => { - if (!requirement.version) { - return resolution; // No version requirement. - } - try { - if ((0, versionIsSatisfied)(actualVersion, requirement.version)) { - return resolution; - } - return { - ...resolution, - error: true, - message: `'${resolution.command}' version is '${actualVersion}' but doesn't satisfy requirement '${requirement.version}'. Please update.`, - actualVersion: actualVersion, - neededVersion: requirement.version, - }; - } - catch (_a) { - return { - ...resolution, - error: true, - message: `Couldn't parse the version ${actualVersion}. This is not a valid semver version.`, - actualVersion: actualVersion, - neededVersion: requirement.version, - }; - } -}; - -const tryPython = async (requirement, command, additionalArgs = []) => { - const resolution = { - name: PythonRequirement, - command, - additionalArgs: additionalArgs.length > 0 ? additionalArgs : undefined, - }; - try { - const result = await (0, execute)(command, [...additionalArgs, "-c", `"${PRINT_PYTHON_VERSION_SCRIPT}"`]); - return (0, validateVersionRequirement)(resolution, result.stdout.trim(), requirement); - } - catch (e) { - return { - error: true, - ...resolution, - message: `'${command}' command line is not found in the path. Make sure to have it installed.`, - }; - } -}; - -/** - * Returns the path to the executable as asked in the requirement. - * @param requirement System requirement definition. - * @returns If the requirement provide an environment variable for the path returns the value of that environment variable. undefined otherwise. - */ -const getExecutablePath = (requirement) => requirement.environmentVariable && process.env[requirement.environmentVariable]; - -const createPythonErrorMessage = (requirement, errors) => { - var _a; - const versionReq = (_a = requirement.version) !== null && _a !== void 0 ? _a : "*"; - const lines = [ - `Couldn't find a valid python interpreter satisfying the requirement (version: ${versionReq}). Tried:`, - ...errors.map((x) => ` - ${x.command} (${x.message})`), - ]; - return { - error: true, - name: "python", - command: "python", - message: lines.join("\n"), - }; -}; - -const resolvePythonRequirement = async (requirement) => { - var _a; - // Hardcoding AUTOREST_PYTHON_EXE is for backward compatibility - const path = (_a = (0, getExecutablePath)(requirement)) !== null && _a !== void 0 ? _a : process.env["AUTOREST_PYTHON_EXE"]; - if (path) { - return await tryPython(requirement, path); - } - const errors = []; - // On windows try `py` executable with `-3` flag. - if (process.platform === "win32") { - const pyResult = await tryPython(requirement, "py", ["-3"]); - if ("error" in pyResult) { - errors.push(pyResult); - } - else { - return pyResult; - } - } - const python3Result = await tryPython(requirement, "python3"); - if ("error" in python3Result) { - errors.push(python3Result); - } - else { - return python3Result; - } - const pythonResult = await tryPython(requirement, "python"); - if ("error" in pythonResult) { - errors.push(pythonResult); - } - else { - return pythonResult; - } - return createPythonErrorMessage(requirement, errors); -}; - -/** - * @param command list of the command and arguments. First item in array must be a python exe @see KnownPythonExe. (e.g. ["python", "mypythonfile.py"] - * @param requirement - */ -const patchPythonPath = async (command, requirement) => { - var _a; - const [_, ...args] = command; - const resolution = await (0, resolvePythonRequirement)(requirement); - if ("error" in resolution) { - throw new Error(`Failed to find compatible python version. ${resolution.message}`); - } - return [resolution.command, ...((_a = resolution.additionalArgs) !== null && _a !== void 0 ? _a : []), ...args]; -}; diff --git a/packages/pygen/install.py b/packages/pygen/scripts/install.py similarity index 97% rename from packages/pygen/install.py rename to packages/pygen/scripts/install.py index 237a982b0c6..ec223c860a5 100644 --- a/packages/pygen/install.py +++ b/packages/pygen/scripts/install.py @@ -27,7 +27,7 @@ from venvtools import ExtendedEnvBuilder, python_run -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent def main(): diff --git a/packages/pygen/prepare.py b/packages/pygen/scripts/prepare.py similarity index 96% rename from packages/pygen/prepare.py rename to packages/pygen/scripts/prepare.py index b7e9b2dc781..2121ac8b3a0 100644 --- a/packages/pygen/prepare.py +++ b/packages/pygen/scripts/prepare.py @@ -16,7 +16,7 @@ from venvtools import python_run -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent def main(): diff --git a/packages/pygen/scripts/run-python3.cjs b/packages/pygen/scripts/run-python3.cjs new file mode 100644 index 00000000000..6059b4cce09 --- /dev/null +++ b/packages/pygen/scripts/run-python3.cjs @@ -0,0 +1,22 @@ +// This script wraps logic in @azure-tools/extension to resolve +// the path to Python 3 so that a Python script file can be run +// from an npm script in package.json. It uses the same Python 3 +// path resolution algorithm as AutoRest so that the behavior +// is fully consistent (and also supports AUTOREST_PYTHON_EXE). +// +// Invoke it like so: "node run-python3.cjs script.py" + +const cp = require("child_process"); +const extension = require("./system-requirements.cjs"); + +async function runPython3(scriptName, ...args) { + const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE" }); + cp.execSync(command.join(" "), { + stdio: [0, 1, 2] + }); + } + + runPython3(...process.argv.slice(2)).catch(err => { + console.error(err.toString()); + process.exit(1); + }); diff --git a/packages/pygen/run_tsp.py b/packages/pygen/scripts/run_tsp.py similarity index 97% rename from packages/pygen/run_tsp.py rename to packages/pygen/scripts/run_tsp.py index d9bb0d20910..311e470ba58 100644 --- a/packages/pygen/run_tsp.py +++ b/packages/pygen/scripts/run_tsp.py @@ -9,7 +9,7 @@ from pathlib import Path from venvtools import python_run -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent _LOGGER = logging.getLogger(__name__) diff --git a/packages/pygen/scripts/system-requirements.cjs b/packages/pygen/scripts/system-requirements.cjs new file mode 100644 index 00000000000..2b7ced2c400 --- /dev/null +++ b/packages/pygen/scripts/system-requirements.cjs @@ -0,0 +1,180 @@ +const semver = require("semver"); +const child_process = require("child_process"); + +/* + * Copied from @autorest/system-requirements + */ + +const PythonRequirement = "python"; +const PRINT_PYTHON_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))"; + + +function execute(command, cmdlineargs, options = {}) { + return new Promise((resolve, reject) => { + const cp = child_process.spawn(command, cmdlineargs, { ...options, stdio: "pipe", shell: true }); + if (options.onCreate) { + options.onCreate(cp); + } + + options.onStdOutData ? cp.stdout.on("data", options.onStdOutData) : cp; + options.onStdErrData ? cp.stderr.on("data", options.onStdErrData) : cp; + + let err = ""; + let out = ""; + let all = ""; + cp.stderr.on("data", (chunk) => { + err += chunk; + all += chunk; + }); + cp.stdout.on("data", (chunk) => { + out += chunk; + all += chunk; + }); + + cp.on("error", (err) => { + reject(err); + }); + cp.on("close", (code, signal) => + resolve({ + stdout: out, + stderr: err, + log: all, + error: code ? new Error("Process Failed.") : null, + code, + }), + ); + }); +}; + +function versionIsSatisfied(version, requirement) { + const cleanedVersion = semver.coerce(version); + if (!cleanedVersion) { + throw new Error(`Invalid version ${version}.`); + } + return semver.satisfies(cleanedVersion, requirement, true); +}; + +/** + * Validate the provided system requirement resolution is satisfying the version requirement if applicable. + * @param resolution Command resolution. + * @param actualVersion Version for that resolution. + * @param requirement Requirement. + * @returns the resolution if it is valid or an @see SystemRequirementError if not. + */ +function validateVersionRequirement(resolution, actualVersion, requirement) { + if (!requirement.version) { + return resolution; // No version requirement. + } + + try { + if (versionIsSatisfied(actualVersion, requirement.version)) { + return resolution; + } + return { + ...resolution, + error: true, + message: `'${resolution.command}' version is '${actualVersion}' but doesn't satisfy requirement '${requirement.version}'. Please update.`, + actualVersion: actualVersion, + neededVersion: requirement.version, + }; + } catch { + return { + ...resolution, + error: true, + message: `Couldn't parse the version ${actualVersion}. This is not a valid semver version.`, + actualVersion: actualVersion, + neededVersion: requirement.version, + }; + } +}; + +async function tryPython(requirement, command, additionalArgs = []) { + const resolution = { + name: PythonRequirement, + command, + additionalArgs: additionalArgs.length > 0 ? additionalArgs : undefined, + }; + + try { + const result = await execute(command, [...additionalArgs, "-c", `"${PRINT_PYTHON_VERSION_SCRIPT}"`]); + return validateVersionRequirement(resolution, result.stdout.trim(), requirement); + } catch (e) { + return { + error: true, + ...resolution, + message: `'${command}' command line is not found in the path. Make sure to have it installed.`, + }; + } +}; + +/** + * Returns the path to the executable as asked in the requirement. + * @param requirement System requirement definition. + * @returns If the requirement provide an environment variable for the path returns the value of that environment variable. undefined otherwise. + */ +function getExecutablePath(requirement) { + return requirement.environmentVariable && process.env[requirement.environmentVariable]; +} + +function createPythonErrorMessage(requirement, errors) { + const versionReq = requirement.version ?? "*"; + const lines = [ + `Couldn't find a valid python interpreter satisfying the requirement (version: ${versionReq}). Tried:`, + ...errors.map((x) => ` - ${x.command} (${x.message})`), + ]; + + return { + error: true, + name: "python", + command: "python", + message: lines.join("\n"), + }; +}; + +async function resolvePythonRequirement(requirement) { + const path = getExecutablePath(requirement) ?? process.env["AUTOREST_PYTHON_EXE"]; + if (path) { + return await tryPython(requirement, path); + } + + const errors = []; + // On windows try `py` executable with `-3` flag. + if (process.platform === "win32") { + const pyResult = await tryPython(requirement, "py", ["-3"]); + if ("error" in pyResult) { + errors.push(pyResult); + } else { + return pyResult; + } + } + + const python3Result = await tryPython(requirement, "python3"); + if ("error" in python3Result) { + errors.push(python3Result); + } else { + return python3Result; + } + + const pythonResult = await tryPython(requirement, "python"); + if ("error" in pythonResult) { + errors.push(pythonResult); + } else { + return pythonResult; + } + + return createPythonErrorMessage(requirement, errors); +}; + +/** + * @param command list of the command and arguments. First item in array must be a python exe @see KnownPythonExe. (e.g. ["python", "mypythonfile.py"] + * @param requirement + */ +async function patchPythonPath(command, requirement) { + const [_, ...args] = command; + const resolution = await resolvePythonRequirement(requirement); + if ("error" in resolution) { + throw new Error(`Failed to find compatible python version. ${resolution.message}`); + } + return [resolution.command, ...(resolution.additionalArgs ?? []), ...args]; +}; +module.exports.patchPythonPath = patchPythonPath; diff --git a/packages/pygen/venvtools.py b/packages/pygen/scripts/venvtools.py similarity index 98% rename from packages/pygen/venvtools.py rename to packages/pygen/scripts/venvtools.py index 01e1300b1c8..944ff96e36b 100644 --- a/packages/pygen/venvtools.py +++ b/packages/pygen/scripts/venvtools.py @@ -11,7 +11,7 @@ from pathlib import Path -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent class ExtendedEnvBuilder(venv.EnvBuilder): diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 007fac31f70..4a93aff75bd 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -27,8 +27,9 @@ "node": ">=14.0.0" }, "scripts": { + "postinstall": "node ./scripts/post-install.js && node ./dist/pygen/scripts/run-python3.cjs ./dist/pygen/scripts/install.py && node ./dist/pygen/scripts/run-python3.cjs ./dist/pygen/scripts/prepare.py", "clean": "rimraf ./dist ./temp", - "build": "tsc -p . && node ./scripts/post-build.js", + "build": "tsc -p .", "watch": "tsc -p . --watch", "test": "mocha", "test-official": "c8 mocha --forbid-only", diff --git a/packages/typespec-python/scripts/post-build.js b/packages/typespec-python/scripts/post-install.js similarity index 89% rename from packages/typespec-python/scripts/post-build.js rename to packages/typespec-python/scripts/post-install.js index e6483f0d742..c4f67cc7249 100644 --- a/packages/typespec-python/scripts/post-build.js +++ b/packages/typespec-python/scripts/post-install.js @@ -6,7 +6,7 @@ const __dirname = dirname(fileURLToPath(import.meta.url)) // Define the source and destination directories const sourceDir = join(__dirname, "..", "..", 'pygen'); -const destDir = join(__dirname, "..", 'dist', "src", "pygen"); +const destDir = join(__dirname, "..", 'dist', "pygen"); // Define the filter function. Don't want to copy node_modules const filterFunc = (src) => { diff --git a/packages/typespec-python/src/emitter.ts b/packages/typespec-python/src/emitter.ts index 63e35f41bbd..785c30a8f1b 100644 --- a/packages/typespec-python/src/emitter.ts +++ b/packages/typespec-python/src/emitter.ts @@ -67,14 +67,14 @@ function createPythonSdkContext( export async function $onEmit(context: EmitContext) { const program = context.program; const sdkContext = createPythonSdkContext(context); - const root = path.join(dirname(fileURLToPath(import.meta.url)), "pygen"); + const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "pygen"); const outputDir = context.emitterOutputDir; const yamlMap = emitCodeModel(sdkContext); addDefaultOptions(sdkContext); const yamlPath = await saveCodeModelAsYaml("typespec-python-yaml-map", yamlMap); const commandArgs = [ - `${root}/run-python3.js`, - `${root}/run_tsp.py`, + `${root}/scripts/run-python3.cjs`, + `${root}/scripts/run_tsp.py`, `--output-folder=${outputDir}`, `--cadl-file=${yamlPath}`, ]; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0fdbf7113e..10174aa8a8b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,6 +47,12 @@ importers: '@autorest/system-requirements': specifier: ~1.0.2 version: 1.0.2 + fs-extra: + specifier: ^10.0.0 + version: 10.1.0 + semver: + specifier: ^7.3.5 + version: 7.6.2 devDependencies: '@microsoft.azure/autorest.testserver': specifier: ^3.3.46 @@ -166,7 +172,7 @@ packages: dependencies: '@azure/logger': 1.0.4 command-exists: 1.2.9 - semver: 7.5.4 + semver: 7.6.2 dev: false /@azure-tools/cadl-ranch-api@0.4.3: @@ -3475,6 +3481,15 @@ packages: engines: {node: '>= 0.6'} dev: true + /fs-extra@10.1.0: + resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} + engines: {node: '>=12'} + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.1 + dev: false + /fs-extra@11.1.1: resolution: {integrity: sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==} engines: {node: '>=14.14'} @@ -4511,6 +4526,7 @@ packages: engines: {node: '>=10'} dependencies: yallist: 4.0.0 + dev: true /make-dir@4.0.0: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} @@ -5685,14 +5701,6 @@ packages: lru-cache: 6.0.0 dev: true - /semver@7.5.4: - resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} - engines: {node: '>=10'} - hasBin: true - dependencies: - lru-cache: 6.0.0 - dev: false - /semver@7.6.0: resolution: {integrity: sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==} engines: {node: '>=10'} @@ -6688,6 +6696,7 @@ packages: /yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: true /yaml@2.4.1: resolution: {integrity: sha512-pIXzoImaqmfOrL7teGUBt/T7ZDnyeGBWyXQBvOVhLkWLN37GXv8NMLK406UY6dS51JfcQHsmcW5cJ441bHg6Lg==} From c1f8423683194cade1287a0f3a1edc4a68ebf1c2 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 17:36:51 -0400 Subject: [PATCH 31/75] update chronus --- .chronus/changes/link_emitters-2024-4-13-14-39-2.md | 4 ++-- .chronus/config.yaml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.chronus/changes/link_emitters-2024-4-13-14-39-2.md b/.chronus/changes/link_emitters-2024-4-13-14-39-2.md index ceda1c8c4ca..91280ecdf58 100644 --- a/.chronus/changes/link_emitters-2024-4-13-14-39-2.md +++ b/.chronus/changes/link_emitters-2024-4-13-14-39-2.md @@ -2,7 +2,7 @@ changeKind: feature packages: - "@autorest/python" - - "@azure-tools/python-client-generator-core" + - "@azure-tools/typespec-python" --- -add package pygen that both autorest.python and typespec-python will rely on \ No newline at end of file +add package pygen that both autorest.python and typespec-python will rely on diff --git a/.chronus/config.yaml b/.chronus/config.yaml index 8001a421ecd..a16058fb0e3 100644 --- a/.chronus/config.yaml +++ b/.chronus/config.yaml @@ -38,7 +38,6 @@ versionPolicies: packages: - "@azure-tools/typespec-python" - "@autorest/python" - - "@azure-tools/python-client-generator-core" changelog: ["@chronus/github/changelog", { repo: "azure/autorest.python" }] From 93b88b2f2ee7109cf29d4bd0d30f59b08fd45727 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 17:40:50 -0400 Subject: [PATCH 32/75] fix version mismatch --- packages/autorest.python/package.json | 4 ++-- packages/typespec-python/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index c821ca61ea0..9139bff2ca3 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -26,8 +26,8 @@ "homepage": "https://github.com/Azure/autorest.python/blob/main/README.md", "dependencies": { "@autorest/system-requirements": "~1.0.2", - "fs-extra": "^10.0.0", - "semver": "^7.3.5" + "fs-extra": "~11.2.0", + "semver": "~7.6.2" }, "devDependencies": { "@microsoft.azure/autorest.testserver": "^3.3.46", diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 36145b00d72..fac552ca549 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -61,7 +61,7 @@ "js-yaml": "~4.1.0", "@typespec/openapi3": "~0.56.0", "@autorest/system-requirements": "~1.0.2", - "fs-extra": "11.2.0", + "fs-extra": "~11.2.0", "semver": "7.6.2" }, "devDependencies": { From 04ada1a1a8a9635e03234807e4f3ef351bb633e6 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 17:44:52 -0400 Subject: [PATCH 33/75] lockfile --- pnpm-lock.yaml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dfb215eab0d..cfdb4089f4e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,10 +48,10 @@ importers: specifier: ~1.0.2 version: 1.0.2 fs-extra: - specifier: ^10.0.0 - version: 10.1.0 + specifier: ~11.2.0 + version: 11.2.0 semver: - specifier: ^7.3.5 + specifier: ~7.6.2 version: 7.6.2 devDependencies: '@microsoft.azure/autorest.testserver': @@ -70,7 +70,7 @@ importers: specifier: ~0.56.0 version: 0.56.0(@typespec/compiler@0.56.0)(@typespec/http@0.56.0)(@typespec/openapi@0.56.0)(@typespec/versioning@0.56.0) fs-extra: - specifier: 11.2.0 + specifier: ~11.2.0 version: 11.2.0 js-yaml: specifier: ~4.1.0 @@ -3481,15 +3481,6 @@ packages: engines: {node: '>= 0.6'} dev: true - /fs-extra@10.1.0: - resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} - engines: {node: '>=12'} - dependencies: - graceful-fs: 4.2.11 - jsonfile: 6.1.0 - universalify: 2.0.1 - dev: false - /fs-extra@11.1.1: resolution: {integrity: sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==} engines: {node: '>=14.14'} From be564c39bb329064865f9a6776e54ca142a63a51 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 17:53:31 -0400 Subject: [PATCH 34/75] add venv folder path --- eng/pipelines/ci-template.yml | 10 ++++++++++ eng/pipelines/ci.yml | 2 ++ eng/pipelines/internal-ci.yml | 1 + eng/pipelines/nightly.yml | 1 + 4 files changed, 14 insertions(+) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index 2b49a88122e..b425f2343a2 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -8,6 +8,7 @@ parameters: regenerate: false checkChange: true updateToLatestTypespec: false + venvFolderPath: "" steps: - checkout: self @@ -80,6 +81,15 @@ steps: displayName: List installed packages workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: | + python -m venv venv + if [[ "$(Agent.OS)" == "Windows_NT" ]]; then + venv\Scripts\activate + else + source venv/bin/activate + displayName: 'Create and activate virtual environment' + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/{{parameter.venvFolderPath}} + - script: pylint ${{parameters.pythonFolderName}} displayName: Pylint workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} diff --git a/eng/pipelines/ci.yml b/eng/pipelines/ci.yml index 3c4e6f81def..0dc6e3da9a4 100644 --- a/eng/pipelines/ci.yml +++ b/eng/pipelines/ci.yml @@ -79,6 +79,7 @@ jobs: pythonCodeChecks: true pythonFolderName: autorest regenerate: true + venvFolderPath: "node_modules/pygen/venv" - script: | pip install pip @@ -178,6 +179,7 @@ jobs: installCadlRanch: true folderName: "typespec-python" regenerate: true + venvFolderPath: "dist/pygen/venv" - template: generated-code-checks-template.yml parameters: diff --git a/eng/pipelines/internal-ci.yml b/eng/pipelines/internal-ci.yml index 4361daa226c..e344f99c44a 100644 --- a/eng/pipelines/internal-ci.yml +++ b/eng/pipelines/internal-ci.yml @@ -21,6 +21,7 @@ steps: installCadlRanch: true folderName: "typespec-python" regenerate: true + venvFolderPath: "dist/pygen/venv" - script: | tox run -e ci diff --git a/eng/pipelines/nightly.yml b/eng/pipelines/nightly.yml index 52e20605056..576b3507814 100644 --- a/eng/pipelines/nightly.yml +++ b/eng/pipelines/nightly.yml @@ -28,6 +28,7 @@ steps: regenerate: true updateToLatestTypespec: true checkChange: false + venvFolderPath: "dist/pygen/venv" - template: generated-code-checks-template.yml parameters: From 3f014764b5959677c174ec18165455ff44dce242 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 17:59:38 -0400 Subject: [PATCH 35/75] fix template --- eng/pipelines/ci-template.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index b425f2343a2..a2f653e6e8d 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -82,13 +82,12 @@ steps: workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - script: | - python -m venv venv if [[ "$(Agent.OS)" == "Windows_NT" ]]; then venv\Scripts\activate else source venv/bin/activate displayName: 'Create and activate virtual environment' - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/{{parameter.venvFolderPath}} + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/{{parameters.venvFolderPath}} - script: pylint ${{parameters.pythonFolderName}} displayName: Pylint From 3c699e7ef3ec110a7852ff7427d1a64b34ff2953 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 18:05:21 -0400 Subject: [PATCH 36/75] add venv folder path with $ sign --- eng/pipelines/ci-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index a2f653e6e8d..30f2ecca07b 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -87,7 +87,7 @@ steps: else source venv/bin/activate displayName: 'Create and activate virtual environment' - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/{{parameters.venvFolderPath}} + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/${{parameters.venvFolderPath}} - script: pylint ${{parameters.pythonFolderName}} displayName: Pylint From 975728ad2e4df6ae18215efd3292b1b30faebb45 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 18:10:48 -0400 Subject: [PATCH 37/75] don't put venv in the venvFolderPath --- eng/pipelines/ci.yml | 4 ++-- eng/pipelines/internal-ci.yml | 2 +- eng/pipelines/nightly.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/ci.yml b/eng/pipelines/ci.yml index 0dc6e3da9a4..472a9a39c43 100644 --- a/eng/pipelines/ci.yml +++ b/eng/pipelines/ci.yml @@ -79,7 +79,7 @@ jobs: pythonCodeChecks: true pythonFolderName: autorest regenerate: true - venvFolderPath: "node_modules/pygen/venv" + venvFolderPath: "node_modules/pygen" - script: | pip install pip @@ -179,7 +179,7 @@ jobs: installCadlRanch: true folderName: "typespec-python" regenerate: true - venvFolderPath: "dist/pygen/venv" + venvFolderPath: "dist/pygen" - template: generated-code-checks-template.yml parameters: diff --git a/eng/pipelines/internal-ci.yml b/eng/pipelines/internal-ci.yml index e344f99c44a..5e30a575439 100644 --- a/eng/pipelines/internal-ci.yml +++ b/eng/pipelines/internal-ci.yml @@ -21,7 +21,7 @@ steps: installCadlRanch: true folderName: "typespec-python" regenerate: true - venvFolderPath: "dist/pygen/venv" + venvFolderPath: "dist/pygen" - script: | tox run -e ci diff --git a/eng/pipelines/nightly.yml b/eng/pipelines/nightly.yml index 576b3507814..e30b37b3acb 100644 --- a/eng/pipelines/nightly.yml +++ b/eng/pipelines/nightly.yml @@ -28,7 +28,7 @@ steps: regenerate: true updateToLatestTypespec: true checkChange: false - venvFolderPath: "dist/pygen/venv" + venvFolderPath: "dist/pygen" - template: generated-code-checks-template.yml parameters: From ef76337f40cb783ce93032a995ba44e21f631d46 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 18:17:33 -0400 Subject: [PATCH 38/75] add fi to venv script --- eng/pipelines/ci-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index 30f2ecca07b..78b6bf24a31 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -86,6 +86,7 @@ steps: venv\Scripts\activate else source venv/bin/activate + fi displayName: 'Create and activate virtual environment' workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/${{parameters.venvFolderPath}} From 866eae1224f580ca54527527a53ef2ca08ff81f2 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 18:21:22 -0400 Subject: [PATCH 39/75] add tilde to semver --- packages/typespec-python/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index fac552ca549..c1ed41a66f7 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -62,7 +62,7 @@ "@typespec/openapi3": "~0.56.0", "@autorest/system-requirements": "~1.0.2", "fs-extra": "~11.2.0", - "semver": "7.6.2" + "semver": "~7.6.2" }, "devDependencies": { "@azure-tools/typespec-azure-resource-manager": "~0.42.0", From 87d109dc48cc7d73ed64955205ef059cee9ccf7c Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 18:27:29 -0400 Subject: [PATCH 40/75] rmeove venv from pipelines, too complicated --- eng/pipelines/ci-template.yml | 14 ++++---------- eng/pipelines/ci.yml | 2 -- eng/pipelines/internal-ci.yml | 1 - eng/pipelines/nightly.yml | 1 - 4 files changed, 4 insertions(+), 14 deletions(-) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index 78b6bf24a31..af3273c85fe 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -8,7 +8,6 @@ parameters: regenerate: false checkChange: true updateToLatestTypespec: false - venvFolderPath: "" steps: - checkout: self @@ -73,6 +72,10 @@ steps: displayName: List installed packages workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pip install -r requirements.txt + displayName: Pip install requirements + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pip install -r dev_requirements.txt displayName: Pip install dev requirements workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} @@ -81,15 +84,6 @@ steps: displayName: List installed packages workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - - script: | - if [[ "$(Agent.OS)" == "Windows_NT" ]]; then - venv\Scripts\activate - else - source venv/bin/activate - fi - displayName: 'Create and activate virtual environment' - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}/${{parameters.venvFolderPath}} - - script: pylint ${{parameters.pythonFolderName}} displayName: Pylint workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} diff --git a/eng/pipelines/ci.yml b/eng/pipelines/ci.yml index 472a9a39c43..3c4e6f81def 100644 --- a/eng/pipelines/ci.yml +++ b/eng/pipelines/ci.yml @@ -79,7 +79,6 @@ jobs: pythonCodeChecks: true pythonFolderName: autorest regenerate: true - venvFolderPath: "node_modules/pygen" - script: | pip install pip @@ -179,7 +178,6 @@ jobs: installCadlRanch: true folderName: "typespec-python" regenerate: true - venvFolderPath: "dist/pygen" - template: generated-code-checks-template.yml parameters: diff --git a/eng/pipelines/internal-ci.yml b/eng/pipelines/internal-ci.yml index 5e30a575439..4361daa226c 100644 --- a/eng/pipelines/internal-ci.yml +++ b/eng/pipelines/internal-ci.yml @@ -21,7 +21,6 @@ steps: installCadlRanch: true folderName: "typespec-python" regenerate: true - venvFolderPath: "dist/pygen" - script: | tox run -e ci diff --git a/eng/pipelines/nightly.yml b/eng/pipelines/nightly.yml index e30b37b3acb..52e20605056 100644 --- a/eng/pipelines/nightly.yml +++ b/eng/pipelines/nightly.yml @@ -28,7 +28,6 @@ steps: regenerate: true updateToLatestTypespec: true checkChange: false - venvFolderPath: "dist/pygen" - template: generated-code-checks-template.yml parameters: From ab7a61fc3c3a88eb696f1daab74b712a5fdcb7c3 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 18:31:17 -0400 Subject: [PATCH 41/75] udpate lockfile --- pnpm-lock.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cfdb4089f4e..d5dd4d86141 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -76,7 +76,7 @@ importers: specifier: ~4.1.0 version: 4.1.0 semver: - specifier: 7.6.2 + specifier: ~7.6.2 version: 7.6.2 devDependencies: '@azure-tools/cadl-ranch-expect': From 1391c6faede3a64278dbc25d2210b1da00d503ec Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 18:39:19 -0400 Subject: [PATCH 42/75] revert dev_requirements.txt back to current in main --- eng/pipelines/ci-template.yml | 4 ---- packages/autorest.python/dev_requirements.txt | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index af3273c85fe..2b49a88122e 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -72,10 +72,6 @@ steps: displayName: List installed packages workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - - script: pip install -r requirements.txt - displayName: Pip install requirements - workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} - - script: pip install -r dev_requirements.txt displayName: Pip install dev requirements workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} diff --git a/packages/autorest.python/dev_requirements.txt b/packages/autorest.python/dev_requirements.txt index d6e1198b1ab..d458756b7d8 100644 --- a/packages/autorest.python/dev_requirements.txt +++ b/packages/autorest.python/dev_requirements.txt @@ -1 +1,5 @@ -e . +-r ../../eng/requirements.txt +-r ../../eng/dev_requirements.txt +ptvsd==4.3.2 +types-PyYAML==6.0.12.8 From 88784ead90d77aebc23718b057d9c5bb0c8b5258 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 18:49:52 -0400 Subject: [PATCH 43/75] remove pygen from reqs list --- eng/requirements.txt | 1 - packages/autorest.python/requirements.txt | 1 - packages/autorest.python/test/unittests/requirements.txt | 1 - 3 files changed, 3 deletions(-) diff --git a/eng/requirements.txt b/eng/requirements.txt index 01a0dd95c02..ec4654ea52f 100644 --- a/eng/requirements.txt +++ b/eng/requirements.txt @@ -3,4 +3,3 @@ pyright==1.1.362 pylint==3.1.0 tox==4.15.0 mypy==1.10.0 --e ../pygen diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index 169d3121b06..6d73aedfbd7 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -1,2 +1 @@ json-rpc==1.14.0 --e ../pygen diff --git a/packages/autorest.python/test/unittests/requirements.txt b/packages/autorest.python/test/unittests/requirements.txt index 26bf5c043ae..e68ca624cb2 100644 --- a/packages/autorest.python/test/unittests/requirements.txt +++ b/packages/autorest.python/test/unittests/requirements.txt @@ -4,4 +4,3 @@ pytest pytest-cov azure-core==1.30.0 -e ../../. --e ../../../pygen From 85bb831fd903d519bf90fd035e12cb7a5a4c0d39 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 18:54:54 -0400 Subject: [PATCH 44/75] add pygen to dev reqs --- packages/autorest.python/dev_requirements.txt | 1 + packages/typespec-python/dev_requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/autorest.python/dev_requirements.txt b/packages/autorest.python/dev_requirements.txt index d458756b7d8..758b713fbb4 100644 --- a/packages/autorest.python/dev_requirements.txt +++ b/packages/autorest.python/dev_requirements.txt @@ -1,5 +1,6 @@ -e . -r ../../eng/requirements.txt -r ../../eng/dev_requirements.txt +-e node_modules/pygen ptvsd==4.3.2 types-PyYAML==6.0.12.8 diff --git a/packages/typespec-python/dev_requirements.txt b/packages/typespec-python/dev_requirements.txt index 3d4fe7e401b..1fcf6600f3a 100644 --- a/packages/typespec-python/dev_requirements.txt +++ b/packages/typespec-python/dev_requirements.txt @@ -1,3 +1,4 @@ -r ../../eng/requirements.txt -r ../../eng/dev_requirements.txt +-e dist/pygen From abab87206690e9d645811fd075bd9c60e2666b8a Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 16 May 2024 19:00:46 -0400 Subject: [PATCH 45/75] add pygen back to unittests reqs --- packages/autorest.python/test/unittests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/autorest.python/test/unittests/requirements.txt b/packages/autorest.python/test/unittests/requirements.txt index e68ca624cb2..e562b3d6ab7 100644 --- a/packages/autorest.python/test/unittests/requirements.txt +++ b/packages/autorest.python/test/unittests/requirements.txt @@ -4,3 +4,4 @@ pytest pytest-cov azure-core==1.30.0 -e ../../. +-e ../../node_modules/pygen From 823092f9cabbf670988e08c0551b0634cf14acf9 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Wed, 29 May 2024 10:02:46 -0400 Subject: [PATCH 46/75] temp --- packages/typespec-python/package.json | 1 - packages/{ => typespec-python}/pygen/LICENSE | 0 packages/{ => typespec-python}/pygen/README.md | 0 packages/{ => typespec-python}/pygen/dev_requirements.txt | 0 packages/{ => typespec-python}/pygen/pygen/__init__.py | 0 packages/{ => typespec-python}/pygen/pygen/_version.py | 0 packages/{ => typespec-python}/pygen/pygen/black/__init__.py | 0 packages/{ => typespec-python}/pygen/pygen/codegen/__init__.py | 0 packages/{ => typespec-python}/pygen/pygen/codegen/_utils.py | 0 .../{ => typespec-python}/pygen/pygen/codegen/models/__init__.py | 0 .../{ => typespec-python}/pygen/pygen/codegen/models/base.py | 0 .../pygen/pygen/codegen/models/base_builder.py | 0 .../{ => typespec-python}/pygen/pygen/codegen/models/client.py | 0 .../pygen/pygen/codegen/models/code_model.py | 0 .../pygen/pygen/codegen/models/combined_type.py | 0 .../pygen/pygen/codegen/models/constant_type.py | 0 .../pygen/pygen/codegen/models/credential_types.py | 0 .../pygen/pygen/codegen/models/dictionary_type.py | 0 .../pygen/pygen/codegen/models/enum_type.py | 0 .../{ => typespec-python}/pygen/pygen/codegen/models/imports.py | 0 .../pygen/pygen/codegen/models/list_type.py | 0 .../pygen/pygen/codegen/models/lro_operation.py | 0 .../pygen/pygen/codegen/models/lro_paging_operation.py | 0 .../pygen/pygen/codegen/models/model_type.py | 0 .../pygen/pygen/codegen/models/operation.py | 0 .../pygen/pygen/codegen/models/operation_group.py | 0 .../pygen/pygen/codegen/models/paging_operation.py | 0 .../pygen/pygen/codegen/models/parameter.py | 0 .../pygen/pygen/codegen/models/parameter_list.py | 0 .../pygen/pygen/codegen/models/primitive_types.py | 0 .../{ => typespec-python}/pygen/pygen/codegen/models/property.py | 0 .../pygen/pygen/codegen/models/request_builder.py | 0 .../pygen/pygen/codegen/models/request_builder_parameter.py | 0 .../{ => typespec-python}/pygen/pygen/codegen/models/response.py | 0 .../{ => typespec-python}/pygen/pygen/codegen/models/utils.py | 0 .../pygen/pygen/codegen/serializers/__init__.py | 0 .../pygen/pygen/codegen/serializers/base_serializer.py | 0 .../pygen/pygen/codegen/serializers/builder_serializer.py | 0 .../pygen/pygen/codegen/serializers/client_serializer.py | 0 .../pygen/pygen/codegen/serializers/enum_serializer.py | 0 .../pygen/pygen/codegen/serializers/general_serializer.py | 0 .../pygen/pygen/codegen/serializers/import_serializer.py | 0 .../pygen/pygen/codegen/serializers/metadata_serializer.py | 0 .../pygen/pygen/codegen/serializers/model_init_serializer.py | 0 .../pygen/pygen/codegen/serializers/model_serializer.py | 0 .../pygen/codegen/serializers/operation_groups_serializer.py | 0 .../pygen/codegen/serializers/operations_init_serializer.py | 0 .../pygen/pygen/codegen/serializers/parameter_serializer.py | 0 .../pygen/pygen/codegen/serializers/patch_serializer.py | 0 .../pygen/codegen/serializers/request_builders_serializer.py | 0 .../pygen/pygen/codegen/serializers/sample_serializer.py | 0 .../pygen/pygen/codegen/serializers/test_serializer.py | 0 .../pygen/pygen/codegen/serializers/types_serializer.py | 0 .../pygen/pygen/codegen/serializers/utils.py | 0 .../pygen/pygen/codegen/templates/client.py.jinja2 | 0 .../pygen/pygen/codegen/templates/client_container.py.jinja2 | 0 .../pygen/pygen/codegen/templates/config.py.jinja2 | 0 .../pygen/pygen/codegen/templates/config_container.py.jinja2 | 0 .../pygen/pygen/codegen/templates/conftest.py.jinja2 | 0 .../pygen/pygen/codegen/templates/enum.py.jinja2 | 0 .../pygen/pygen/codegen/templates/enum_container.py.jinja2 | 0 .../pygen/pygen/codegen/templates/init.py.jinja2 | 0 .../pygen/pygen/codegen/templates/keywords.jinja2 | 0 .../pygen/pygen/codegen/templates/lro_operation.py.jinja2 | 0 .../pygen/pygen/codegen/templates/lro_paging_operation.py.jinja2 | 0 .../pygen/pygen/codegen/templates/macros.jinja2 | 0 .../pygen/pygen/codegen/templates/metadata.json.jinja2 | 0 .../pygen/pygen/codegen/templates/model_base.py.jinja2 | 0 .../pygen/pygen/codegen/templates/model_container.py.jinja2 | 0 .../pygen/pygen/codegen/templates/model_dpg.py.jinja2 | 0 .../pygen/pygen/codegen/templates/model_init.py.jinja2 | 0 .../pygen/pygen/codegen/templates/model_msrest.py.jinja2 | 0 .../pygen/pygen/codegen/templates/operation.py.jinja2 | 0 .../pygen/pygen/codegen/templates/operation_group.py.jinja2 | 0 .../pygen/codegen/templates/operation_groups_container.py.jinja2 | 0 .../pygen/pygen/codegen/templates/operation_tools.jinja2 | 0 .../pygen/codegen/templates/operations_folder_init.py.jinja2 | 0 .../codegen/templates/packaging_templates/CHANGELOG.md.jinja2 | 0 .../pygen/codegen/templates/packaging_templates/LICENSE.jinja2 | 0 .../codegen/templates/packaging_templates/MANIFEST.in.jinja2 | 0 .../pygen/codegen/templates/packaging_templates/README.md.jinja2 | 0 .../templates/packaging_templates/dev_requirements.txt.jinja2 | 0 .../pygen/codegen/templates/packaging_templates/setup.py.jinja2 | 0 .../pygen/pygen/codegen/templates/paging_operation.py.jinja2 | 0 .../pygen/pygen/codegen/templates/patch.py.jinja2 | 0 .../pygen/pygen/codegen/templates/pkgutil_init.py.jinja2 | 0 .../pygen/pygen/codegen/templates/request_builder.py.jinja2 | 0 .../pygen/pygen/codegen/templates/request_builders.py.jinja2 | 0 .../pygen/pygen/codegen/templates/rest_init.py.jinja2 | 0 .../pygen/pygen/codegen/templates/sample.py.jinja2 | 0 .../pygen/pygen/codegen/templates/serialization.py.jinja2 | 0 .../pygen/pygen/codegen/templates/test.py.jinja2 | 0 .../pygen/pygen/codegen/templates/testpreparer.py.jinja2 | 0 .../pygen/pygen/codegen/templates/types.py.jinja2 | 0 .../pygen/pygen/codegen/templates/validation.py.jinja2 | 0 .../pygen/pygen/codegen/templates/vendor.py.jinja2 | 0 .../pygen/pygen/codegen/templates/version.py.jinja2 | 0 packages/{ => typespec-python}/pygen/pygen/m2r/__init__.py | 0 .../{ => typespec-python}/pygen/pygen/postprocess/__init__.py | 0 .../{ => typespec-python}/pygen/pygen/postprocess/get_all.py | 0 .../{ => typespec-python}/pygen/pygen/postprocess/venvtools.py | 0 .../{ => typespec-python}/pygen/pygen/preprocess/__init__.py | 0 packages/{ => typespec-python}/pygen/pygen/preprocess/helpers.py | 0 .../pygen/pygen/preprocess/python_mappings.py | 0 packages/{ => typespec-python}/pygen/pygen/utils.py | 0 packages/{ => typespec-python}/pygen/requirements.txt | 0 packages/{ => typespec-python}/pygen/scripts/install.py | 0 packages/{ => typespec-python}/pygen/scripts/prepare.py | 0 packages/{ => typespec-python}/pygen/scripts/run-python3.cjs | 0 packages/{ => typespec-python}/pygen/scripts/run_tsp.py | 0 .../{ => typespec-python}/pygen/scripts/system-requirements.cjs | 0 packages/{ => typespec-python}/pygen/scripts/venvtools.py | 0 packages/{ => typespec-python}/pygen/setup.py | 0 packages/typespec-python/scripts/post-install.js | 1 + 114 files changed, 1 insertion(+), 1 deletion(-) rename packages/{ => typespec-python}/pygen/LICENSE (100%) rename packages/{ => typespec-python}/pygen/README.md (100%) rename packages/{ => typespec-python}/pygen/dev_requirements.txt (100%) rename packages/{ => typespec-python}/pygen/pygen/__init__.py (100%) rename packages/{ => typespec-python}/pygen/pygen/_version.py (100%) rename packages/{ => typespec-python}/pygen/pygen/black/__init__.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/__init__.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/_utils.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/__init__.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/base.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/base_builder.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/client.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/code_model.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/combined_type.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/constant_type.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/credential_types.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/dictionary_type.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/enum_type.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/imports.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/list_type.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/lro_operation.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/lro_paging_operation.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/model_type.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/operation.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/operation_group.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/paging_operation.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/parameter.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/parameter_list.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/primitive_types.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/property.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/request_builder.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/request_builder_parameter.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/response.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/models/utils.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/__init__.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/base_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/builder_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/client_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/enum_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/general_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/import_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/metadata_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/model_init_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/model_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/operation_groups_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/operations_init_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/parameter_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/patch_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/request_builders_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/sample_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/test_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/types_serializer.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/serializers/utils.py (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/client.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/client_container.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/config.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/config_container.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/conftest.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/enum.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/enum_container.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/init.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/keywords.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/lro_operation.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/lro_paging_operation.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/macros.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/metadata.json.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/model_base.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/model_container.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/model_dpg.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/model_init.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/model_msrest.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/operation.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/operation_group.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/operation_groups_container.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/operation_tools.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/operations_folder_init.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/packaging_templates/README.md.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/packaging_templates/setup.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/paging_operation.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/patch.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/pkgutil_init.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/request_builder.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/request_builders.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/rest_init.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/sample.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/serialization.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/test.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/testpreparer.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/types.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/validation.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/vendor.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/codegen/templates/version.py.jinja2 (100%) rename packages/{ => typespec-python}/pygen/pygen/m2r/__init__.py (100%) rename packages/{ => typespec-python}/pygen/pygen/postprocess/__init__.py (100%) rename packages/{ => typespec-python}/pygen/pygen/postprocess/get_all.py (100%) rename packages/{ => typespec-python}/pygen/pygen/postprocess/venvtools.py (100%) rename packages/{ => typespec-python}/pygen/pygen/preprocess/__init__.py (100%) rename packages/{ => typespec-python}/pygen/pygen/preprocess/helpers.py (100%) rename packages/{ => typespec-python}/pygen/pygen/preprocess/python_mappings.py (100%) rename packages/{ => typespec-python}/pygen/pygen/utils.py (100%) rename packages/{ => typespec-python}/pygen/requirements.txt (100%) rename packages/{ => typespec-python}/pygen/scripts/install.py (100%) rename packages/{ => typespec-python}/pygen/scripts/prepare.py (100%) rename packages/{ => typespec-python}/pygen/scripts/run-python3.cjs (100%) rename packages/{ => typespec-python}/pygen/scripts/run_tsp.py (100%) rename packages/{ => typespec-python}/pygen/scripts/system-requirements.cjs (100%) rename packages/{ => typespec-python}/pygen/scripts/venvtools.py (100%) rename packages/{ => typespec-python}/pygen/setup.py (100%) diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 62b60065f57..dc42f0bf43f 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -27,7 +27,6 @@ "node": ">=14.0.0" }, "scripts": { - "postinstall": "node ./scripts/post-install.js && node ./dist/pygen/scripts/run-python3.cjs ./dist/pygen/scripts/install.py && node ./dist/pygen/scripts/run-python3.cjs ./dist/pygen/scripts/prepare.py", "clean": "rimraf ./dist ./temp", "build": "tsc -p .", "watch": "tsc -p . --watch", diff --git a/packages/pygen/LICENSE b/packages/typespec-python/pygen/LICENSE similarity index 100% rename from packages/pygen/LICENSE rename to packages/typespec-python/pygen/LICENSE diff --git a/packages/pygen/README.md b/packages/typespec-python/pygen/README.md similarity index 100% rename from packages/pygen/README.md rename to packages/typespec-python/pygen/README.md diff --git a/packages/pygen/dev_requirements.txt b/packages/typespec-python/pygen/dev_requirements.txt similarity index 100% rename from packages/pygen/dev_requirements.txt rename to packages/typespec-python/pygen/dev_requirements.txt diff --git a/packages/pygen/pygen/__init__.py b/packages/typespec-python/pygen/pygen/__init__.py similarity index 100% rename from packages/pygen/pygen/__init__.py rename to packages/typespec-python/pygen/pygen/__init__.py diff --git a/packages/pygen/pygen/_version.py b/packages/typespec-python/pygen/pygen/_version.py similarity index 100% rename from packages/pygen/pygen/_version.py rename to packages/typespec-python/pygen/pygen/_version.py diff --git a/packages/pygen/pygen/black/__init__.py b/packages/typespec-python/pygen/pygen/black/__init__.py similarity index 100% rename from packages/pygen/pygen/black/__init__.py rename to packages/typespec-python/pygen/pygen/black/__init__.py diff --git a/packages/pygen/pygen/codegen/__init__.py b/packages/typespec-python/pygen/pygen/codegen/__init__.py similarity index 100% rename from packages/pygen/pygen/codegen/__init__.py rename to packages/typespec-python/pygen/pygen/codegen/__init__.py diff --git a/packages/pygen/pygen/codegen/_utils.py b/packages/typespec-python/pygen/pygen/codegen/_utils.py similarity index 100% rename from packages/pygen/pygen/codegen/_utils.py rename to packages/typespec-python/pygen/pygen/codegen/_utils.py diff --git a/packages/pygen/pygen/codegen/models/__init__.py b/packages/typespec-python/pygen/pygen/codegen/models/__init__.py similarity index 100% rename from packages/pygen/pygen/codegen/models/__init__.py rename to packages/typespec-python/pygen/pygen/codegen/models/__init__.py diff --git a/packages/pygen/pygen/codegen/models/base.py b/packages/typespec-python/pygen/pygen/codegen/models/base.py similarity index 100% rename from packages/pygen/pygen/codegen/models/base.py rename to packages/typespec-python/pygen/pygen/codegen/models/base.py diff --git a/packages/pygen/pygen/codegen/models/base_builder.py b/packages/typespec-python/pygen/pygen/codegen/models/base_builder.py similarity index 100% rename from packages/pygen/pygen/codegen/models/base_builder.py rename to packages/typespec-python/pygen/pygen/codegen/models/base_builder.py diff --git a/packages/pygen/pygen/codegen/models/client.py b/packages/typespec-python/pygen/pygen/codegen/models/client.py similarity index 100% rename from packages/pygen/pygen/codegen/models/client.py rename to packages/typespec-python/pygen/pygen/codegen/models/client.py diff --git a/packages/pygen/pygen/codegen/models/code_model.py b/packages/typespec-python/pygen/pygen/codegen/models/code_model.py similarity index 100% rename from packages/pygen/pygen/codegen/models/code_model.py rename to packages/typespec-python/pygen/pygen/codegen/models/code_model.py diff --git a/packages/pygen/pygen/codegen/models/combined_type.py b/packages/typespec-python/pygen/pygen/codegen/models/combined_type.py similarity index 100% rename from packages/pygen/pygen/codegen/models/combined_type.py rename to packages/typespec-python/pygen/pygen/codegen/models/combined_type.py diff --git a/packages/pygen/pygen/codegen/models/constant_type.py b/packages/typespec-python/pygen/pygen/codegen/models/constant_type.py similarity index 100% rename from packages/pygen/pygen/codegen/models/constant_type.py rename to packages/typespec-python/pygen/pygen/codegen/models/constant_type.py diff --git a/packages/pygen/pygen/codegen/models/credential_types.py b/packages/typespec-python/pygen/pygen/codegen/models/credential_types.py similarity index 100% rename from packages/pygen/pygen/codegen/models/credential_types.py rename to packages/typespec-python/pygen/pygen/codegen/models/credential_types.py diff --git a/packages/pygen/pygen/codegen/models/dictionary_type.py b/packages/typespec-python/pygen/pygen/codegen/models/dictionary_type.py similarity index 100% rename from packages/pygen/pygen/codegen/models/dictionary_type.py rename to packages/typespec-python/pygen/pygen/codegen/models/dictionary_type.py diff --git a/packages/pygen/pygen/codegen/models/enum_type.py b/packages/typespec-python/pygen/pygen/codegen/models/enum_type.py similarity index 100% rename from packages/pygen/pygen/codegen/models/enum_type.py rename to packages/typespec-python/pygen/pygen/codegen/models/enum_type.py diff --git a/packages/pygen/pygen/codegen/models/imports.py b/packages/typespec-python/pygen/pygen/codegen/models/imports.py similarity index 100% rename from packages/pygen/pygen/codegen/models/imports.py rename to packages/typespec-python/pygen/pygen/codegen/models/imports.py diff --git a/packages/pygen/pygen/codegen/models/list_type.py b/packages/typespec-python/pygen/pygen/codegen/models/list_type.py similarity index 100% rename from packages/pygen/pygen/codegen/models/list_type.py rename to packages/typespec-python/pygen/pygen/codegen/models/list_type.py diff --git a/packages/pygen/pygen/codegen/models/lro_operation.py b/packages/typespec-python/pygen/pygen/codegen/models/lro_operation.py similarity index 100% rename from packages/pygen/pygen/codegen/models/lro_operation.py rename to packages/typespec-python/pygen/pygen/codegen/models/lro_operation.py diff --git a/packages/pygen/pygen/codegen/models/lro_paging_operation.py b/packages/typespec-python/pygen/pygen/codegen/models/lro_paging_operation.py similarity index 100% rename from packages/pygen/pygen/codegen/models/lro_paging_operation.py rename to packages/typespec-python/pygen/pygen/codegen/models/lro_paging_operation.py diff --git a/packages/pygen/pygen/codegen/models/model_type.py b/packages/typespec-python/pygen/pygen/codegen/models/model_type.py similarity index 100% rename from packages/pygen/pygen/codegen/models/model_type.py rename to packages/typespec-python/pygen/pygen/codegen/models/model_type.py diff --git a/packages/pygen/pygen/codegen/models/operation.py b/packages/typespec-python/pygen/pygen/codegen/models/operation.py similarity index 100% rename from packages/pygen/pygen/codegen/models/operation.py rename to packages/typespec-python/pygen/pygen/codegen/models/operation.py diff --git a/packages/pygen/pygen/codegen/models/operation_group.py b/packages/typespec-python/pygen/pygen/codegen/models/operation_group.py similarity index 100% rename from packages/pygen/pygen/codegen/models/operation_group.py rename to packages/typespec-python/pygen/pygen/codegen/models/operation_group.py diff --git a/packages/pygen/pygen/codegen/models/paging_operation.py b/packages/typespec-python/pygen/pygen/codegen/models/paging_operation.py similarity index 100% rename from packages/pygen/pygen/codegen/models/paging_operation.py rename to packages/typespec-python/pygen/pygen/codegen/models/paging_operation.py diff --git a/packages/pygen/pygen/codegen/models/parameter.py b/packages/typespec-python/pygen/pygen/codegen/models/parameter.py similarity index 100% rename from packages/pygen/pygen/codegen/models/parameter.py rename to packages/typespec-python/pygen/pygen/codegen/models/parameter.py diff --git a/packages/pygen/pygen/codegen/models/parameter_list.py b/packages/typespec-python/pygen/pygen/codegen/models/parameter_list.py similarity index 100% rename from packages/pygen/pygen/codegen/models/parameter_list.py rename to packages/typespec-python/pygen/pygen/codegen/models/parameter_list.py diff --git a/packages/pygen/pygen/codegen/models/primitive_types.py b/packages/typespec-python/pygen/pygen/codegen/models/primitive_types.py similarity index 100% rename from packages/pygen/pygen/codegen/models/primitive_types.py rename to packages/typespec-python/pygen/pygen/codegen/models/primitive_types.py diff --git a/packages/pygen/pygen/codegen/models/property.py b/packages/typespec-python/pygen/pygen/codegen/models/property.py similarity index 100% rename from packages/pygen/pygen/codegen/models/property.py rename to packages/typespec-python/pygen/pygen/codegen/models/property.py diff --git a/packages/pygen/pygen/codegen/models/request_builder.py b/packages/typespec-python/pygen/pygen/codegen/models/request_builder.py similarity index 100% rename from packages/pygen/pygen/codegen/models/request_builder.py rename to packages/typespec-python/pygen/pygen/codegen/models/request_builder.py diff --git a/packages/pygen/pygen/codegen/models/request_builder_parameter.py b/packages/typespec-python/pygen/pygen/codegen/models/request_builder_parameter.py similarity index 100% rename from packages/pygen/pygen/codegen/models/request_builder_parameter.py rename to packages/typespec-python/pygen/pygen/codegen/models/request_builder_parameter.py diff --git a/packages/pygen/pygen/codegen/models/response.py b/packages/typespec-python/pygen/pygen/codegen/models/response.py similarity index 100% rename from packages/pygen/pygen/codegen/models/response.py rename to packages/typespec-python/pygen/pygen/codegen/models/response.py diff --git a/packages/pygen/pygen/codegen/models/utils.py b/packages/typespec-python/pygen/pygen/codegen/models/utils.py similarity index 100% rename from packages/pygen/pygen/codegen/models/utils.py rename to packages/typespec-python/pygen/pygen/codegen/models/utils.py diff --git a/packages/pygen/pygen/codegen/serializers/__init__.py b/packages/typespec-python/pygen/pygen/codegen/serializers/__init__.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/__init__.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/__init__.py diff --git a/packages/pygen/pygen/codegen/serializers/base_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/base_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/base_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/base_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/builder_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/builder_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/builder_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/builder_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/client_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/client_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/client_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/client_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/enum_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/enum_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/enum_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/enum_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/general_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/general_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/general_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/general_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/import_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/import_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/import_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/import_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/metadata_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/metadata_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/metadata_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/metadata_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/model_init_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/model_init_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/model_init_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/model_init_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/model_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/model_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/model_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/model_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/operation_groups_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/operation_groups_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/operation_groups_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/operation_groups_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/operations_init_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/operations_init_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/operations_init_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/operations_init_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/parameter_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/parameter_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/parameter_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/parameter_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/patch_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/patch_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/patch_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/patch_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/request_builders_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/request_builders_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/request_builders_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/request_builders_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/sample_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/sample_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/sample_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/sample_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/test_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/test_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/test_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/test_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/types_serializer.py b/packages/typespec-python/pygen/pygen/codegen/serializers/types_serializer.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/types_serializer.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/types_serializer.py diff --git a/packages/pygen/pygen/codegen/serializers/utils.py b/packages/typespec-python/pygen/pygen/codegen/serializers/utils.py similarity index 100% rename from packages/pygen/pygen/codegen/serializers/utils.py rename to packages/typespec-python/pygen/pygen/codegen/serializers/utils.py diff --git a/packages/pygen/pygen/codegen/templates/client.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/client.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/client.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/client.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/client_container.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/client_container.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/client_container.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/client_container.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/config.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/config.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/config.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/config.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/config_container.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/config_container.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/config_container.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/config_container.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/conftest.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/conftest.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/conftest.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/conftest.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/enum.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/enum.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/enum.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/enum.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/enum_container.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/enum_container.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/enum_container.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/enum_container.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/init.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/init.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/init.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/init.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/keywords.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/keywords.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/keywords.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/keywords.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/lro_operation.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/lro_operation.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/lro_operation.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/lro_operation.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/lro_paging_operation.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/lro_paging_operation.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/lro_paging_operation.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/lro_paging_operation.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/macros.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/macros.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/macros.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/macros.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/metadata.json.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/metadata.json.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/metadata.json.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/metadata.json.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/model_base.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/model_base.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/model_base.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/model_base.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/model_container.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/model_container.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/model_container.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/model_container.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/model_dpg.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/model_dpg.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/model_dpg.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/model_dpg.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/model_init.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/model_init.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/model_init.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/model_init.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/model_msrest.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/model_msrest.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/model_msrest.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/model_msrest.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/operation.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/operation.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/operation.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/operation.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/operation_group.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/operation_group.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/operation_group.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/operation_group.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/operation_groups_container.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/operation_groups_container.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/operation_groups_container.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/operation_groups_container.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/operation_tools.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/operation_tools.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/operation_tools.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/operation_tools.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/operations_folder_init.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/operations_folder_init.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/operations_folder_init.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/operations_folder_init.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/packaging_templates/README.md.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/README.md.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/packaging_templates/README.md.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/README.md.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/packaging_templates/setup.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/setup.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/packaging_templates/setup.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/setup.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/paging_operation.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/paging_operation.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/paging_operation.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/paging_operation.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/patch.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/patch.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/patch.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/patch.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/pkgutil_init.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/pkgutil_init.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/pkgutil_init.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/pkgutil_init.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/request_builder.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/request_builder.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/request_builder.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/request_builder.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/request_builders.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/request_builders.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/request_builders.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/request_builders.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/rest_init.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/rest_init.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/rest_init.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/rest_init.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/sample.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/sample.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/sample.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/sample.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/serialization.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/serialization.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/serialization.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/serialization.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/test.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/test.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/test.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/test.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/testpreparer.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/testpreparer.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/testpreparer.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/testpreparer.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/types.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/types.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/types.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/types.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/validation.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/validation.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/validation.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/validation.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/vendor.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/vendor.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/vendor.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/vendor.py.jinja2 diff --git a/packages/pygen/pygen/codegen/templates/version.py.jinja2 b/packages/typespec-python/pygen/pygen/codegen/templates/version.py.jinja2 similarity index 100% rename from packages/pygen/pygen/codegen/templates/version.py.jinja2 rename to packages/typespec-python/pygen/pygen/codegen/templates/version.py.jinja2 diff --git a/packages/pygen/pygen/m2r/__init__.py b/packages/typespec-python/pygen/pygen/m2r/__init__.py similarity index 100% rename from packages/pygen/pygen/m2r/__init__.py rename to packages/typespec-python/pygen/pygen/m2r/__init__.py diff --git a/packages/pygen/pygen/postprocess/__init__.py b/packages/typespec-python/pygen/pygen/postprocess/__init__.py similarity index 100% rename from packages/pygen/pygen/postprocess/__init__.py rename to packages/typespec-python/pygen/pygen/postprocess/__init__.py diff --git a/packages/pygen/pygen/postprocess/get_all.py b/packages/typespec-python/pygen/pygen/postprocess/get_all.py similarity index 100% rename from packages/pygen/pygen/postprocess/get_all.py rename to packages/typespec-python/pygen/pygen/postprocess/get_all.py diff --git a/packages/pygen/pygen/postprocess/venvtools.py b/packages/typespec-python/pygen/pygen/postprocess/venvtools.py similarity index 100% rename from packages/pygen/pygen/postprocess/venvtools.py rename to packages/typespec-python/pygen/pygen/postprocess/venvtools.py diff --git a/packages/pygen/pygen/preprocess/__init__.py b/packages/typespec-python/pygen/pygen/preprocess/__init__.py similarity index 100% rename from packages/pygen/pygen/preprocess/__init__.py rename to packages/typespec-python/pygen/pygen/preprocess/__init__.py diff --git a/packages/pygen/pygen/preprocess/helpers.py b/packages/typespec-python/pygen/pygen/preprocess/helpers.py similarity index 100% rename from packages/pygen/pygen/preprocess/helpers.py rename to packages/typespec-python/pygen/pygen/preprocess/helpers.py diff --git a/packages/pygen/pygen/preprocess/python_mappings.py b/packages/typespec-python/pygen/pygen/preprocess/python_mappings.py similarity index 100% rename from packages/pygen/pygen/preprocess/python_mappings.py rename to packages/typespec-python/pygen/pygen/preprocess/python_mappings.py diff --git a/packages/pygen/pygen/utils.py b/packages/typespec-python/pygen/pygen/utils.py similarity index 100% rename from packages/pygen/pygen/utils.py rename to packages/typespec-python/pygen/pygen/utils.py diff --git a/packages/pygen/requirements.txt b/packages/typespec-python/pygen/requirements.txt similarity index 100% rename from packages/pygen/requirements.txt rename to packages/typespec-python/pygen/requirements.txt diff --git a/packages/pygen/scripts/install.py b/packages/typespec-python/pygen/scripts/install.py similarity index 100% rename from packages/pygen/scripts/install.py rename to packages/typespec-python/pygen/scripts/install.py diff --git a/packages/pygen/scripts/prepare.py b/packages/typespec-python/pygen/scripts/prepare.py similarity index 100% rename from packages/pygen/scripts/prepare.py rename to packages/typespec-python/pygen/scripts/prepare.py diff --git a/packages/pygen/scripts/run-python3.cjs b/packages/typespec-python/pygen/scripts/run-python3.cjs similarity index 100% rename from packages/pygen/scripts/run-python3.cjs rename to packages/typespec-python/pygen/scripts/run-python3.cjs diff --git a/packages/pygen/scripts/run_tsp.py b/packages/typespec-python/pygen/scripts/run_tsp.py similarity index 100% rename from packages/pygen/scripts/run_tsp.py rename to packages/typespec-python/pygen/scripts/run_tsp.py diff --git a/packages/pygen/scripts/system-requirements.cjs b/packages/typespec-python/pygen/scripts/system-requirements.cjs similarity index 100% rename from packages/pygen/scripts/system-requirements.cjs rename to packages/typespec-python/pygen/scripts/system-requirements.cjs diff --git a/packages/pygen/scripts/venvtools.py b/packages/typespec-python/pygen/scripts/venvtools.py similarity index 100% rename from packages/pygen/scripts/venvtools.py rename to packages/typespec-python/pygen/scripts/venvtools.py diff --git a/packages/pygen/setup.py b/packages/typespec-python/pygen/setup.py similarity index 100% rename from packages/pygen/setup.py rename to packages/typespec-python/pygen/setup.py diff --git a/packages/typespec-python/scripts/post-install.js b/packages/typespec-python/scripts/post-install.js index c4f67cc7249..a61870d98d6 100644 --- a/packages/typespec-python/scripts/post-install.js +++ b/packages/typespec-python/scripts/post-install.js @@ -3,6 +3,7 @@ import {join, dirname} from 'path'; import { fileURLToPath } from 'url'; const __dirname = dirname(fileURLToPath(import.meta.url)) +console.warn(__dirname); // Define the source and destination directories const sourceDir = join(__dirname, "..", "..", 'pygen'); From 9abe97c3d82a35383a84cc1e92c68d1287713420 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 30 May 2024 14:39:32 -0400 Subject: [PATCH 47/75] continue moving pygen to into typespec-python --- packages/typespec-python/package.json | 3 ++- .../typespec-python/scripts/post-install.js | 18 ------------------ 2 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 packages/typespec-python/scripts/post-install.js diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index dc42f0bf43f..352d859ec92 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -38,7 +38,8 @@ "files": [ "lib/*.cadl", "dist/**", - "!dist/test/**" + "!dist/test/**", + "pygen/**" ], "peerDependencies": { "@azure-tools/typespec-azure-core": ">=0.42.0 <1.0.0", diff --git a/packages/typespec-python/scripts/post-install.js b/packages/typespec-python/scripts/post-install.js deleted file mode 100644 index a61870d98d6..00000000000 --- a/packages/typespec-python/scripts/post-install.js +++ /dev/null @@ -1,18 +0,0 @@ -import fs from 'fs-extra'; -import {join, dirname} from 'path'; -import { fileURLToPath } from 'url'; - -const __dirname = dirname(fileURLToPath(import.meta.url)) -console.warn(__dirname); - -// Define the source and destination directories -const sourceDir = join(__dirname, "..", "..", 'pygen'); -const destDir = join(__dirname, "..", 'dist', "pygen"); - -// Define the filter function. Don't want to copy node_modules -const filterFunc = (src) => { - return src.indexOf('node_modules') === -1; - }; - -// Copy the source directory to the destination directory -fs.copySync(sourceDir, destDir, { filter: filterFunc }); From ffbb90f589b5be66687cc3a7079bd7e7dd46fb00 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 30 May 2024 14:55:18 -0400 Subject: [PATCH 48/75] add dep on typespec-python in autorest.python --- packages/autorest.python/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 9139bff2ca3..1c17b16afce 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -27,7 +27,8 @@ "dependencies": { "@autorest/system-requirements": "~1.0.2", "fs-extra": "~11.2.0", - "semver": "~7.6.2" + "semver": "~7.6.2", + "@autorest/python": "workspace:^" }, "devDependencies": { "@microsoft.azure/autorest.testserver": "^3.3.46", From ecbd17248734d0d8ec46b65f319496bad0130751 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 30 May 2024 15:26:54 -0400 Subject: [PATCH 49/75] can generate autorest --- packages/autorest.python/package.json | 9 ++++----- packages/autorest.python/scripts/install.py | 12 ++---------- packages/autorest.python/scripts/start.py | 2 +- packages/typespec-python/package.json | 4 +++- packages/typespec-python/pygen/dev_requirements.txt | 4 ++-- packages/typespec-python/pygen/scripts/install.py | 2 +- pnpm-lock.yaml | 3 +++ 7 files changed, 16 insertions(+), 20 deletions(-) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 1c17b16afce..4c526b798de 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -3,10 +3,9 @@ "version": "6.13.17", "description": "The Python extension for generators in AutoRest.", "scripts": { - "prepare": "node ./node_modules/pygen/scripts/run-python3.cjs ./node_modules/pygen/scripts/prepare.py", - "start": "node ./node_modules/pygen/scripts/run-python3.cjs ./scripts/start.py", - "install": "node ./scripts/copy-pygen.js && node ./node_modules/pygen/scripts/run-python3.cjs ./node_modules/pygen/scripts/install.py && node ./node_modules/pygen/scripts/run-python3.cjs ./scripts/install.py", - "debug": "node ./node_modules/pygen/scripts/run-python3.cjs ./scripts/start.py --debug" + "start": "node ./node_modules/@azure-tools/typespec-python/pygen/scripts/run-python3.cjs ./scripts/start.py", + "install": "node ./node_modules/@azure-tools/typespec-python/pygen/scripts/run-python3.cjs ./scripts/install.py", + "debug": "node ./node_modules/@azure-tools/typespec-python/pygen/scripts/run-python3.cjs ./scripts/start.py --debug" }, "main": "index.js", "repository": { @@ -28,7 +27,7 @@ "@autorest/system-requirements": "~1.0.2", "fs-extra": "~11.2.0", "semver": "~7.6.2", - "@autorest/python": "workspace:^" + "@azure-tools/typespec-python": "workspace:^" }, "devDependencies": { "@microsoft.azure/autorest.testserver": "^3.3.46", diff --git a/packages/autorest.python/scripts/install.py b/packages/autorest.python/scripts/install.py index 32a5e47f499..02c4db75001 100644 --- a/packages/autorest.python/scripts/install.py +++ b/packages/autorest.python/scripts/install.py @@ -31,20 +31,12 @@ _ROOT_DIR = Path(__file__).parent.parent -def ignore_node_modules(dirname, filenames): - return ["node_modules"] if "node_modules" in filenames else [] - - def main(): # Define the source and destination directories - source_dir = _ROOT_DIR.parent / "pygen" - dest_dir = _ROOT_DIR / "node_modules" / "pygen" - - # Copy the source directory to the destination directory - shutil.copytree(source_dir, dest_dir, dirs_exist_ok=True, ignore=ignore_node_modules) + pygen_dir = _ROOT_DIR / "node_modules" / "@azure-tools/typespec-python" / "pygen" # we use pygen's venv - venv_path = dest_dir / "venv" + venv_path = pygen_dir / "venv" assert venv_path.exists() # Otherwise install was not done env_builder = venv.EnvBuilder(with_pip=True) diff --git a/packages/autorest.python/scripts/start.py b/packages/autorest.python/scripts/start.py index 38fb337b59b..5bf2f97033d 100644 --- a/packages/autorest.python/scripts/start.py +++ b/packages/autorest.python/scripts/start.py @@ -20,7 +20,7 @@ def main(): # we use pygen's venv - venv_path = _ROOT_DIR / "node_modules" / "pygen" / "venv" + venv_path = _ROOT_DIR / "node_modules" / "@azure-tools/typespec-python" / "pygen" / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 352d859ec92..a0b88ddc40f 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -33,7 +33,9 @@ "test": "mocha", "test-official": "c8 mocha --forbid-only", "lint": "eslint . --ext .ts --max-warnings=0", - "lint:fix": "eslint . --fix --ext .ts" + "lint:fix": "eslint . --fix --ext .ts", + "install": "node ./pygen/scripts/run-python3.cjs ./pygen/scripts/install.py", + "prepare": "node ./pygen/scripts/run-python3.cjs ./pygen/scripts/prepare.py" }, "files": [ "lib/*.cadl", diff --git a/packages/typespec-python/pygen/dev_requirements.txt b/packages/typespec-python/pygen/dev_requirements.txt index 652362d3aa5..9859a339b00 100644 --- a/packages/typespec-python/pygen/dev_requirements.txt +++ b/packages/typespec-python/pygen/dev_requirements.txt @@ -1,5 +1,5 @@ -e . --r ../../../../eng/requirements.txt --r ../../../../eng/dev_requirements.txt +-r ../../../eng/requirements.txt +-r ../../../eng/dev_requirements.txt ptvsd==4.3.2 types-PyYAML==6.0.12.8 diff --git a/packages/typespec-python/pygen/scripts/install.py b/packages/typespec-python/pygen/scripts/install.py index ec223c860a5..07c5da68843 100644 --- a/packages/typespec-python/pygen/scripts/install.py +++ b/packages/typespec-python/pygen/scripts/install.py @@ -41,7 +41,7 @@ def main(): venv_context = env_builder.context python_run(venv_context, "pip", ["install", "-U", "pip"]) - python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/requirements.txt"]) python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5d3d4d52e16..78d5138b477 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,6 +50,9 @@ importers: '@autorest/system-requirements': specifier: ~1.0.2 version: 1.0.2 + '@azure-tools/typespec-python': + specifier: workspace:^ + version: link:../typespec-python fs-extra: specifier: ~11.2.0 version: 11.2.0 From 3d341ab6440a43ac0230d85a5d315a117ee99675 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 30 May 2024 15:36:49 -0400 Subject: [PATCH 50/75] can generate typespec locally --- packages/typespec-python/src/emitter.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/typespec-python/src/emitter.ts b/packages/typespec-python/src/emitter.ts index 785c30a8f1b..d7fa1c38b93 100644 --- a/packages/typespec-python/src/emitter.ts +++ b/packages/typespec-python/src/emitter.ts @@ -5,7 +5,7 @@ import { SdkHttpOperation, SdkServiceOperation, } from "@azure-tools/typespec-client-generator-core"; -import { resolveModuleRoot, saveCodeModelAsYaml } from "./external-process.js"; +import { saveCodeModelAsYaml } from "./external-process.js"; import { dirname } from "path"; import { fileURLToPath } from "url"; import { execFileSync } from "child_process"; @@ -67,7 +67,7 @@ function createPythonSdkContext( export async function $onEmit(context: EmitContext) { const program = context.program; const sdkContext = createPythonSdkContext(context); - const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "pygen"); + const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "..", "pygen"); const outputDir = context.emitterOutputDir; const yamlMap = emitCodeModel(sdkContext); addDefaultOptions(sdkContext); From 73b7570293b343a15ebd24b4d97f33d111cde46e Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 30 May 2024 15:59:27 -0400 Subject: [PATCH 51/75] update reqs --- packages/autorest.python/dev_requirements.txt | 2 +- packages/autorest.python/test/unittests/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/autorest.python/dev_requirements.txt b/packages/autorest.python/dev_requirements.txt index 758b713fbb4..399a9c799b3 100644 --- a/packages/autorest.python/dev_requirements.txt +++ b/packages/autorest.python/dev_requirements.txt @@ -1,6 +1,6 @@ -e . -r ../../eng/requirements.txt -r ../../eng/dev_requirements.txt --e node_modules/pygen +-e node_modules/@azure-tools/typespec-python/pygen ptvsd==4.3.2 types-PyYAML==6.0.12.8 diff --git a/packages/autorest.python/test/unittests/requirements.txt b/packages/autorest.python/test/unittests/requirements.txt index e562b3d6ab7..fd3e9c4335c 100644 --- a/packages/autorest.python/test/unittests/requirements.txt +++ b/packages/autorest.python/test/unittests/requirements.txt @@ -4,4 +4,4 @@ pytest pytest-cov azure-core==1.30.0 -e ../../. --e ../../node_modules/pygen +-e ../../node_modules/@azure-tools/typespec-python/pygen From 6578aa66bf01907462bce342c4aadf12a84a8e94 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 3 Jun 2024 14:22:17 -0400 Subject: [PATCH 52/75] duplicate runpython3 script in autorest, fix dev reqs path --- packages/autorest.python/package.json | 6 ++--- .../autorest.python/scripts/run-python3.js | 22 +++++++++++++++++++ packages/typespec-python/dev_requirements.txt | 2 +- 3 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 packages/autorest.python/scripts/run-python3.js diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 4c526b798de..249fd2315bf 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -3,9 +3,9 @@ "version": "6.13.17", "description": "The Python extension for generators in AutoRest.", "scripts": { - "start": "node ./node_modules/@azure-tools/typespec-python/pygen/scripts/run-python3.cjs ./scripts/start.py", - "install": "node ./node_modules/@azure-tools/typespec-python/pygen/scripts/run-python3.cjs ./scripts/install.py", - "debug": "node ./node_modules/@azure-tools/typespec-python/pygen/scripts/run-python3.cjs ./scripts/start.py --debug" + "start": "node ./scripts/run-python3.js ./scripts/start.py", + "install": "node ./scripts/run-python3.js ./scripts/install.py", + "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" }, "main": "index.js", "repository": { diff --git a/packages/autorest.python/scripts/run-python3.js b/packages/autorest.python/scripts/run-python3.js new file mode 100644 index 00000000000..020cbce0a0f --- /dev/null +++ b/packages/autorest.python/scripts/run-python3.js @@ -0,0 +1,22 @@ +// This script wraps logic in @azure-tools/extension to resolve +// the path to Python 3 so that a Python script file can be run +// from an npm script in package.json. It uses the same Python 3 +// path resolution algorithm as AutoRest so that the behavior +// is fully consistent (and also supports AUTOREST_PYTHON_EXE). +// +// Invoke it like so: "node run-python3.js script.py" + +const cp = require("child_process"); +const extension = require("@autorest/system-requirements"); + +async function runPython3(scriptName, ...args) { + const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE" }); + cp.execSync(command.join(" "), { + stdio: [0, 1, 2] + }); +} + +runPython3(...process.argv.slice(2)).catch(err => { + console.error(err.toString()); + process.exit(1); +}); diff --git a/packages/typespec-python/dev_requirements.txt b/packages/typespec-python/dev_requirements.txt index 1fcf6600f3a..d64c38e4493 100644 --- a/packages/typespec-python/dev_requirements.txt +++ b/packages/typespec-python/dev_requirements.txt @@ -1,4 +1,4 @@ -r ../../eng/requirements.txt -r ../../eng/dev_requirements.txt --e dist/pygen +-e pygen From b989020e4b8ab9beeee49b7ab0b9151e1ddf9187 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 3 Jun 2024 15:00:03 -0400 Subject: [PATCH 53/75] move scripts for typespec out of pygen folder --- packages/autorest.python/scripts/install.py | 4 ++-- packages/autorest.python/scripts/start.py | 2 +- packages/typespec-python/package.json | 4 ++-- packages/typespec-python/pygen/dev_requirements.txt | 2 +- packages/typespec-python/{pygen => }/scripts/install.py | 4 ++-- packages/typespec-python/{pygen => }/scripts/prepare.py | 3 +-- packages/typespec-python/{pygen => }/scripts/run-python3.cjs | 0 packages/typespec-python/{pygen => }/scripts/run_tsp.py | 0 .../{pygen => }/scripts/system-requirements.cjs | 0 packages/typespec-python/{pygen => }/scripts/venvtools.py | 0 10 files changed, 9 insertions(+), 10 deletions(-) rename packages/typespec-python/{pygen => }/scripts/install.py (92%) rename packages/typespec-python/{pygen => }/scripts/prepare.py (87%) rename packages/typespec-python/{pygen => }/scripts/run-python3.cjs (100%) rename packages/typespec-python/{pygen => }/scripts/run_tsp.py (100%) rename packages/typespec-python/{pygen => }/scripts/system-requirements.cjs (100%) rename packages/typespec-python/{pygen => }/scripts/venvtools.py (100%) diff --git a/packages/autorest.python/scripts/install.py b/packages/autorest.python/scripts/install.py index 02c4db75001..9a39d8abac1 100644 --- a/packages/autorest.python/scripts/install.py +++ b/packages/autorest.python/scripts/install.py @@ -33,10 +33,10 @@ def main(): # Define the source and destination directories - pygen_dir = _ROOT_DIR / "node_modules" / "@azure-tools/typespec-python" / "pygen" + typespec_python_dir = _ROOT_DIR / "node_modules" / "@azure-tools/typespec-python" # we use pygen's venv - venv_path = pygen_dir / "venv" + venv_path = typespec_python_dir / "venv" assert venv_path.exists() # Otherwise install was not done env_builder = venv.EnvBuilder(with_pip=True) diff --git a/packages/autorest.python/scripts/start.py b/packages/autorest.python/scripts/start.py index 5bf2f97033d..17119175cb5 100644 --- a/packages/autorest.python/scripts/start.py +++ b/packages/autorest.python/scripts/start.py @@ -20,7 +20,7 @@ def main(): # we use pygen's venv - venv_path = _ROOT_DIR / "node_modules" / "@azure-tools/typespec-python" / "pygen" / "venv" + venv_path = _ROOT_DIR / "node_modules" / "@azure-tools/typespec-python" / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 53836e49a15..0dbac6f58dd 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -34,8 +34,8 @@ "test-official": "c8 mocha --forbid-only", "lint": "eslint . --ext .ts --max-warnings=0", "lint:fix": "eslint . --fix --ext .ts", - "install": "node ./pygen/scripts/run-python3.cjs ./pygen/scripts/install.py", - "prepare": "node ./pygen/scripts/run-python3.cjs ./pygen/scripts/prepare.py" + "install": "node ./scripts/run-python3.cjs ./scripts/install.py", + "prepare": "node ./scripts/run-python3.cjs ./scripts/prepare.py" }, "files": [ "lib/*.cadl", diff --git a/packages/typespec-python/pygen/dev_requirements.txt b/packages/typespec-python/pygen/dev_requirements.txt index 9859a339b00..63879b17670 100644 --- a/packages/typespec-python/pygen/dev_requirements.txt +++ b/packages/typespec-python/pygen/dev_requirements.txt @@ -1,4 +1,4 @@ --e . +-e pygen/ -r ../../../eng/requirements.txt -r ../../../eng/dev_requirements.txt ptvsd==4.3.2 diff --git a/packages/typespec-python/pygen/scripts/install.py b/packages/typespec-python/scripts/install.py similarity index 92% rename from packages/typespec-python/pygen/scripts/install.py rename to packages/typespec-python/scripts/install.py index 07c5da68843..7e217cb642a 100644 --- a/packages/typespec-python/pygen/scripts/install.py +++ b/packages/typespec-python/scripts/install.py @@ -41,8 +41,8 @@ def main(): venv_context = env_builder.context python_run(venv_context, "pip", ["install", "-U", "pip"]) - python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/requirements.txt"]) - python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/pygen/requirements.txt"]) + python_run(venv_context, "pip", ["install", "-e", f"{_ROOT_DIR}/pygen"]) if __name__ == "__main__": diff --git a/packages/typespec-python/pygen/scripts/prepare.py b/packages/typespec-python/scripts/prepare.py similarity index 87% rename from packages/typespec-python/pygen/scripts/prepare.py rename to packages/typespec-python/scripts/prepare.py index 2121ac8b3a0..3c728d1e096 100644 --- a/packages/typespec-python/pygen/scripts/prepare.py +++ b/packages/typespec-python/scripts/prepare.py @@ -27,9 +27,8 @@ def main(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) - requirements_path = _ROOT_DIR / "dev_requirements.txt" try: - python_run(venv_context, "pip", ["install", "-r", str(requirements_path)]) + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/pygen/dev_requirements.txt"]) except FileNotFoundError as e: raise ValueError(e.filename) diff --git a/packages/typespec-python/pygen/scripts/run-python3.cjs b/packages/typespec-python/scripts/run-python3.cjs similarity index 100% rename from packages/typespec-python/pygen/scripts/run-python3.cjs rename to packages/typespec-python/scripts/run-python3.cjs diff --git a/packages/typespec-python/pygen/scripts/run_tsp.py b/packages/typespec-python/scripts/run_tsp.py similarity index 100% rename from packages/typespec-python/pygen/scripts/run_tsp.py rename to packages/typespec-python/scripts/run_tsp.py diff --git a/packages/typespec-python/pygen/scripts/system-requirements.cjs b/packages/typespec-python/scripts/system-requirements.cjs similarity index 100% rename from packages/typespec-python/pygen/scripts/system-requirements.cjs rename to packages/typespec-python/scripts/system-requirements.cjs diff --git a/packages/typespec-python/pygen/scripts/venvtools.py b/packages/typespec-python/scripts/venvtools.py similarity index 100% rename from packages/typespec-python/pygen/scripts/venvtools.py rename to packages/typespec-python/scripts/venvtools.py From f8c8a65316aec5f674c6e8206f4911859cf55528 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 3 Jun 2024 15:13:13 -0400 Subject: [PATCH 54/75] add scripts to incldued files --- packages/autorest.python/package.json | 4 +--- packages/typespec-python/package.json | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index d1e7d02a839..990e50a29ca 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -38,8 +38,6 @@ "autorest/**/*.jinja2", "scripts/", "setup.py", - "venvtools.py", - "requirements.txt", - "run_cadl.py" + "requirements.txt" ] } diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 0dbac6f58dd..9e56efa1f43 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -38,10 +38,10 @@ "prepare": "node ./scripts/run-python3.cjs ./scripts/prepare.py" }, "files": [ - "lib/*.cadl", "dist/**", "!dist/test/**", - "pygen/**" + "pygen/**", + "scripts/**" ], "peerDependencies": { "@azure-tools/typespec-azure-core": ">=0.42.0 <1.0.0", From 20b1c7396bbc75f5cc12b60d5857862e4a3a9e00 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Mon, 3 Jun 2024 16:11:41 -0400 Subject: [PATCH 55/75] copyfiles venv into autorest.python --- packages/autorest.python/package.json | 3 +- packages/autorest.python/scripts/install.py | 7 +- packages/autorest.python/scripts/start.py | 3 +- pnpm-lock.yaml | 75 ++++++++++++++------- 4 files changed, 57 insertions(+), 31 deletions(-) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 990e50a29ca..b3da459d4ac 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -4,7 +4,7 @@ "description": "The Python extension for generators in AutoRest.", "scripts": { "start": "node ./scripts/run-python3.js ./scripts/start.py", - "install": "node ./scripts/run-python3.js ./scripts/install.py", + "install": "copyfiles -a /node_modules/@azure-tools/typespec-python/venv/* ./venv && node ./scripts/run-python3.js ./scripts/install.py", "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" }, "main": "index.js", @@ -27,6 +27,7 @@ "@autorest/system-requirements": "~1.0.2", "fs-extra": "~11.2.0", "semver": "~7.6.2", + "copyfiles":"~2.4.1", "@azure-tools/typespec-python": "workspace:^" }, "devDependencies": { diff --git a/packages/autorest.python/scripts/install.py b/packages/autorest.python/scripts/install.py index 9a39d8abac1..b94c76034b7 100644 --- a/packages/autorest.python/scripts/install.py +++ b/packages/autorest.python/scripts/install.py @@ -32,11 +32,8 @@ def main(): - # Define the source and destination directories - typespec_python_dir = _ROOT_DIR / "node_modules" / "@azure-tools/typespec-python" - - # we use pygen's venv - venv_path = typespec_python_dir / "venv" + # we copy pygen's venv into autorest.python + venv_path = _ROOT_DIR / "venv" assert venv_path.exists() # Otherwise install was not done env_builder = venv.EnvBuilder(with_pip=True) diff --git a/packages/autorest.python/scripts/start.py b/packages/autorest.python/scripts/start.py index 17119175cb5..714fc50155f 100644 --- a/packages/autorest.python/scripts/start.py +++ b/packages/autorest.python/scripts/start.py @@ -19,8 +19,7 @@ def main(): - # we use pygen's venv - venv_path = _ROOT_DIR / "node_modules" / "@azure-tools/typespec-python" / "venv" + venv_path = _ROOT_DIR / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 78d5138b477..5e1ada4de0e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,6 +53,9 @@ importers: '@azure-tools/typespec-python': specifier: workspace:^ version: link:../typespec-python + copyfiles: + specifier: ~2.4.1 + version: 2.4.1 fs-extra: specifier: ~11.2.0 version: 11.2.0 @@ -2060,7 +2063,6 @@ packages: /balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - dev: true /basic-auth@2.0.1: resolution: {integrity: sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==} @@ -2133,7 +2135,6 @@ packages: dependencies: balanced-match: 1.0.2 concat-map: 0.0.1 - dev: true /brace-expansion@2.0.1: resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==} @@ -2337,7 +2338,6 @@ packages: string-width: 4.2.3 strip-ansi: 6.0.1 wrap-ansi: 7.0.0 - dev: true /cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} @@ -2413,7 +2413,6 @@ packages: /concat-map@0.0.1: resolution: {integrity: sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=} - dev: true /concat-stream@1.6.2: resolution: {integrity: sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==} @@ -2455,6 +2454,19 @@ packages: engines: {node: '>= 0.6'} dev: true + /copyfiles@2.4.1: + resolution: {integrity: sha512-fereAvAvxDrQDOXybk3Qu3dPbOoKoysFMWtkY3mv5BsL8//OSZVL5DCLYqgRfY5cWirgRzlC+WSrxp6Bo3eNZg==} + hasBin: true + dependencies: + glob: 7.2.3 + minimatch: 3.1.2 + mkdirp: 1.0.4 + noms: 0.0.0 + through2: 2.0.5 + untildify: 4.0.0 + yargs: 16.2.0 + dev: false + /core-js-compat@3.37.0: resolution: {integrity: sha512-vYq4L+T8aS5UuFg4UwDhc7YNRWVeVZwltad9C/jV3R2LgVOpS9BDr7l/WL6BN0dbV3k1XejPTHqqEzJgsa0frA==} dependencies: @@ -2463,7 +2475,6 @@ packages: /core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} - dev: true /cosmiconfig@8.1.3: resolution: {integrity: sha512-/UkO2JKI18b5jVMJUp0lvKFMpa/Gye+ZgZjKD+DGEN9y7NRcf/nK1A0sp67ONmKtnDCNMS44E6jrk0Yc3bDuUw==} @@ -3521,7 +3532,6 @@ packages: /fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} - dev: true /fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} @@ -3649,7 +3659,6 @@ packages: minimatch: 3.1.2 once: 1.4.0 path-is-absolute: 1.0.1 - dev: true /glob@8.1.0: resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} @@ -3908,11 +3917,9 @@ packages: dependencies: once: 1.4.0 wrappy: 1.0.2 - dev: true /inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - dev: true /internal-slot@1.0.5: resolution: {integrity: sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ==} @@ -4207,9 +4214,12 @@ packages: is-docker: 2.2.1 dev: true + /isarray@0.0.1: + resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==} + dev: false + /isarray@1.0.0: resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} - dev: true /isarray@2.0.5: resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} @@ -4607,7 +4617,6 @@ packages: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} dependencies: brace-expansion: 1.1.11 - dev: true /minimatch@5.0.1: resolution: {integrity: sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==} @@ -4735,7 +4744,6 @@ packages: resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} engines: {node: '>=10'} hasBin: true - dev: true /mocha@10.2.0: resolution: {integrity: sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==} @@ -4883,6 +4891,13 @@ packages: resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} dev: true + /noms@0.0.0: + resolution: {integrity: sha512-lNDU9VJaOPxUmXcLb+HQFeUgQQPtMI24Gt6hgfuMHRJgMRHMF/qZ4HJD3GDru4sSw9IQl2jPjAYnQrdIeLbwow==} + dependencies: + inherits: 2.0.4 + readable-stream: 1.0.34 + dev: false + /nopt@7.2.0: resolution: {integrity: sha512-CVDtwCdhYIvnAzFoJ6NJ6dX3oga9/HyciQDnG1vQDjSLMeKLJ4A93ZqYKDrgYSr1FBY5/hMYC+2VCi24pgpkGA==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -5102,7 +5117,6 @@ packages: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} dependencies: wrappy: 1.0.2 - dev: true /one-time@1.0.0: resolution: {integrity: sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==} @@ -5229,7 +5243,6 @@ packages: /path-is-absolute@1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} - dev: true /path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} @@ -5325,7 +5338,6 @@ packages: /process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} - dev: true /process@0.11.10: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} @@ -5463,6 +5475,15 @@ packages: strip-bom: 4.0.0 dev: true + /readable-stream@1.0.34: + resolution: {integrity: sha512-ok1qVCJuRkNmvebYikljxJA/UEsKwLl2nI1OmaqAu4/UE+h0wKCHok4XkL/gvi39OacXvw59RJUOFUkDib2rHg==} + dependencies: + core-util-is: 1.0.2 + inherits: 2.0.4 + isarray: 0.0.1 + string_decoder: 0.10.31 + dev: false + /readable-stream@2.3.8: resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} dependencies: @@ -5473,7 +5494,6 @@ packages: safe-buffer: 5.1.2 string_decoder: 1.1.1 util-deprecate: 1.0.2 - dev: true /readable-stream@3.6.2: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} @@ -5642,7 +5662,6 @@ packages: /safe-buffer@5.1.2: resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} - dev: true /safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -6015,11 +6034,14 @@ packages: es-object-atoms: 1.0.0 dev: true + /string_decoder@0.10.31: + resolution: {integrity: sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==} + dev: false + /string_decoder@1.1.1: resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} dependencies: safe-buffer: 5.1.2 - dev: true /string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} @@ -6141,6 +6163,13 @@ packages: resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} dev: true + /through2@2.0.5: + resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} + dependencies: + readable-stream: 2.3.8 + xtend: 4.0.2 + dev: false + /tightrope@0.1.0: resolution: {integrity: sha512-HHHNYdCAIYwl1jOslQBT455zQpdeSo8/A346xpIb/uuqhSg+tCvYNsP5f11QW+z9VZ3vSX8YIfzTApjjuGH63w==} engines: {node: '>=14'} @@ -6406,6 +6435,11 @@ packages: engines: {node: '>= 0.8'} dev: true + /untildify@4.0.0: + resolution: {integrity: sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==} + engines: {node: '>=8'} + dev: false + /update-browserslist-db@1.0.15(browserslist@4.23.0): resolution: {integrity: sha512-K9HWH62x3/EalU1U6sjSZiylm9C8tgq2mSvshZpqc7QE69RaA2qjhkW2HlNA0tFpEbtyFz7HTqbSdN4MSwUodA==} hasBin: true @@ -6424,7 +6458,6 @@ packages: /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} - dev: true /utils-merge@1.0.1: resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} @@ -6659,7 +6692,6 @@ packages: /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - dev: true /xml2js@0.4.23: resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==} @@ -6685,7 +6717,6 @@ packages: /xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} - dev: true /y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} @@ -6708,7 +6739,6 @@ packages: /yargs-parser@20.2.9: resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} engines: {node: '>=10'} - dev: true /yargs-parser@21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} @@ -6735,7 +6765,6 @@ packages: string-width: 4.2.3 y18n: 5.0.8 yargs-parser: 20.2.9 - dev: true /yargs@17.7.2: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} From 6757221d6c0c62a1956162f426750cfafb0097dc Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 4 Jun 2024 12:03:53 -0400 Subject: [PATCH 56/75] duplicate venv --- packages/autorest.python/dev_requirements.txt | 1 - packages/autorest.python/package.json | 3 +- packages/autorest.python/requirements.txt | 1 + .../autorest.python/scripts/copy-pygen.js | 15 -------- packages/autorest.python/scripts/install.py | 21 ++++++----- packages/autorest.python/scripts/prepare.py | 35 +++++++++++++++++++ 6 files changed, 50 insertions(+), 26 deletions(-) delete mode 100644 packages/autorest.python/scripts/copy-pygen.js create mode 100644 packages/autorest.python/scripts/prepare.py diff --git a/packages/autorest.python/dev_requirements.txt b/packages/autorest.python/dev_requirements.txt index 399a9c799b3..d458756b7d8 100644 --- a/packages/autorest.python/dev_requirements.txt +++ b/packages/autorest.python/dev_requirements.txt @@ -1,6 +1,5 @@ -e . -r ../../eng/requirements.txt -r ../../eng/dev_requirements.txt --e node_modules/@azure-tools/typespec-python/pygen ptvsd==4.3.2 types-PyYAML==6.0.12.8 diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index b3da459d4ac..98fcc88d83d 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -3,8 +3,9 @@ "version": "6.13.18", "description": "The Python extension for generators in AutoRest.", "scripts": { + "prepare": "node ./scripts/run-python3.js ./scripts/prepare.py", "start": "node ./scripts/run-python3.js ./scripts/start.py", - "install": "copyfiles -a /node_modules/@azure-tools/typespec-python/venv/* ./venv && node ./scripts/run-python3.js ./scripts/install.py", + "install": "node ./scripts/run-python3.js ./scripts/install.py", "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" }, "main": "index.js", diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index 6d73aedfbd7..3b9005eae1d 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -1 +1,2 @@ json-rpc==1.14.0 +-e ./node_modules/@azure-tools/typespec-python/pygen diff --git a/packages/autorest.python/scripts/copy-pygen.js b/packages/autorest.python/scripts/copy-pygen.js deleted file mode 100644 index 800132d30c6..00000000000 --- a/packages/autorest.python/scripts/copy-pygen.js +++ /dev/null @@ -1,15 +0,0 @@ -const fs = require('fs-extra'); -const path = require('path'); -const url = require('url'); - -// Define the source and destination directories -const sourceDir = path.join(__dirname, "..", "..", 'pygen'); -const destDir = path.join(__dirname, "..", "node_modules", "pygen"); - -// Define the filter function. Don't want to copy node_modules -const filterFunc = (src) => { - return src.indexOf('node_modules') === -1; - }; - -// Copy the source directory to the destination directory -fs.copySync(sourceDir, destDir, { filter: filterFunc }); diff --git a/packages/autorest.python/scripts/install.py b/packages/autorest.python/scripts/install.py index b94c76034b7..f0d2eddee93 100644 --- a/packages/autorest.python/scripts/install.py +++ b/packages/autorest.python/scripts/install.py @@ -24,23 +24,26 @@ # Now we have pip and Py >= 3.8, go to work from pathlib import Path -import shutil -from venvtools import python_run +from venvtools import ExtendedEnvBuilder, python_run _ROOT_DIR = Path(__file__).parent.parent def main(): - # we copy pygen's venv into autorest.python venv_path = _ROOT_DIR / "venv" - assert venv_path.exists() # Otherwise install was not done - env_builder = venv.EnvBuilder(with_pip=True) - venv_context = env_builder.ensure_directories(venv_path) - # install autorest.python specific stuff into it - python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) - python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) + if venv_path.exists(): + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + else: + env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True) + env_builder.create(venv_path) + venv_context = env_builder.context + + python_run(venv_context, "pip", ["install", "-U", "pip"]) + python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) + python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) if __name__ == "__main__": diff --git a/packages/autorest.python/scripts/prepare.py b/packages/autorest.python/scripts/prepare.py new file mode 100644 index 00000000000..4593c828c52 --- /dev/null +++ b/packages/autorest.python/scripts/prepare.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys + +if not sys.version_info >= (3, 8, 0): + raise Exception("Autorest for Python extension requires Python 3.8 at least") + +from pathlib import Path +import venv + +from venvtools import python_run + +_ROOT_DIR = Path(__file__).parent.parent + + +def main(): + venv_path = _ROOT_DIR / "venv" + venv_prexists = venv_path.exists() + + assert venv_prexists # Otherwise install was not done + + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + requirements_path = _ROOT_DIR / "dev_requirements.txt" + + python_run(venv_context, "pip", ["install", "-r", str(requirements_path)]) + + +if __name__ == "__main__": + main() From 40aec679c1d67409d0f5da5b2c180bdd12d47a19 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 4 Jun 2024 13:36:51 -0400 Subject: [PATCH 57/75] move venv into pygen folder --- packages/typespec-python/scripts/install.py | 2 +- packages/typespec-python/scripts/prepare.py | 2 +- packages/typespec-python/scripts/run_tsp.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/typespec-python/scripts/install.py b/packages/typespec-python/scripts/install.py index 7e217cb642a..83ad20445e7 100644 --- a/packages/typespec-python/scripts/install.py +++ b/packages/typespec-python/scripts/install.py @@ -31,7 +31,7 @@ def main(): - venv_path = _ROOT_DIR / "venv" + venv_path = _ROOT_DIR / "pygen" / "venv" if venv_path.exists(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) diff --git a/packages/typespec-python/scripts/prepare.py b/packages/typespec-python/scripts/prepare.py index 3c728d1e096..fe29b6da616 100644 --- a/packages/typespec-python/scripts/prepare.py +++ b/packages/typespec-python/scripts/prepare.py @@ -20,7 +20,7 @@ def main(): - venv_path = _ROOT_DIR / "venv" + venv_path = _ROOT_DIR / "pygen" / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/packages/typespec-python/scripts/run_tsp.py b/packages/typespec-python/scripts/run_tsp.py index 311e470ba58..5365d316bb8 100644 --- a/packages/typespec-python/scripts/run_tsp.py +++ b/packages/typespec-python/scripts/run_tsp.py @@ -14,7 +14,7 @@ _LOGGER = logging.getLogger(__name__) if __name__ == "__main__": - venv_path = _ROOT_DIR / "venv" + venv_path = _ROOT_DIR / "pygen" / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done From de79d1e179de412d4a80413c2f29254e520edb43 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 4 Jun 2024 13:47:42 -0400 Subject: [PATCH 58/75] can generate typespec --- packages/typespec-python/scripts/venvtools.py | 2 +- packages/typespec-python/src/emitter.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/typespec-python/scripts/venvtools.py b/packages/typespec-python/scripts/venvtools.py index 944ff96e36b..ecddd2fc0bf 100644 --- a/packages/typespec-python/scripts/venvtools.py +++ b/packages/typespec-python/scripts/venvtools.py @@ -73,7 +73,7 @@ def python_run(venv_context, module, command=None, *, additional_dir="."): print("Executing: {}".format(" ".join(cmd_line))) subprocess.run( cmd_line, - cwd=_ROOT_DIR / additional_dir, + cwd=_ROOT_DIR / "pygen" / additional_dir, check=True, ) except subprocess.CalledProcessError as err: diff --git a/packages/typespec-python/src/emitter.ts b/packages/typespec-python/src/emitter.ts index d7fa1c38b93..bbe4b8d2b77 100644 --- a/packages/typespec-python/src/emitter.ts +++ b/packages/typespec-python/src/emitter.ts @@ -67,7 +67,7 @@ function createPythonSdkContext( export async function $onEmit(context: EmitContext) { const program = context.program; const sdkContext = createPythonSdkContext(context); - const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", "..", "pygen"); + const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const outputDir = context.emitterOutputDir; const yamlMap = emitCodeModel(sdkContext); addDefaultOptions(sdkContext); From 856d0b2c393d4a29405c5a0d431cda0f83bf7b3a Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 4 Jun 2024 13:54:56 -0400 Subject: [PATCH 59/75] fix installation for tsp and try to copy pygen into autorest.python --- packages/autorest.python/package.json | 3 ++- packages/typespec-python/pygen/dev_requirements.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 98fcc88d83d..025030c6524 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -5,7 +5,8 @@ "scripts": { "prepare": "node ./scripts/run-python3.js ./scripts/prepare.py", "start": "node ./scripts/run-python3.js ./scripts/start.py", - "install": "node ./scripts/run-python3.js ./scripts/install.py", + "copy-pygen": "copyfiles -a ./node_modules/@azure-tools/typespec-python ./pygen", + "install": "npm run copy-pygen && node ./scripts/run-python3.js ./scripts/install.py", "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" }, "main": "index.js", diff --git a/packages/typespec-python/pygen/dev_requirements.txt b/packages/typespec-python/pygen/dev_requirements.txt index 63879b17670..9859a339b00 100644 --- a/packages/typespec-python/pygen/dev_requirements.txt +++ b/packages/typespec-python/pygen/dev_requirements.txt @@ -1,4 +1,4 @@ --e pygen/ +-e . -r ../../../eng/requirements.txt -r ../../../eng/dev_requirements.txt ptvsd==4.3.2 From ee0722cf5e7e79263cfcd4bc6ca47743ac613eca Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 4 Jun 2024 14:11:20 -0400 Subject: [PATCH 60/75] use own script for copying pygen --- packages/autorest.python/README.md | 220 +----------------- packages/autorest.python/dev_requirements.txt | 4 +- packages/autorest.python/package.json | 2 +- packages/autorest.python/requirements.txt | 14 +- .../autorest.python/scripts/copy-pygen.js | 10 + packages/autorest.python/setup.py | 8 +- 6 files changed, 30 insertions(+), 228 deletions(-) create mode 100644 packages/autorest.python/scripts/copy-pygen.js diff --git a/packages/autorest.python/README.md b/packages/autorest.python/README.md index 671c0924699..3d8a65a8919 100644 --- a/packages/autorest.python/README.md +++ b/packages/autorest.python/README.md @@ -1,219 +1 @@ -# Generating with Autorest for Python - -See [here](https://github.com/Azure/autorest.python/wiki/Generating-with-autorest-for-python-v5.0.0) for Python-specific docs, and [here] for general docs - -# Contributing - -This project welcomes contributions and suggestions. Most contributions require you to agree to a -Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit https://cla.microsoft.com. - -When you submit a pull request, a CLA-bot will automatically determine whether you need to provide -a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions -provided by the bot. You will only need to do this once across all repos using our CLA. - -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - -### Autorest plugin configuration - -- Please don't edit this section unless you're re-configuring how the powershell extension plugs in to AutoRest - AutoRest needs the below config to pick this up as a plug-in - see https://github.com/Azure/autorest/blob/master/docs/developer/architecture/AutoRest-extension.md - -#### Python code gen - -```yaml !$(multiapiscript) && !$(multiclientscript) -# default values for version tolerant and black -black: true -``` - -```yaml !$(low-level-client) -version-tolerant: true -``` - -```yaml !$(low-level-client) && !$(version-tolerant) -modelerfour: - group-parameters: true - flatten-models: true - flatten-payloads: true -``` - -```yaml $(postprocess) -allow-no-input: true -``` - -```yaml !$(multiapiscript) && !$(multiclientscript) -pass-thru: - - model-deduplicator - - subset-reducer -version: ~3.8.1 -use-extension: - "@autorest/modelerfour": ~4.23.5 - -modelerfour: - resolve-schema-name-collisons: true - always-create-content-type-parameter: true - multiple-request-parameter-flattening: false - naming: - parameter: snakecase - property: snakecase - operation: snakecase - operationGroup: pascalcase - choice: pascalcase - choiceValue: uppercase - constant: snakecase - constantParameter: snakecase - type: pascalcase - local: _ + snakecase - global: snakecase - preserve-uppercase-max-length: 6 - override: - $host: $host - base64: base64 - IncludeAPIs: include_apis - -pipeline: - python: - # Doesn't process anything, just makes it so that the 'python:' config section is loaded and available for the next plugins. - pass-thru: true - input: modelerfour/identity - - python/m2r: - input: python - - python/m4reformatter: - input: python/m2r - - python/preprocess: - input: python/m4reformatter - - python/codegen: - input: python/preprocess - output-artifact: python-files - - python/codegen/emitter: - input: codegen - scope: scope-codegen/emitter - -scope-codegen/emitter: - input-artifact: python-files - output-uri-expr: $key - -output-artifact: python-files -``` - -# Multiapi script pipeline - -```yaml $(multiapiscript) -pipeline: - python/multiapiscript: - scope: multiapiscript - output-artifact: python-files - - python/multiapiscript/emitter: - input: multiapiscript - scope: scope-multiapiscript/emitter - -scope-multiapiscript/emitter: - input-artifact: python-files - output-uri-expr: $key - -output-artifact: python-files -``` - -# Black script pipeline - -```yaml $(black) -pipeline: - python/black: - scope: black - input: python/codegen - output-artifact: python-files - - python/black/emitter: - input: black - scope: scope-black/emitter - -scope-black/emitter: - input-artifact: python-files - output-uri-expr: $key - -output-artifact: python-files -``` - -# Post-process customized code for mypy pipeline - -```yaml $(postprocess) -pipeline: - python/postprocess: - scope: postprocess - output-artifact: python-files - - python/postprocess/emitter: - input: postprocess - scope: scope-postprocess/emitter - -scope-postprocess/emitter: - input-artifact: python-files - output-uri-expr: $key - -output-artifact: python-files -``` - -# Multi Client pipeline - -```yaml $(multiclientscript) -pipeline: - python/multiclientscript: - scope: multiclientscript - output-artifact: python-files - - python/multiclientscript/emitter: - input: multiclientscript - scope: scope-multiclientscript/emitter - -scope-multiclientscript/emitter: - input-artifact: python-files - output-uri-expr: $key - -output-artifact: python-files -``` - -# Help - -```yaml -help-content: - python: # type: Help as defined in autorest-core/help.ts - activationScope: python - categoryFriendlyName: Python Generator - settings: - - key: python-sdks-folder - description: The path to the root directory of your azure-sdk-for-python clone. Be sure to note that we include `sdk` in the folder path. - - key: black - description: Runs black formatting on your generated files. Defaults to `false`. - type: string - - key: basic-setup-py - description: Whether to generate a build script for setuptools to package your SDK. Defaults to `false`, generally not suggested if you are going to wrap the generated code - type: bool - - key: multiapi - description: Whether to generate a multiapi client. - type: bool - - key: default-api - description: In the case of `--multiapi`, you can override the default service API version with this flag. If not specified, we use the latest GA service version as the default API. - type: string - - key: no-namespace-folders - description: Specify if you don't want pkgutil-style namespace folders. Defaults to `false`. - type: bool - - key: credential-default-policy-type - description: Specify the default credential policy (authentication policy) for your client. Use in conjunction with `--add-credential`. Currently only supports `BearerTokenCredentialPolicy`, `ARMChallengeAuthenticationPolicy` and `AzureKeyCredentialPolicy`. Default value is `BearerTokenCredentialPolicy`(data-plan)/`ARMChallengeAuthenticationPolicy`(mgmt-plan). `--credential-scopes` is tied with `BearerTokenCredentialPolicy` and `ARMChallengeAuthenticationPolicy`, do not pass them in if you want `AzureKeyCredentialPolicy`. - type: string - - key: credential-key-header-name - description: The name of the header which will pass the credential. Use if you have `--credential-default-policy-type` set to `AzureKeyCredentialPolicy`. For example, if generating cognitive services code, you might use `--credential-key-header-name=Ocp-Apim-Subscription-Key` - type: string -``` - - - -[python_docs]: https://github.com/Azure/autorest.python/tree/main/docs/readme.md -[main_docs]: https://github.com/Azure/autorest/tree/master/docs +# Core Library for Python Generation diff --git a/packages/autorest.python/dev_requirements.txt b/packages/autorest.python/dev_requirements.txt index d458756b7d8..9859a339b00 100644 --- a/packages/autorest.python/dev_requirements.txt +++ b/packages/autorest.python/dev_requirements.txt @@ -1,5 +1,5 @@ -e . --r ../../eng/requirements.txt --r ../../eng/dev_requirements.txt +-r ../../../eng/requirements.txt +-r ../../../eng/dev_requirements.txt ptvsd==4.3.2 types-PyYAML==6.0.12.8 diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 025030c6524..9a2778ffd09 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -5,7 +5,7 @@ "scripts": { "prepare": "node ./scripts/run-python3.js ./scripts/prepare.py", "start": "node ./scripts/run-python3.js ./scripts/start.py", - "copy-pygen": "copyfiles -a ./node_modules/@azure-tools/typespec-python ./pygen", + "copy-pygen": "node ./scripts/copy-pygen.js", "install": "npm run copy-pygen && node ./scripts/run-python3.js ./scripts/install.py", "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" }, diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index 3b9005eae1d..bebbbb5a645 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -1,2 +1,12 @@ -json-rpc==1.14.0 --e ./node_modules/@azure-tools/typespec-python/pygen +black==24.4.0 +click==8.1.3 +docutils==0.19 +Jinja2==3.1.4 +m2r2==0.3.3 +MarkupSafe==2.1.2 +mistune==0.8.4 +pathspec==0.11.1 +platformdirs==3.2.0 +PyYAML==6.0.1 +tomli==2.0.1 +setuptools==69.2.0 diff --git a/packages/autorest.python/scripts/copy-pygen.js b/packages/autorest.python/scripts/copy-pygen.js new file mode 100644 index 00000000000..089e8c696d9 --- /dev/null +++ b/packages/autorest.python/scripts/copy-pygen.js @@ -0,0 +1,10 @@ +const fs = require('fs-extra'); +const path = require('path'); +const url = require('url'); + +// Define the source and destination directories +const sourceDir = path.join(__dirname, "..", "..", "node_modules", "@azure-tools", "typespec-python", "pygen"); +const destDir = path.join(__dirname, "..", "..", "pygen"); + +// Copy the source directory to the destination directory +fs.copySync(sourceDir, destDir); diff --git a/packages/autorest.python/setup.py b/packages/autorest.python/setup.py index b22fa2172ca..79bba224efe 100644 --- a/packages/autorest.python/setup.py +++ b/packages/autorest.python/setup.py @@ -14,22 +14,22 @@ # Version extraction inspired from 'requests' -with open(os.path.join("autorest", "_version.py"), "r") as fd: +with open(os.path.join("pygen", "_version.py"), "r") as fd: version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) if not version: raise RuntimeError("Cannot find version information") setup( - name="autorest", + name="pygen", version=version, - description="Microsoft Autorest Plugins for Python", + description="Core Library for Python Generation", long_description=open("README.md", "r").read(), long_description_content_type="text/markdown", license="MIT License", author="Microsoft Corporation", author_email="azpysdkhelp@microsoft.com", - url="https://github.com/Azure/autorest.python", + url="https://github.com/Azure/autorest.python/packages/core", classifiers=[ "Development Status :: 4 - Beta", "Programming Language :: Python", From a3b5e26d6d5b00cc8d49f0cff8d994c908413c8f Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 4 Jun 2024 14:33:55 -0400 Subject: [PATCH 61/75] fix readme and setup.py of autorest.pyton --- .gitignore | 1 + packages/autorest.python/README.md | 220 +++++++++++++++++- .../autorest.python/scripts/copy-pygen.js | 4 +- packages/autorest.python/setup.py | 8 +- 4 files changed, 226 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 5fbc2245061..4a669c950e6 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,4 @@ node_modules/ # Generated test folders test/services/*/_generated +**/autorest.python/pygen diff --git a/packages/autorest.python/README.md b/packages/autorest.python/README.md index 3d8a65a8919..671c0924699 100644 --- a/packages/autorest.python/README.md +++ b/packages/autorest.python/README.md @@ -1 +1,219 @@ -# Core Library for Python Generation +# Generating with Autorest for Python + +See [here](https://github.com/Azure/autorest.python/wiki/Generating-with-autorest-for-python-v5.0.0) for Python-specific docs, and [here] for general docs + +# Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a +Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us +the rights to use your contribution. For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide +a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions +provided by the bot. You will only need to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +### Autorest plugin configuration + +- Please don't edit this section unless you're re-configuring how the powershell extension plugs in to AutoRest + AutoRest needs the below config to pick this up as a plug-in - see https://github.com/Azure/autorest/blob/master/docs/developer/architecture/AutoRest-extension.md + +#### Python code gen + +```yaml !$(multiapiscript) && !$(multiclientscript) +# default values for version tolerant and black +black: true +``` + +```yaml !$(low-level-client) +version-tolerant: true +``` + +```yaml !$(low-level-client) && !$(version-tolerant) +modelerfour: + group-parameters: true + flatten-models: true + flatten-payloads: true +``` + +```yaml $(postprocess) +allow-no-input: true +``` + +```yaml !$(multiapiscript) && !$(multiclientscript) +pass-thru: + - model-deduplicator + - subset-reducer +version: ~3.8.1 +use-extension: + "@autorest/modelerfour": ~4.23.5 + +modelerfour: + resolve-schema-name-collisons: true + always-create-content-type-parameter: true + multiple-request-parameter-flattening: false + naming: + parameter: snakecase + property: snakecase + operation: snakecase + operationGroup: pascalcase + choice: pascalcase + choiceValue: uppercase + constant: snakecase + constantParameter: snakecase + type: pascalcase + local: _ + snakecase + global: snakecase + preserve-uppercase-max-length: 6 + override: + $host: $host + base64: base64 + IncludeAPIs: include_apis + +pipeline: + python: + # Doesn't process anything, just makes it so that the 'python:' config section is loaded and available for the next plugins. + pass-thru: true + input: modelerfour/identity + + python/m2r: + input: python + + python/m4reformatter: + input: python/m2r + + python/preprocess: + input: python/m4reformatter + + python/codegen: + input: python/preprocess + output-artifact: python-files + + python/codegen/emitter: + input: codegen + scope: scope-codegen/emitter + +scope-codegen/emitter: + input-artifact: python-files + output-uri-expr: $key + +output-artifact: python-files +``` + +# Multiapi script pipeline + +```yaml $(multiapiscript) +pipeline: + python/multiapiscript: + scope: multiapiscript + output-artifact: python-files + + python/multiapiscript/emitter: + input: multiapiscript + scope: scope-multiapiscript/emitter + +scope-multiapiscript/emitter: + input-artifact: python-files + output-uri-expr: $key + +output-artifact: python-files +``` + +# Black script pipeline + +```yaml $(black) +pipeline: + python/black: + scope: black + input: python/codegen + output-artifact: python-files + + python/black/emitter: + input: black + scope: scope-black/emitter + +scope-black/emitter: + input-artifact: python-files + output-uri-expr: $key + +output-artifact: python-files +``` + +# Post-process customized code for mypy pipeline + +```yaml $(postprocess) +pipeline: + python/postprocess: + scope: postprocess + output-artifact: python-files + + python/postprocess/emitter: + input: postprocess + scope: scope-postprocess/emitter + +scope-postprocess/emitter: + input-artifact: python-files + output-uri-expr: $key + +output-artifact: python-files +``` + +# Multi Client pipeline + +```yaml $(multiclientscript) +pipeline: + python/multiclientscript: + scope: multiclientscript + output-artifact: python-files + + python/multiclientscript/emitter: + input: multiclientscript + scope: scope-multiclientscript/emitter + +scope-multiclientscript/emitter: + input-artifact: python-files + output-uri-expr: $key + +output-artifact: python-files +``` + +# Help + +```yaml +help-content: + python: # type: Help as defined in autorest-core/help.ts + activationScope: python + categoryFriendlyName: Python Generator + settings: + - key: python-sdks-folder + description: The path to the root directory of your azure-sdk-for-python clone. Be sure to note that we include `sdk` in the folder path. + - key: black + description: Runs black formatting on your generated files. Defaults to `false`. + type: string + - key: basic-setup-py + description: Whether to generate a build script for setuptools to package your SDK. Defaults to `false`, generally not suggested if you are going to wrap the generated code + type: bool + - key: multiapi + description: Whether to generate a multiapi client. + type: bool + - key: default-api + description: In the case of `--multiapi`, you can override the default service API version with this flag. If not specified, we use the latest GA service version as the default API. + type: string + - key: no-namespace-folders + description: Specify if you don't want pkgutil-style namespace folders. Defaults to `false`. + type: bool + - key: credential-default-policy-type + description: Specify the default credential policy (authentication policy) for your client. Use in conjunction with `--add-credential`. Currently only supports `BearerTokenCredentialPolicy`, `ARMChallengeAuthenticationPolicy` and `AzureKeyCredentialPolicy`. Default value is `BearerTokenCredentialPolicy`(data-plan)/`ARMChallengeAuthenticationPolicy`(mgmt-plan). `--credential-scopes` is tied with `BearerTokenCredentialPolicy` and `ARMChallengeAuthenticationPolicy`, do not pass them in if you want `AzureKeyCredentialPolicy`. + type: string + - key: credential-key-header-name + description: The name of the header which will pass the credential. Use if you have `--credential-default-policy-type` set to `AzureKeyCredentialPolicy`. For example, if generating cognitive services code, you might use `--credential-key-header-name=Ocp-Apim-Subscription-Key` + type: string +``` + + + +[python_docs]: https://github.com/Azure/autorest.python/tree/main/docs/readme.md +[main_docs]: https://github.com/Azure/autorest/tree/master/docs diff --git a/packages/autorest.python/scripts/copy-pygen.js b/packages/autorest.python/scripts/copy-pygen.js index 089e8c696d9..8fd38738b4d 100644 --- a/packages/autorest.python/scripts/copy-pygen.js +++ b/packages/autorest.python/scripts/copy-pygen.js @@ -3,8 +3,8 @@ const path = require('path'); const url = require('url'); // Define the source and destination directories -const sourceDir = path.join(__dirname, "..", "..", "node_modules", "@azure-tools", "typespec-python", "pygen"); -const destDir = path.join(__dirname, "..", "..", "pygen"); +const sourceDir = path.join(__dirname, "..", "node_modules", "@azure-tools", "typespec-python", "pygen"); +const destDir = path.join(__dirname, "..", "pygen"); // Copy the source directory to the destination directory fs.copySync(sourceDir, destDir); diff --git a/packages/autorest.python/setup.py b/packages/autorest.python/setup.py index 79bba224efe..b22fa2172ca 100644 --- a/packages/autorest.python/setup.py +++ b/packages/autorest.python/setup.py @@ -14,22 +14,22 @@ # Version extraction inspired from 'requests' -with open(os.path.join("pygen", "_version.py"), "r") as fd: +with open(os.path.join("autorest", "_version.py"), "r") as fd: version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) if not version: raise RuntimeError("Cannot find version information") setup( - name="pygen", + name="autorest", version=version, - description="Core Library for Python Generation", + description="Microsoft Autorest Plugins for Python", long_description=open("README.md", "r").read(), long_description_content_type="text/markdown", license="MIT License", author="Microsoft Corporation", author_email="azpysdkhelp@microsoft.com", - url="https://github.com/Azure/autorest.python/packages/core", + url="https://github.com/Azure/autorest.python", classifiers=[ "Development Status :: 4 - Beta", "Programming Language :: Python", From 96e1f56acddc24feb80f94140aa6eb8e40589507 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 4 Jun 2024 14:44:05 -0400 Subject: [PATCH 62/75] copy pygen into autorest.python --- packages/autorest.python/package.json | 2 - packages/autorest.python/requirements.txt | 14 +--- packages/autorest.python/scripts/install.py | 25 +++---- packages/autorest.python/scripts/prepare.py | 35 ---------- packages/autorest.python/scripts/start.py | 2 +- pnpm-lock.yaml | 75 +++++++-------------- 6 files changed, 37 insertions(+), 116 deletions(-) delete mode 100644 packages/autorest.python/scripts/prepare.py diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 9a2778ffd09..acbabf5c074 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -3,7 +3,6 @@ "version": "6.13.18", "description": "The Python extension for generators in AutoRest.", "scripts": { - "prepare": "node ./scripts/run-python3.js ./scripts/prepare.py", "start": "node ./scripts/run-python3.js ./scripts/start.py", "copy-pygen": "node ./scripts/copy-pygen.js", "install": "npm run copy-pygen && node ./scripts/run-python3.js ./scripts/install.py", @@ -29,7 +28,6 @@ "@autorest/system-requirements": "~1.0.2", "fs-extra": "~11.2.0", "semver": "~7.6.2", - "copyfiles":"~2.4.1", "@azure-tools/typespec-python": "workspace:^" }, "devDependencies": { diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index bebbbb5a645..1308f75c4a9 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -1,12 +1,2 @@ -black==24.4.0 -click==8.1.3 -docutils==0.19 -Jinja2==3.1.4 -m2r2==0.3.3 -MarkupSafe==2.1.2 -mistune==0.8.4 -pathspec==0.11.1 -platformdirs==3.2.0 -PyYAML==6.0.1 -tomli==2.0.1 -setuptools==69.2.0 +json-rpc==1.14.0 +-e ./pygen diff --git a/packages/autorest.python/scripts/install.py b/packages/autorest.python/scripts/install.py index f0d2eddee93..573330574d6 100644 --- a/packages/autorest.python/scripts/install.py +++ b/packages/autorest.python/scripts/install.py @@ -24,26 +24,23 @@ # Now we have pip and Py >= 3.8, go to work from pathlib import Path +import shutil -from venvtools import ExtendedEnvBuilder, python_run +from venvtools import python_run _ROOT_DIR = Path(__file__).parent.parent def main(): - venv_path = _ROOT_DIR / "venv" - - if venv_path.exists(): - env_builder = venv.EnvBuilder(with_pip=True) - venv_context = env_builder.ensure_directories(venv_path) - else: - env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True) - env_builder.create(venv_path) - venv_context = env_builder.context - - python_run(venv_context, "pip", ["install", "-U", "pip"]) - python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) - python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) + # we copy pygen's venv into autorest.python + venv_path = _ROOT_DIR / "pygen" / "venv" + assert venv_path.exists() # Otherwise install was not done + + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + # install autorest.python specific stuff into it + python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) + python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) if __name__ == "__main__": diff --git a/packages/autorest.python/scripts/prepare.py b/packages/autorest.python/scripts/prepare.py deleted file mode 100644 index 4593c828c52..00000000000 --- a/packages/autorest.python/scripts/prepare.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python - -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import sys - -if not sys.version_info >= (3, 8, 0): - raise Exception("Autorest for Python extension requires Python 3.8 at least") - -from pathlib import Path -import venv - -from venvtools import python_run - -_ROOT_DIR = Path(__file__).parent.parent - - -def main(): - venv_path = _ROOT_DIR / "venv" - venv_prexists = venv_path.exists() - - assert venv_prexists # Otherwise install was not done - - env_builder = venv.EnvBuilder(with_pip=True) - venv_context = env_builder.ensure_directories(venv_path) - requirements_path = _ROOT_DIR / "dev_requirements.txt" - - python_run(venv_context, "pip", ["install", "-r", str(requirements_path)]) - - -if __name__ == "__main__": - main() diff --git a/packages/autorest.python/scripts/start.py b/packages/autorest.python/scripts/start.py index 714fc50155f..2b77b7439d0 100644 --- a/packages/autorest.python/scripts/start.py +++ b/packages/autorest.python/scripts/start.py @@ -19,7 +19,7 @@ def main(): - venv_path = _ROOT_DIR / "venv" + venv_path = _ROOT_DIR / "pygen" / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5e1ada4de0e..78d5138b477 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,9 +53,6 @@ importers: '@azure-tools/typespec-python': specifier: workspace:^ version: link:../typespec-python - copyfiles: - specifier: ~2.4.1 - version: 2.4.1 fs-extra: specifier: ~11.2.0 version: 11.2.0 @@ -2063,6 +2060,7 @@ packages: /balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + dev: true /basic-auth@2.0.1: resolution: {integrity: sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==} @@ -2135,6 +2133,7 @@ packages: dependencies: balanced-match: 1.0.2 concat-map: 0.0.1 + dev: true /brace-expansion@2.0.1: resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==} @@ -2338,6 +2337,7 @@ packages: string-width: 4.2.3 strip-ansi: 6.0.1 wrap-ansi: 7.0.0 + dev: true /cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} @@ -2413,6 +2413,7 @@ packages: /concat-map@0.0.1: resolution: {integrity: sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=} + dev: true /concat-stream@1.6.2: resolution: {integrity: sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==} @@ -2454,19 +2455,6 @@ packages: engines: {node: '>= 0.6'} dev: true - /copyfiles@2.4.1: - resolution: {integrity: sha512-fereAvAvxDrQDOXybk3Qu3dPbOoKoysFMWtkY3mv5BsL8//OSZVL5DCLYqgRfY5cWirgRzlC+WSrxp6Bo3eNZg==} - hasBin: true - dependencies: - glob: 7.2.3 - minimatch: 3.1.2 - mkdirp: 1.0.4 - noms: 0.0.0 - through2: 2.0.5 - untildify: 4.0.0 - yargs: 16.2.0 - dev: false - /core-js-compat@3.37.0: resolution: {integrity: sha512-vYq4L+T8aS5UuFg4UwDhc7YNRWVeVZwltad9C/jV3R2LgVOpS9BDr7l/WL6BN0dbV3k1XejPTHqqEzJgsa0frA==} dependencies: @@ -2475,6 +2463,7 @@ packages: /core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} + dev: true /cosmiconfig@8.1.3: resolution: {integrity: sha512-/UkO2JKI18b5jVMJUp0lvKFMpa/Gye+ZgZjKD+DGEN9y7NRcf/nK1A0sp67ONmKtnDCNMS44E6jrk0Yc3bDuUw==} @@ -3532,6 +3521,7 @@ packages: /fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + dev: true /fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} @@ -3659,6 +3649,7 @@ packages: minimatch: 3.1.2 once: 1.4.0 path-is-absolute: 1.0.1 + dev: true /glob@8.1.0: resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} @@ -3917,9 +3908,11 @@ packages: dependencies: once: 1.4.0 wrappy: 1.0.2 + dev: true /inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + dev: true /internal-slot@1.0.5: resolution: {integrity: sha512-Y+R5hJrzs52QCG2laLn4udYVnxsfny9CpOhNhUvk/SSSVyF6T27FzRbF0sroPidSu3X8oEAkOn2K804mjpt6UQ==} @@ -4214,12 +4207,9 @@ packages: is-docker: 2.2.1 dev: true - /isarray@0.0.1: - resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==} - dev: false - /isarray@1.0.0: resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + dev: true /isarray@2.0.5: resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} @@ -4617,6 +4607,7 @@ packages: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} dependencies: brace-expansion: 1.1.11 + dev: true /minimatch@5.0.1: resolution: {integrity: sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==} @@ -4744,6 +4735,7 @@ packages: resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} engines: {node: '>=10'} hasBin: true + dev: true /mocha@10.2.0: resolution: {integrity: sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==} @@ -4891,13 +4883,6 @@ packages: resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} dev: true - /noms@0.0.0: - resolution: {integrity: sha512-lNDU9VJaOPxUmXcLb+HQFeUgQQPtMI24Gt6hgfuMHRJgMRHMF/qZ4HJD3GDru4sSw9IQl2jPjAYnQrdIeLbwow==} - dependencies: - inherits: 2.0.4 - readable-stream: 1.0.34 - dev: false - /nopt@7.2.0: resolution: {integrity: sha512-CVDtwCdhYIvnAzFoJ6NJ6dX3oga9/HyciQDnG1vQDjSLMeKLJ4A93ZqYKDrgYSr1FBY5/hMYC+2VCi24pgpkGA==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -5117,6 +5102,7 @@ packages: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} dependencies: wrappy: 1.0.2 + dev: true /one-time@1.0.0: resolution: {integrity: sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==} @@ -5243,6 +5229,7 @@ packages: /path-is-absolute@1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} + dev: true /path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} @@ -5338,6 +5325,7 @@ packages: /process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + dev: true /process@0.11.10: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} @@ -5475,15 +5463,6 @@ packages: strip-bom: 4.0.0 dev: true - /readable-stream@1.0.34: - resolution: {integrity: sha512-ok1qVCJuRkNmvebYikljxJA/UEsKwLl2nI1OmaqAu4/UE+h0wKCHok4XkL/gvi39OacXvw59RJUOFUkDib2rHg==} - dependencies: - core-util-is: 1.0.2 - inherits: 2.0.4 - isarray: 0.0.1 - string_decoder: 0.10.31 - dev: false - /readable-stream@2.3.8: resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} dependencies: @@ -5494,6 +5473,7 @@ packages: safe-buffer: 5.1.2 string_decoder: 1.1.1 util-deprecate: 1.0.2 + dev: true /readable-stream@3.6.2: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} @@ -5662,6 +5642,7 @@ packages: /safe-buffer@5.1.2: resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + dev: true /safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -6034,14 +6015,11 @@ packages: es-object-atoms: 1.0.0 dev: true - /string_decoder@0.10.31: - resolution: {integrity: sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==} - dev: false - /string_decoder@1.1.1: resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} dependencies: safe-buffer: 5.1.2 + dev: true /string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} @@ -6163,13 +6141,6 @@ packages: resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} dev: true - /through2@2.0.5: - resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} - dependencies: - readable-stream: 2.3.8 - xtend: 4.0.2 - dev: false - /tightrope@0.1.0: resolution: {integrity: sha512-HHHNYdCAIYwl1jOslQBT455zQpdeSo8/A346xpIb/uuqhSg+tCvYNsP5f11QW+z9VZ3vSX8YIfzTApjjuGH63w==} engines: {node: '>=14'} @@ -6435,11 +6406,6 @@ packages: engines: {node: '>= 0.8'} dev: true - /untildify@4.0.0: - resolution: {integrity: sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==} - engines: {node: '>=8'} - dev: false - /update-browserslist-db@1.0.15(browserslist@4.23.0): resolution: {integrity: sha512-K9HWH62x3/EalU1U6sjSZiylm9C8tgq2mSvshZpqc7QE69RaA2qjhkW2HlNA0tFpEbtyFz7HTqbSdN4MSwUodA==} hasBin: true @@ -6458,6 +6424,7 @@ packages: /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + dev: true /utils-merge@1.0.1: resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} @@ -6692,6 +6659,7 @@ packages: /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + dev: true /xml2js@0.4.23: resolution: {integrity: sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==} @@ -6717,6 +6685,7 @@ packages: /xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} + dev: true /y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} @@ -6739,6 +6708,7 @@ packages: /yargs-parser@20.2.9: resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} engines: {node: '>=10'} + dev: true /yargs-parser@21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} @@ -6765,6 +6735,7 @@ packages: string-width: 4.2.3 y18n: 5.0.8 yargs-parser: 20.2.9 + dev: true /yargs@17.7.2: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} From a4fb4dcd2beb16e15533f27ad55670659798563d Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Tue, 4 Jun 2024 15:29:32 -0400 Subject: [PATCH 63/75] try moving pygen up a folder --- packages/typespec-python/pygen/{pygen => }/__init__.py | 0 packages/typespec-python/pygen/{pygen => }/_version.py | 0 packages/typespec-python/pygen/{pygen => }/black/__init__.py | 0 packages/typespec-python/pygen/{pygen => }/codegen/__init__.py | 0 packages/typespec-python/pygen/{pygen => }/codegen/_utils.py | 0 .../typespec-python/pygen/{pygen => }/codegen/models/__init__.py | 0 packages/typespec-python/pygen/{pygen => }/codegen/models/base.py | 0 .../pygen/{pygen => }/codegen/models/base_builder.py | 0 .../typespec-python/pygen/{pygen => }/codegen/models/client.py | 0 .../pygen/{pygen => }/codegen/models/code_model.py | 0 .../pygen/{pygen => }/codegen/models/combined_type.py | 0 .../pygen/{pygen => }/codegen/models/constant_type.py | 0 .../pygen/{pygen => }/codegen/models/credential_types.py | 0 .../pygen/{pygen => }/codegen/models/dictionary_type.py | 0 .../typespec-python/pygen/{pygen => }/codegen/models/enum_type.py | 0 .../typespec-python/pygen/{pygen => }/codegen/models/imports.py | 0 .../typespec-python/pygen/{pygen => }/codegen/models/list_type.py | 0 .../pygen/{pygen => }/codegen/models/lro_operation.py | 0 .../pygen/{pygen => }/codegen/models/lro_paging_operation.py | 0 .../pygen/{pygen => }/codegen/models/model_type.py | 0 .../typespec-python/pygen/{pygen => }/codegen/models/operation.py | 0 .../pygen/{pygen => }/codegen/models/operation_group.py | 0 .../pygen/{pygen => }/codegen/models/paging_operation.py | 0 .../typespec-python/pygen/{pygen => }/codegen/models/parameter.py | 0 .../pygen/{pygen => }/codegen/models/parameter_list.py | 0 .../pygen/{pygen => }/codegen/models/primitive_types.py | 0 .../typespec-python/pygen/{pygen => }/codegen/models/property.py | 0 .../pygen/{pygen => }/codegen/models/request_builder.py | 0 .../pygen/{pygen => }/codegen/models/request_builder_parameter.py | 0 .../typespec-python/pygen/{pygen => }/codegen/models/response.py | 0 .../typespec-python/pygen/{pygen => }/codegen/models/utils.py | 0 .../pygen/{pygen => }/codegen/serializers/__init__.py | 0 .../pygen/{pygen => }/codegen/serializers/base_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/builder_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/client_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/enum_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/general_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/import_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/metadata_serializer.py | 0 .../{pygen => }/codegen/serializers/model_init_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/model_serializer.py | 0 .../codegen/serializers/operation_groups_serializer.py | 0 .../{pygen => }/codegen/serializers/operations_init_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/parameter_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/patch_serializer.py | 0 .../codegen/serializers/request_builders_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/sample_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/test_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/types_serializer.py | 0 .../pygen/{pygen => }/codegen/serializers/utils.py | 0 .../pygen/{pygen => }/codegen/templates/client.py.jinja2 | 0 .../{pygen => }/codegen/templates/client_container.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/config.py.jinja2 | 0 .../{pygen => }/codegen/templates/config_container.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/conftest.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/enum.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/enum_container.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/init.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/keywords.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/lro_operation.py.jinja2 | 0 .../{pygen => }/codegen/templates/lro_paging_operation.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/macros.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/metadata.json.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/model_base.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/model_container.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/model_dpg.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/model_init.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/model_msrest.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/operation.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/operation_group.py.jinja2 | 0 .../codegen/templates/operation_groups_container.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/operation_tools.jinja2 | 0 .../codegen/templates/operations_folder_init.py.jinja2 | 0 .../codegen/templates/packaging_templates/CHANGELOG.md.jinja2 | 0 .../codegen/templates/packaging_templates/LICENSE.jinja2 | 0 .../codegen/templates/packaging_templates/MANIFEST.in.jinja2 | 0 .../codegen/templates/packaging_templates/README.md.jinja2 | 0 .../templates/packaging_templates/dev_requirements.txt.jinja2 | 0 .../codegen/templates/packaging_templates/setup.py.jinja2 | 0 .../{pygen => }/codegen/templates/paging_operation.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/patch.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/pkgutil_init.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/request_builder.py.jinja2 | 0 .../{pygen => }/codegen/templates/request_builders.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/rest_init.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/sample.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/serialization.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/test.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/testpreparer.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/types.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/validation.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/vendor.py.jinja2 | 0 .../pygen/{pygen => }/codegen/templates/version.py.jinja2 | 0 packages/typespec-python/pygen/{pygen => }/m2r/__init__.py | 0 .../typespec-python/pygen/{pygen => }/postprocess/__init__.py | 0 packages/typespec-python/pygen/{pygen => }/postprocess/get_all.py | 0 .../typespec-python/pygen/{pygen => }/postprocess/venvtools.py | 0 packages/typespec-python/pygen/{pygen => }/preprocess/__init__.py | 0 packages/typespec-python/pygen/{pygen => }/preprocess/helpers.py | 0 .../pygen/{pygen => }/preprocess/python_mappings.py | 0 packages/typespec-python/pygen/{pygen => }/utils.py | 0 101 files changed, 0 insertions(+), 0 deletions(-) rename packages/typespec-python/pygen/{pygen => }/__init__.py (100%) rename packages/typespec-python/pygen/{pygen => }/_version.py (100%) rename packages/typespec-python/pygen/{pygen => }/black/__init__.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/__init__.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/_utils.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/__init__.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/base.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/base_builder.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/client.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/code_model.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/combined_type.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/constant_type.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/credential_types.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/dictionary_type.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/enum_type.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/imports.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/list_type.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/lro_operation.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/lro_paging_operation.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/model_type.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/operation.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/operation_group.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/paging_operation.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/parameter.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/parameter_list.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/primitive_types.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/property.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/request_builder.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/request_builder_parameter.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/response.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/models/utils.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/__init__.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/base_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/builder_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/client_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/enum_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/general_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/import_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/metadata_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/model_init_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/model_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/operation_groups_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/operations_init_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/parameter_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/patch_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/request_builders_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/sample_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/test_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/types_serializer.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/serializers/utils.py (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/client.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/client_container.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/config.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/config_container.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/conftest.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/enum.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/enum_container.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/init.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/keywords.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/lro_operation.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/lro_paging_operation.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/macros.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/metadata.json.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/model_base.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/model_container.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/model_dpg.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/model_init.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/model_msrest.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/operation.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/operation_group.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/operation_groups_container.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/operation_tools.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/operations_folder_init.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/packaging_templates/LICENSE.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/packaging_templates/MANIFEST.in.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/packaging_templates/README.md.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/packaging_templates/setup.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/paging_operation.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/patch.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/pkgutil_init.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/request_builder.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/request_builders.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/rest_init.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/sample.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/serialization.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/test.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/testpreparer.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/types.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/validation.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/vendor.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/codegen/templates/version.py.jinja2 (100%) rename packages/typespec-python/pygen/{pygen => }/m2r/__init__.py (100%) rename packages/typespec-python/pygen/{pygen => }/postprocess/__init__.py (100%) rename packages/typespec-python/pygen/{pygen => }/postprocess/get_all.py (100%) rename packages/typespec-python/pygen/{pygen => }/postprocess/venvtools.py (100%) rename packages/typespec-python/pygen/{pygen => }/preprocess/__init__.py (100%) rename packages/typespec-python/pygen/{pygen => }/preprocess/helpers.py (100%) rename packages/typespec-python/pygen/{pygen => }/preprocess/python_mappings.py (100%) rename packages/typespec-python/pygen/{pygen => }/utils.py (100%) diff --git a/packages/typespec-python/pygen/pygen/__init__.py b/packages/typespec-python/pygen/__init__.py similarity index 100% rename from packages/typespec-python/pygen/pygen/__init__.py rename to packages/typespec-python/pygen/__init__.py diff --git a/packages/typespec-python/pygen/pygen/_version.py b/packages/typespec-python/pygen/_version.py similarity index 100% rename from packages/typespec-python/pygen/pygen/_version.py rename to packages/typespec-python/pygen/_version.py diff --git a/packages/typespec-python/pygen/pygen/black/__init__.py b/packages/typespec-python/pygen/black/__init__.py similarity index 100% rename from packages/typespec-python/pygen/pygen/black/__init__.py rename to packages/typespec-python/pygen/black/__init__.py diff --git a/packages/typespec-python/pygen/pygen/codegen/__init__.py b/packages/typespec-python/pygen/codegen/__init__.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/__init__.py rename to packages/typespec-python/pygen/codegen/__init__.py diff --git a/packages/typespec-python/pygen/pygen/codegen/_utils.py b/packages/typespec-python/pygen/codegen/_utils.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/_utils.py rename to packages/typespec-python/pygen/codegen/_utils.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/__init__.py b/packages/typespec-python/pygen/codegen/models/__init__.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/__init__.py rename to packages/typespec-python/pygen/codegen/models/__init__.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/base.py b/packages/typespec-python/pygen/codegen/models/base.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/base.py rename to packages/typespec-python/pygen/codegen/models/base.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/base_builder.py b/packages/typespec-python/pygen/codegen/models/base_builder.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/base_builder.py rename to packages/typespec-python/pygen/codegen/models/base_builder.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/client.py b/packages/typespec-python/pygen/codegen/models/client.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/client.py rename to packages/typespec-python/pygen/codegen/models/client.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/code_model.py b/packages/typespec-python/pygen/codegen/models/code_model.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/code_model.py rename to packages/typespec-python/pygen/codegen/models/code_model.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/combined_type.py b/packages/typespec-python/pygen/codegen/models/combined_type.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/combined_type.py rename to packages/typespec-python/pygen/codegen/models/combined_type.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/constant_type.py b/packages/typespec-python/pygen/codegen/models/constant_type.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/constant_type.py rename to packages/typespec-python/pygen/codegen/models/constant_type.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/credential_types.py b/packages/typespec-python/pygen/codegen/models/credential_types.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/credential_types.py rename to packages/typespec-python/pygen/codegen/models/credential_types.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/dictionary_type.py b/packages/typespec-python/pygen/codegen/models/dictionary_type.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/dictionary_type.py rename to packages/typespec-python/pygen/codegen/models/dictionary_type.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/enum_type.py b/packages/typespec-python/pygen/codegen/models/enum_type.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/enum_type.py rename to packages/typespec-python/pygen/codegen/models/enum_type.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/imports.py b/packages/typespec-python/pygen/codegen/models/imports.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/imports.py rename to packages/typespec-python/pygen/codegen/models/imports.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/list_type.py b/packages/typespec-python/pygen/codegen/models/list_type.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/list_type.py rename to packages/typespec-python/pygen/codegen/models/list_type.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/lro_operation.py b/packages/typespec-python/pygen/codegen/models/lro_operation.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/lro_operation.py rename to packages/typespec-python/pygen/codegen/models/lro_operation.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/lro_paging_operation.py b/packages/typespec-python/pygen/codegen/models/lro_paging_operation.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/lro_paging_operation.py rename to packages/typespec-python/pygen/codegen/models/lro_paging_operation.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/model_type.py b/packages/typespec-python/pygen/codegen/models/model_type.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/model_type.py rename to packages/typespec-python/pygen/codegen/models/model_type.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/operation.py b/packages/typespec-python/pygen/codegen/models/operation.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/operation.py rename to packages/typespec-python/pygen/codegen/models/operation.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/operation_group.py b/packages/typespec-python/pygen/codegen/models/operation_group.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/operation_group.py rename to packages/typespec-python/pygen/codegen/models/operation_group.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/paging_operation.py b/packages/typespec-python/pygen/codegen/models/paging_operation.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/paging_operation.py rename to packages/typespec-python/pygen/codegen/models/paging_operation.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/parameter.py b/packages/typespec-python/pygen/codegen/models/parameter.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/parameter.py rename to packages/typespec-python/pygen/codegen/models/parameter.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/parameter_list.py b/packages/typespec-python/pygen/codegen/models/parameter_list.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/parameter_list.py rename to packages/typespec-python/pygen/codegen/models/parameter_list.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/primitive_types.py b/packages/typespec-python/pygen/codegen/models/primitive_types.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/primitive_types.py rename to packages/typespec-python/pygen/codegen/models/primitive_types.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/property.py b/packages/typespec-python/pygen/codegen/models/property.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/property.py rename to packages/typespec-python/pygen/codegen/models/property.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/request_builder.py b/packages/typespec-python/pygen/codegen/models/request_builder.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/request_builder.py rename to packages/typespec-python/pygen/codegen/models/request_builder.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/request_builder_parameter.py b/packages/typespec-python/pygen/codegen/models/request_builder_parameter.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/request_builder_parameter.py rename to packages/typespec-python/pygen/codegen/models/request_builder_parameter.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/response.py b/packages/typespec-python/pygen/codegen/models/response.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/response.py rename to packages/typespec-python/pygen/codegen/models/response.py diff --git a/packages/typespec-python/pygen/pygen/codegen/models/utils.py b/packages/typespec-python/pygen/codegen/models/utils.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/models/utils.py rename to packages/typespec-python/pygen/codegen/models/utils.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/__init__.py b/packages/typespec-python/pygen/codegen/serializers/__init__.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/__init__.py rename to packages/typespec-python/pygen/codegen/serializers/__init__.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/base_serializer.py b/packages/typespec-python/pygen/codegen/serializers/base_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/base_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/base_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/builder_serializer.py b/packages/typespec-python/pygen/codegen/serializers/builder_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/builder_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/builder_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/client_serializer.py b/packages/typespec-python/pygen/codegen/serializers/client_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/client_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/client_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/enum_serializer.py b/packages/typespec-python/pygen/codegen/serializers/enum_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/enum_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/enum_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/general_serializer.py b/packages/typespec-python/pygen/codegen/serializers/general_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/general_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/general_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/import_serializer.py b/packages/typespec-python/pygen/codegen/serializers/import_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/import_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/import_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/metadata_serializer.py b/packages/typespec-python/pygen/codegen/serializers/metadata_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/metadata_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/metadata_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/model_init_serializer.py b/packages/typespec-python/pygen/codegen/serializers/model_init_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/model_init_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/model_init_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/model_serializer.py b/packages/typespec-python/pygen/codegen/serializers/model_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/model_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/model_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/operation_groups_serializer.py b/packages/typespec-python/pygen/codegen/serializers/operation_groups_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/operation_groups_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/operation_groups_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/operations_init_serializer.py b/packages/typespec-python/pygen/codegen/serializers/operations_init_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/operations_init_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/operations_init_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/parameter_serializer.py b/packages/typespec-python/pygen/codegen/serializers/parameter_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/parameter_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/parameter_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/patch_serializer.py b/packages/typespec-python/pygen/codegen/serializers/patch_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/patch_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/patch_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/request_builders_serializer.py b/packages/typespec-python/pygen/codegen/serializers/request_builders_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/request_builders_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/request_builders_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/sample_serializer.py b/packages/typespec-python/pygen/codegen/serializers/sample_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/sample_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/sample_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/test_serializer.py b/packages/typespec-python/pygen/codegen/serializers/test_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/test_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/test_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/types_serializer.py b/packages/typespec-python/pygen/codegen/serializers/types_serializer.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/types_serializer.py rename to packages/typespec-python/pygen/codegen/serializers/types_serializer.py diff --git a/packages/typespec-python/pygen/pygen/codegen/serializers/utils.py b/packages/typespec-python/pygen/codegen/serializers/utils.py similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/serializers/utils.py rename to packages/typespec-python/pygen/codegen/serializers/utils.py diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/client.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/client.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/client.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/client.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/client_container.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/client_container.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/client_container.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/client_container.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/config.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/config.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/config.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/config.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/config_container.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/config_container.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/config_container.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/config_container.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/conftest.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/conftest.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/conftest.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/conftest.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/enum.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/enum.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/enum.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/enum.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/enum_container.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/enum_container.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/enum_container.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/enum_container.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/init.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/init.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/init.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/init.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/keywords.jinja2 b/packages/typespec-python/pygen/codegen/templates/keywords.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/keywords.jinja2 rename to packages/typespec-python/pygen/codegen/templates/keywords.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/lro_operation.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/lro_operation.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/lro_operation.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/lro_operation.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/lro_paging_operation.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/lro_paging_operation.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/lro_paging_operation.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/lro_paging_operation.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/macros.jinja2 b/packages/typespec-python/pygen/codegen/templates/macros.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/macros.jinja2 rename to packages/typespec-python/pygen/codegen/templates/macros.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/metadata.json.jinja2 b/packages/typespec-python/pygen/codegen/templates/metadata.json.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/metadata.json.jinja2 rename to packages/typespec-python/pygen/codegen/templates/metadata.json.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/model_base.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/model_base.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/model_base.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/model_base.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/model_container.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/model_container.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/model_container.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/model_container.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/model_dpg.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/model_dpg.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/model_dpg.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/model_dpg.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/model_init.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/model_init.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/model_init.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/model_init.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/model_msrest.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/model_msrest.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/model_msrest.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/model_msrest.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/operation.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/operation.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/operation.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/operation.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/operation_group.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/operation_group.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/operation_group.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/operation_group.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/operation_groups_container.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/operation_groups_container.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/operation_groups_container.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/operation_groups_container.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/operation_tools.jinja2 b/packages/typespec-python/pygen/codegen/templates/operation_tools.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/operation_tools.jinja2 rename to packages/typespec-python/pygen/codegen/templates/operation_tools.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/operations_folder_init.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/operations_folder_init.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/operations_folder_init.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/operations_folder_init.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 b/packages/typespec-python/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 rename to packages/typespec-python/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 b/packages/typespec-python/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 rename to packages/typespec-python/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 b/packages/typespec-python/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 rename to packages/typespec-python/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/README.md.jinja2 b/packages/typespec-python/pygen/codegen/templates/packaging_templates/README.md.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/README.md.jinja2 rename to packages/typespec-python/pygen/codegen/templates/packaging_templates/README.md.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 b/packages/typespec-python/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 rename to packages/typespec-python/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/setup.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/packaging_templates/setup.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/packaging_templates/setup.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/packaging_templates/setup.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/paging_operation.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/paging_operation.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/paging_operation.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/paging_operation.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/patch.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/patch.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/patch.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/patch.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/pkgutil_init.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/pkgutil_init.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/pkgutil_init.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/pkgutil_init.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/request_builder.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/request_builder.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/request_builder.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/request_builder.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/request_builders.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/request_builders.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/request_builders.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/request_builders.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/rest_init.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/rest_init.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/rest_init.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/rest_init.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/sample.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/sample.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/sample.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/sample.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/serialization.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/serialization.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/serialization.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/serialization.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/test.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/test.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/test.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/test.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/testpreparer.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/testpreparer.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/testpreparer.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/testpreparer.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/types.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/types.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/types.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/types.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/validation.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/validation.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/validation.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/validation.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/vendor.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/vendor.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/vendor.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/vendor.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/codegen/templates/version.py.jinja2 b/packages/typespec-python/pygen/codegen/templates/version.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/pygen/codegen/templates/version.py.jinja2 rename to packages/typespec-python/pygen/codegen/templates/version.py.jinja2 diff --git a/packages/typespec-python/pygen/pygen/m2r/__init__.py b/packages/typespec-python/pygen/m2r/__init__.py similarity index 100% rename from packages/typespec-python/pygen/pygen/m2r/__init__.py rename to packages/typespec-python/pygen/m2r/__init__.py diff --git a/packages/typespec-python/pygen/pygen/postprocess/__init__.py b/packages/typespec-python/pygen/postprocess/__init__.py similarity index 100% rename from packages/typespec-python/pygen/pygen/postprocess/__init__.py rename to packages/typespec-python/pygen/postprocess/__init__.py diff --git a/packages/typespec-python/pygen/pygen/postprocess/get_all.py b/packages/typespec-python/pygen/postprocess/get_all.py similarity index 100% rename from packages/typespec-python/pygen/pygen/postprocess/get_all.py rename to packages/typespec-python/pygen/postprocess/get_all.py diff --git a/packages/typespec-python/pygen/pygen/postprocess/venvtools.py b/packages/typespec-python/pygen/postprocess/venvtools.py similarity index 100% rename from packages/typespec-python/pygen/pygen/postprocess/venvtools.py rename to packages/typespec-python/pygen/postprocess/venvtools.py diff --git a/packages/typespec-python/pygen/pygen/preprocess/__init__.py b/packages/typespec-python/pygen/preprocess/__init__.py similarity index 100% rename from packages/typespec-python/pygen/pygen/preprocess/__init__.py rename to packages/typespec-python/pygen/preprocess/__init__.py diff --git a/packages/typespec-python/pygen/pygen/preprocess/helpers.py b/packages/typespec-python/pygen/preprocess/helpers.py similarity index 100% rename from packages/typespec-python/pygen/pygen/preprocess/helpers.py rename to packages/typespec-python/pygen/preprocess/helpers.py diff --git a/packages/typespec-python/pygen/pygen/preprocess/python_mappings.py b/packages/typespec-python/pygen/preprocess/python_mappings.py similarity index 100% rename from packages/typespec-python/pygen/pygen/preprocess/python_mappings.py rename to packages/typespec-python/pygen/preprocess/python_mappings.py diff --git a/packages/typespec-python/pygen/pygen/utils.py b/packages/typespec-python/pygen/utils.py similarity index 100% rename from packages/typespec-python/pygen/pygen/utils.py rename to packages/typespec-python/pygen/utils.py From 175d57db545f29565c34a5b2107bacdfccd438ea Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 6 Jun 2024 16:46:24 -0400 Subject: [PATCH 64/75] make pygen a package again --- packages/autorest.python/package.json | 4 ++-- packages/autorest.python/requirements.txt | 2 +- .../scripts/{copy-pygen.js => copy-generator.js} | 4 ++-- packages/autorest.python/scripts/install.py | 3 +-- packages/autorest.python/scripts/start.py | 2 +- .../autorest.python/test/unittests/requirements.txt | 2 +- packages/typespec-python/dev_requirements.txt | 2 +- packages/typespec-python/{pygen => generator}/LICENSE | 0 .../typespec-python/{pygen => generator}/README.md | 0 .../{pygen => generator}/dev_requirements.txt | 0 .../typespec-python/{ => generator}/pygen/__init__.py | 0 .../typespec-python/{ => generator}/pygen/_version.py | 0 .../{ => generator}/pygen/black/__init__.py | 0 .../{ => generator}/pygen/codegen/__init__.py | 0 .../{ => generator}/pygen/codegen/_utils.py | 0 .../{ => generator}/pygen/codegen/models/__init__.py | 0 .../{ => generator}/pygen/codegen/models/base.py | 0 .../pygen/codegen/models/base_builder.py | 0 .../{ => generator}/pygen/codegen/models/client.py | 0 .../{ => generator}/pygen/codegen/models/code_model.py | 0 .../pygen/codegen/models/combined_type.py | 0 .../pygen/codegen/models/constant_type.py | 0 .../pygen/codegen/models/credential_types.py | 0 .../pygen/codegen/models/dictionary_type.py | 0 .../{ => generator}/pygen/codegen/models/enum_type.py | 0 .../{ => generator}/pygen/codegen/models/imports.py | 0 .../{ => generator}/pygen/codegen/models/list_type.py | 0 .../pygen/codegen/models/lro_operation.py | 0 .../pygen/codegen/models/lro_paging_operation.py | 0 .../{ => generator}/pygen/codegen/models/model_type.py | 0 .../{ => generator}/pygen/codegen/models/operation.py | 0 .../pygen/codegen/models/operation_group.py | 0 .../pygen/codegen/models/paging_operation.py | 0 .../{ => generator}/pygen/codegen/models/parameter.py | 0 .../pygen/codegen/models/parameter_list.py | 0 .../pygen/codegen/models/primitive_types.py | 0 .../{ => generator}/pygen/codegen/models/property.py | 0 .../pygen/codegen/models/request_builder.py | 0 .../pygen/codegen/models/request_builder_parameter.py | 0 .../{ => generator}/pygen/codegen/models/response.py | 0 .../{ => generator}/pygen/codegen/models/utils.py | 0 .../pygen/codegen/serializers/__init__.py | 0 .../pygen/codegen/serializers/base_serializer.py | 0 .../pygen/codegen/serializers/builder_serializer.py | 0 .../pygen/codegen/serializers/client_serializer.py | 0 .../pygen/codegen/serializers/enum_serializer.py | 0 .../pygen/codegen/serializers/general_serializer.py | 0 .../pygen/codegen/serializers/import_serializer.py | 0 .../pygen/codegen/serializers/metadata_serializer.py | 0 .../pygen/codegen/serializers/model_init_serializer.py | 0 .../pygen/codegen/serializers/model_serializer.py | 0 .../codegen/serializers/operation_groups_serializer.py | 0 .../codegen/serializers/operations_init_serializer.py | 0 .../pygen/codegen/serializers/parameter_serializer.py | 0 .../pygen/codegen/serializers/patch_serializer.py | 0 .../codegen/serializers/request_builders_serializer.py | 0 .../pygen/codegen/serializers/sample_serializer.py | 0 .../pygen/codegen/serializers/test_serializer.py | 0 .../pygen/codegen/serializers/types_serializer.py | 0 .../{ => generator}/pygen/codegen/serializers/utils.py | 0 .../pygen/codegen/templates/client.py.jinja2 | 0 .../pygen/codegen/templates/client_container.py.jinja2 | 0 .../pygen/codegen/templates/config.py.jinja2 | 0 .../pygen/codegen/templates/config_container.py.jinja2 | 0 .../pygen/codegen/templates/conftest.py.jinja2 | 0 .../pygen/codegen/templates/enum.py.jinja2 | 0 .../pygen/codegen/templates/enum_container.py.jinja2 | 0 .../pygen/codegen/templates/init.py.jinja2 | 0 .../pygen/codegen/templates/keywords.jinja2 | 0 .../pygen/codegen/templates/lro_operation.py.jinja2 | 0 .../codegen/templates/lro_paging_operation.py.jinja2 | 0 .../pygen/codegen/templates/macros.jinja2 | 0 .../pygen/codegen/templates/metadata.json.jinja2 | 0 .../pygen/codegen/templates/model_base.py.jinja2 | 0 .../pygen/codegen/templates/model_container.py.jinja2 | 0 .../pygen/codegen/templates/model_dpg.py.jinja2 | 0 .../pygen/codegen/templates/model_init.py.jinja2 | 0 .../pygen/codegen/templates/model_msrest.py.jinja2 | 0 .../pygen/codegen/templates/operation.py.jinja2 | 0 .../pygen/codegen/templates/operation_group.py.jinja2 | 0 .../templates/operation_groups_container.py.jinja2 | 0 .../pygen/codegen/templates/operation_tools.jinja2 | 0 .../codegen/templates/operations_folder_init.py.jinja2 | 0 .../templates/packaging_templates/CHANGELOG.md.jinja2 | 0 .../templates/packaging_templates/LICENSE.jinja2 | 0 .../templates/packaging_templates/MANIFEST.in.jinja2 | 0 .../templates/packaging_templates/README.md.jinja2 | 0 .../packaging_templates/dev_requirements.txt.jinja2 | 0 .../templates/packaging_templates/setup.py.jinja2 | 0 .../pygen/codegen/templates/paging_operation.py.jinja2 | 0 .../pygen/codegen/templates/patch.py.jinja2 | 0 .../pygen/codegen/templates/pkgutil_init.py.jinja2 | 0 .../pygen/codegen/templates/request_builder.py.jinja2 | 0 .../pygen/codegen/templates/request_builders.py.jinja2 | 0 .../pygen/codegen/templates/rest_init.py.jinja2 | 0 .../pygen/codegen/templates/sample.py.jinja2 | 0 .../pygen/codegen/templates/serialization.py.jinja2 | 0 .../pygen/codegen/templates/test.py.jinja2 | 0 .../pygen/codegen/templates/testpreparer.py.jinja2 | 0 .../pygen/codegen/templates/types.py.jinja2 | 0 .../pygen/codegen/templates/validation.py.jinja2 | 0 .../pygen/codegen/templates/vendor.py.jinja2 | 0 .../pygen/codegen/templates/version.py.jinja2 | 0 .../{ => generator}/pygen/m2r/__init__.py | 0 .../{ => generator}/pygen/postprocess/__init__.py | 0 .../{ => generator}/pygen/postprocess/get_all.py | 0 .../{ => generator}/pygen/postprocess/venvtools.py | 0 .../{ => generator}/pygen/preprocess/__init__.py | 0 .../{ => generator}/pygen/preprocess/helpers.py | 0 .../pygen/preprocess/python_mappings.py | 0 .../typespec-python/{ => generator}/pygen/utils.py | 0 .../{pygen => generator}/requirements.txt | 0 packages/typespec-python/{pygen => generator}/setup.py | 1 - packages/typespec-python/package.json | 2 +- packages/typespec-python/scripts/install.py | 6 +++--- packages/typespec-python/scripts/prepare.py | 4 ++-- packages/typespec-python/scripts/run_tsp.py | 10 +++++----- packages/typespec-python/scripts/venvtools.py | 2 +- 118 files changed, 21 insertions(+), 23 deletions(-) rename packages/autorest.python/scripts/{copy-pygen.js => copy-generator.js} (74%) rename packages/typespec-python/{pygen => generator}/LICENSE (100%) rename packages/typespec-python/{pygen => generator}/README.md (100%) rename packages/typespec-python/{pygen => generator}/dev_requirements.txt (100%) rename packages/typespec-python/{ => generator}/pygen/__init__.py (100%) rename packages/typespec-python/{ => generator}/pygen/_version.py (100%) rename packages/typespec-python/{ => generator}/pygen/black/__init__.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/__init__.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/_utils.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/__init__.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/base.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/base_builder.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/client.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/code_model.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/combined_type.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/constant_type.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/credential_types.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/dictionary_type.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/enum_type.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/imports.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/list_type.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/lro_operation.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/lro_paging_operation.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/model_type.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/operation.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/operation_group.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/paging_operation.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/parameter.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/parameter_list.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/primitive_types.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/property.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/request_builder.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/request_builder_parameter.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/response.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/models/utils.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/__init__.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/base_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/builder_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/client_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/enum_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/general_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/import_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/metadata_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/model_init_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/model_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/operation_groups_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/operations_init_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/parameter_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/patch_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/request_builders_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/sample_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/test_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/types_serializer.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/serializers/utils.py (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/client.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/client_container.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/config.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/config_container.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/conftest.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/enum.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/enum_container.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/init.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/keywords.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/lro_operation.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/lro_paging_operation.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/macros.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/metadata.json.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/model_base.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/model_container.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/model_dpg.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/model_init.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/model_msrest.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/operation.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/operation_group.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/operation_groups_container.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/operation_tools.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/operations_folder_init.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/packaging_templates/README.md.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/packaging_templates/setup.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/paging_operation.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/patch.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/pkgutil_init.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/request_builder.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/request_builders.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/rest_init.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/sample.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/serialization.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/test.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/testpreparer.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/types.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/validation.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/vendor.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/codegen/templates/version.py.jinja2 (100%) rename packages/typespec-python/{ => generator}/pygen/m2r/__init__.py (100%) rename packages/typespec-python/{ => generator}/pygen/postprocess/__init__.py (100%) rename packages/typespec-python/{ => generator}/pygen/postprocess/get_all.py (100%) rename packages/typespec-python/{ => generator}/pygen/postprocess/venvtools.py (100%) rename packages/typespec-python/{ => generator}/pygen/preprocess/__init__.py (100%) rename packages/typespec-python/{ => generator}/pygen/preprocess/helpers.py (100%) rename packages/typespec-python/{ => generator}/pygen/preprocess/python_mappings.py (100%) rename packages/typespec-python/{ => generator}/pygen/utils.py (100%) rename packages/typespec-python/{pygen => generator}/requirements.txt (100%) rename packages/typespec-python/{pygen => generator}/setup.py (98%) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index acbabf5c074..a24e324cac6 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -4,8 +4,8 @@ "description": "The Python extension for generators in AutoRest.", "scripts": { "start": "node ./scripts/run-python3.js ./scripts/start.py", - "copy-pygen": "node ./scripts/copy-pygen.js", - "install": "npm run copy-pygen && node ./scripts/run-python3.js ./scripts/install.py", + "copy-generator": "node ./scripts/copy-generator.js", + "install": "npm run copy-generator && node ./scripts/run-python3.js ./scripts/install.py", "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" }, "main": "index.js", diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index 1308f75c4a9..edfb2c92452 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -1,2 +1,2 @@ json-rpc==1.14.0 --e ./pygen +-e ./generator diff --git a/packages/autorest.python/scripts/copy-pygen.js b/packages/autorest.python/scripts/copy-generator.js similarity index 74% rename from packages/autorest.python/scripts/copy-pygen.js rename to packages/autorest.python/scripts/copy-generator.js index 8fd38738b4d..9a3c1f8dbf8 100644 --- a/packages/autorest.python/scripts/copy-pygen.js +++ b/packages/autorest.python/scripts/copy-generator.js @@ -3,8 +3,8 @@ const path = require('path'); const url = require('url'); // Define the source and destination directories -const sourceDir = path.join(__dirname, "..", "node_modules", "@azure-tools", "typespec-python", "pygen"); -const destDir = path.join(__dirname, "..", "pygen"); +const sourceDir = path.join(__dirname, "..", "node_modules", "@azure-tools", "typespec-python", "generator"); +const destDir = path.join(__dirname, "..", "generator"); // Copy the source directory to the destination directory fs.copySync(sourceDir, destDir); diff --git a/packages/autorest.python/scripts/install.py b/packages/autorest.python/scripts/install.py index 573330574d6..2610b9227f3 100644 --- a/packages/autorest.python/scripts/install.py +++ b/packages/autorest.python/scripts/install.py @@ -32,8 +32,7 @@ def main(): - # we copy pygen's venv into autorest.python - venv_path = _ROOT_DIR / "pygen" / "venv" + venv_path = _ROOT_DIR / "generator" / "venv" assert venv_path.exists() # Otherwise install was not done env_builder = venv.EnvBuilder(with_pip=True) diff --git a/packages/autorest.python/scripts/start.py b/packages/autorest.python/scripts/start.py index 2b77b7439d0..c9321edd01a 100644 --- a/packages/autorest.python/scripts/start.py +++ b/packages/autorest.python/scripts/start.py @@ -19,7 +19,7 @@ def main(): - venv_path = _ROOT_DIR / "pygen" / "venv" + venv_path = _ROOT_DIR / "generator" / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/packages/autorest.python/test/unittests/requirements.txt b/packages/autorest.python/test/unittests/requirements.txt index fd3e9c4335c..64ba07b3d92 100644 --- a/packages/autorest.python/test/unittests/requirements.txt +++ b/packages/autorest.python/test/unittests/requirements.txt @@ -4,4 +4,4 @@ pytest pytest-cov azure-core==1.30.0 -e ../../. --e ../../node_modules/@azure-tools/typespec-python/pygen +-e ../../node_modules/@azure-tools/typespec-python/generator diff --git a/packages/typespec-python/dev_requirements.txt b/packages/typespec-python/dev_requirements.txt index d64c38e4493..6dc5274999a 100644 --- a/packages/typespec-python/dev_requirements.txt +++ b/packages/typespec-python/dev_requirements.txt @@ -1,4 +1,4 @@ -r ../../eng/requirements.txt -r ../../eng/dev_requirements.txt --e pygen +-e generator diff --git a/packages/typespec-python/pygen/LICENSE b/packages/typespec-python/generator/LICENSE similarity index 100% rename from packages/typespec-python/pygen/LICENSE rename to packages/typespec-python/generator/LICENSE diff --git a/packages/typespec-python/pygen/README.md b/packages/typespec-python/generator/README.md similarity index 100% rename from packages/typespec-python/pygen/README.md rename to packages/typespec-python/generator/README.md diff --git a/packages/typespec-python/pygen/dev_requirements.txt b/packages/typespec-python/generator/dev_requirements.txt similarity index 100% rename from packages/typespec-python/pygen/dev_requirements.txt rename to packages/typespec-python/generator/dev_requirements.txt diff --git a/packages/typespec-python/pygen/__init__.py b/packages/typespec-python/generator/pygen/__init__.py similarity index 100% rename from packages/typespec-python/pygen/__init__.py rename to packages/typespec-python/generator/pygen/__init__.py diff --git a/packages/typespec-python/pygen/_version.py b/packages/typespec-python/generator/pygen/_version.py similarity index 100% rename from packages/typespec-python/pygen/_version.py rename to packages/typespec-python/generator/pygen/_version.py diff --git a/packages/typespec-python/pygen/black/__init__.py b/packages/typespec-python/generator/pygen/black/__init__.py similarity index 100% rename from packages/typespec-python/pygen/black/__init__.py rename to packages/typespec-python/generator/pygen/black/__init__.py diff --git a/packages/typespec-python/pygen/codegen/__init__.py b/packages/typespec-python/generator/pygen/codegen/__init__.py similarity index 100% rename from packages/typespec-python/pygen/codegen/__init__.py rename to packages/typespec-python/generator/pygen/codegen/__init__.py diff --git a/packages/typespec-python/pygen/codegen/_utils.py b/packages/typespec-python/generator/pygen/codegen/_utils.py similarity index 100% rename from packages/typespec-python/pygen/codegen/_utils.py rename to packages/typespec-python/generator/pygen/codegen/_utils.py diff --git a/packages/typespec-python/pygen/codegen/models/__init__.py b/packages/typespec-python/generator/pygen/codegen/models/__init__.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/__init__.py rename to packages/typespec-python/generator/pygen/codegen/models/__init__.py diff --git a/packages/typespec-python/pygen/codegen/models/base.py b/packages/typespec-python/generator/pygen/codegen/models/base.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/base.py rename to packages/typespec-python/generator/pygen/codegen/models/base.py diff --git a/packages/typespec-python/pygen/codegen/models/base_builder.py b/packages/typespec-python/generator/pygen/codegen/models/base_builder.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/base_builder.py rename to packages/typespec-python/generator/pygen/codegen/models/base_builder.py diff --git a/packages/typespec-python/pygen/codegen/models/client.py b/packages/typespec-python/generator/pygen/codegen/models/client.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/client.py rename to packages/typespec-python/generator/pygen/codegen/models/client.py diff --git a/packages/typespec-python/pygen/codegen/models/code_model.py b/packages/typespec-python/generator/pygen/codegen/models/code_model.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/code_model.py rename to packages/typespec-python/generator/pygen/codegen/models/code_model.py diff --git a/packages/typespec-python/pygen/codegen/models/combined_type.py b/packages/typespec-python/generator/pygen/codegen/models/combined_type.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/combined_type.py rename to packages/typespec-python/generator/pygen/codegen/models/combined_type.py diff --git a/packages/typespec-python/pygen/codegen/models/constant_type.py b/packages/typespec-python/generator/pygen/codegen/models/constant_type.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/constant_type.py rename to packages/typespec-python/generator/pygen/codegen/models/constant_type.py diff --git a/packages/typespec-python/pygen/codegen/models/credential_types.py b/packages/typespec-python/generator/pygen/codegen/models/credential_types.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/credential_types.py rename to packages/typespec-python/generator/pygen/codegen/models/credential_types.py diff --git a/packages/typespec-python/pygen/codegen/models/dictionary_type.py b/packages/typespec-python/generator/pygen/codegen/models/dictionary_type.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/dictionary_type.py rename to packages/typespec-python/generator/pygen/codegen/models/dictionary_type.py diff --git a/packages/typespec-python/pygen/codegen/models/enum_type.py b/packages/typespec-python/generator/pygen/codegen/models/enum_type.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/enum_type.py rename to packages/typespec-python/generator/pygen/codegen/models/enum_type.py diff --git a/packages/typespec-python/pygen/codegen/models/imports.py b/packages/typespec-python/generator/pygen/codegen/models/imports.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/imports.py rename to packages/typespec-python/generator/pygen/codegen/models/imports.py diff --git a/packages/typespec-python/pygen/codegen/models/list_type.py b/packages/typespec-python/generator/pygen/codegen/models/list_type.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/list_type.py rename to packages/typespec-python/generator/pygen/codegen/models/list_type.py diff --git a/packages/typespec-python/pygen/codegen/models/lro_operation.py b/packages/typespec-python/generator/pygen/codegen/models/lro_operation.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/lro_operation.py rename to packages/typespec-python/generator/pygen/codegen/models/lro_operation.py diff --git a/packages/typespec-python/pygen/codegen/models/lro_paging_operation.py b/packages/typespec-python/generator/pygen/codegen/models/lro_paging_operation.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/lro_paging_operation.py rename to packages/typespec-python/generator/pygen/codegen/models/lro_paging_operation.py diff --git a/packages/typespec-python/pygen/codegen/models/model_type.py b/packages/typespec-python/generator/pygen/codegen/models/model_type.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/model_type.py rename to packages/typespec-python/generator/pygen/codegen/models/model_type.py diff --git a/packages/typespec-python/pygen/codegen/models/operation.py b/packages/typespec-python/generator/pygen/codegen/models/operation.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/operation.py rename to packages/typespec-python/generator/pygen/codegen/models/operation.py diff --git a/packages/typespec-python/pygen/codegen/models/operation_group.py b/packages/typespec-python/generator/pygen/codegen/models/operation_group.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/operation_group.py rename to packages/typespec-python/generator/pygen/codegen/models/operation_group.py diff --git a/packages/typespec-python/pygen/codegen/models/paging_operation.py b/packages/typespec-python/generator/pygen/codegen/models/paging_operation.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/paging_operation.py rename to packages/typespec-python/generator/pygen/codegen/models/paging_operation.py diff --git a/packages/typespec-python/pygen/codegen/models/parameter.py b/packages/typespec-python/generator/pygen/codegen/models/parameter.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/parameter.py rename to packages/typespec-python/generator/pygen/codegen/models/parameter.py diff --git a/packages/typespec-python/pygen/codegen/models/parameter_list.py b/packages/typespec-python/generator/pygen/codegen/models/parameter_list.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/parameter_list.py rename to packages/typespec-python/generator/pygen/codegen/models/parameter_list.py diff --git a/packages/typespec-python/pygen/codegen/models/primitive_types.py b/packages/typespec-python/generator/pygen/codegen/models/primitive_types.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/primitive_types.py rename to packages/typespec-python/generator/pygen/codegen/models/primitive_types.py diff --git a/packages/typespec-python/pygen/codegen/models/property.py b/packages/typespec-python/generator/pygen/codegen/models/property.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/property.py rename to packages/typespec-python/generator/pygen/codegen/models/property.py diff --git a/packages/typespec-python/pygen/codegen/models/request_builder.py b/packages/typespec-python/generator/pygen/codegen/models/request_builder.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/request_builder.py rename to packages/typespec-python/generator/pygen/codegen/models/request_builder.py diff --git a/packages/typespec-python/pygen/codegen/models/request_builder_parameter.py b/packages/typespec-python/generator/pygen/codegen/models/request_builder_parameter.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/request_builder_parameter.py rename to packages/typespec-python/generator/pygen/codegen/models/request_builder_parameter.py diff --git a/packages/typespec-python/pygen/codegen/models/response.py b/packages/typespec-python/generator/pygen/codegen/models/response.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/response.py rename to packages/typespec-python/generator/pygen/codegen/models/response.py diff --git a/packages/typespec-python/pygen/codegen/models/utils.py b/packages/typespec-python/generator/pygen/codegen/models/utils.py similarity index 100% rename from packages/typespec-python/pygen/codegen/models/utils.py rename to packages/typespec-python/generator/pygen/codegen/models/utils.py diff --git a/packages/typespec-python/pygen/codegen/serializers/__init__.py b/packages/typespec-python/generator/pygen/codegen/serializers/__init__.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/__init__.py rename to packages/typespec-python/generator/pygen/codegen/serializers/__init__.py diff --git a/packages/typespec-python/pygen/codegen/serializers/base_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/base_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/base_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/base_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/builder_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/builder_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/builder_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/builder_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/client_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/client_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/client_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/client_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/enum_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/enum_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/enum_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/enum_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/general_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/general_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/general_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/general_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/import_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/import_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/import_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/import_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/metadata_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/metadata_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/metadata_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/metadata_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/model_init_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/model_init_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/model_init_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/model_init_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/model_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/model_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/model_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/model_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/operation_groups_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/operation_groups_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/operation_groups_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/operation_groups_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/operations_init_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/operations_init_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/operations_init_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/operations_init_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/parameter_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/parameter_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/parameter_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/parameter_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/patch_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/patch_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/patch_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/patch_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/request_builders_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/request_builders_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/request_builders_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/request_builders_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/sample_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/sample_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/sample_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/sample_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/test_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/test_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/test_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/test_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/types_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/types_serializer.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/types_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/types_serializer.py diff --git a/packages/typespec-python/pygen/codegen/serializers/utils.py b/packages/typespec-python/generator/pygen/codegen/serializers/utils.py similarity index 100% rename from packages/typespec-python/pygen/codegen/serializers/utils.py rename to packages/typespec-python/generator/pygen/codegen/serializers/utils.py diff --git a/packages/typespec-python/pygen/codegen/templates/client.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/client.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/client.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/client.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/client_container.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/client_container.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/client_container.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/client_container.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/config.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/config.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/config.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/config.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/config_container.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/config_container.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/config_container.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/config_container.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/conftest.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/conftest.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/conftest.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/conftest.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/enum.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/enum.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/enum.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/enum.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/enum_container.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/enum_container.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/enum_container.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/enum_container.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/init.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/init.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/init.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/init.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/keywords.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/keywords.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/keywords.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/keywords.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/lro_operation.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/lro_operation.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/lro_operation.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/lro_operation.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/lro_paging_operation.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/lro_paging_operation.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/macros.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/macros.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/macros.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/macros.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/metadata.json.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/metadata.json.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/metadata.json.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/metadata.json.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/model_base.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/model_base.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/model_base.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/model_base.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/model_container.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/model_container.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/model_container.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/model_container.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/model_dpg.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/model_dpg.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/model_dpg.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/model_dpg.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/model_init.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/model_init.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/model_init.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/model_init.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/model_msrest.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/model_msrest.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/model_msrest.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/model_msrest.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/operation.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/operation.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/operation.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/operation.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/operation_group.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/operation_group.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/operation_group.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/operation_group.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/operation_groups_container.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/operation_groups_container.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/operation_tools.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/operation_tools.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/operation_tools.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/operation_tools.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/operations_folder_init.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/operations_folder_init.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/packaging_templates/README.md.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/packaging_templates/README.md.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/packaging_templates/setup.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/packaging_templates/setup.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/paging_operation.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/paging_operation.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/paging_operation.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/paging_operation.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/patch.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/patch.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/patch.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/patch.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/pkgutil_init.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/pkgutil_init.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/request_builder.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/request_builder.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/request_builder.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/request_builder.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/request_builders.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/request_builders.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/request_builders.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/request_builders.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/rest_init.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/rest_init.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/rest_init.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/rest_init.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/sample.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/sample.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/sample.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/sample.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/serialization.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/serialization.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/serialization.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/serialization.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/test.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/test.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/test.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/test.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/testpreparer.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/testpreparer.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/testpreparer.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/testpreparer.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/types.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/types.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/types.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/types.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/validation.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/validation.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/validation.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/validation.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/vendor.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/vendor.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/vendor.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/vendor.py.jinja2 diff --git a/packages/typespec-python/pygen/codegen/templates/version.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/version.py.jinja2 similarity index 100% rename from packages/typespec-python/pygen/codegen/templates/version.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/version.py.jinja2 diff --git a/packages/typespec-python/pygen/m2r/__init__.py b/packages/typespec-python/generator/pygen/m2r/__init__.py similarity index 100% rename from packages/typespec-python/pygen/m2r/__init__.py rename to packages/typespec-python/generator/pygen/m2r/__init__.py diff --git a/packages/typespec-python/pygen/postprocess/__init__.py b/packages/typespec-python/generator/pygen/postprocess/__init__.py similarity index 100% rename from packages/typespec-python/pygen/postprocess/__init__.py rename to packages/typespec-python/generator/pygen/postprocess/__init__.py diff --git a/packages/typespec-python/pygen/postprocess/get_all.py b/packages/typespec-python/generator/pygen/postprocess/get_all.py similarity index 100% rename from packages/typespec-python/pygen/postprocess/get_all.py rename to packages/typespec-python/generator/pygen/postprocess/get_all.py diff --git a/packages/typespec-python/pygen/postprocess/venvtools.py b/packages/typespec-python/generator/pygen/postprocess/venvtools.py similarity index 100% rename from packages/typespec-python/pygen/postprocess/venvtools.py rename to packages/typespec-python/generator/pygen/postprocess/venvtools.py diff --git a/packages/typespec-python/pygen/preprocess/__init__.py b/packages/typespec-python/generator/pygen/preprocess/__init__.py similarity index 100% rename from packages/typespec-python/pygen/preprocess/__init__.py rename to packages/typespec-python/generator/pygen/preprocess/__init__.py diff --git a/packages/typespec-python/pygen/preprocess/helpers.py b/packages/typespec-python/generator/pygen/preprocess/helpers.py similarity index 100% rename from packages/typespec-python/pygen/preprocess/helpers.py rename to packages/typespec-python/generator/pygen/preprocess/helpers.py diff --git a/packages/typespec-python/pygen/preprocess/python_mappings.py b/packages/typespec-python/generator/pygen/preprocess/python_mappings.py similarity index 100% rename from packages/typespec-python/pygen/preprocess/python_mappings.py rename to packages/typespec-python/generator/pygen/preprocess/python_mappings.py diff --git a/packages/typespec-python/pygen/utils.py b/packages/typespec-python/generator/pygen/utils.py similarity index 100% rename from packages/typespec-python/pygen/utils.py rename to packages/typespec-python/generator/pygen/utils.py diff --git a/packages/typespec-python/pygen/requirements.txt b/packages/typespec-python/generator/requirements.txt similarity index 100% rename from packages/typespec-python/pygen/requirements.txt rename to packages/typespec-python/generator/requirements.txt diff --git a/packages/typespec-python/pygen/setup.py b/packages/typespec-python/generator/setup.py similarity index 98% rename from packages/typespec-python/pygen/setup.py rename to packages/typespec-python/generator/setup.py index 79bba224efe..b19d4f2692e 100644 --- a/packages/typespec-python/pygen/setup.py +++ b/packages/typespec-python/generator/setup.py @@ -47,7 +47,6 @@ ] ), install_requires=[ - "json-rpc", "Jinja2 >= 2.11", # I need "include" and auto-context + blank line are not indented by default "pyyaml", "m2r2", diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 9e56efa1f43..87f68225510 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -40,7 +40,7 @@ "files": [ "dist/**", "!dist/test/**", - "pygen/**", + "generator/**", "scripts/**" ], "peerDependencies": { diff --git a/packages/typespec-python/scripts/install.py b/packages/typespec-python/scripts/install.py index 83ad20445e7..80138a01803 100644 --- a/packages/typespec-python/scripts/install.py +++ b/packages/typespec-python/scripts/install.py @@ -31,7 +31,7 @@ def main(): - venv_path = _ROOT_DIR / "pygen" / "venv" + venv_path = _ROOT_DIR / "generator" / "venv" if venv_path.exists(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) @@ -41,8 +41,8 @@ def main(): venv_context = env_builder.context python_run(venv_context, "pip", ["install", "-U", "pip"]) - python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/pygen/requirements.txt"]) - python_run(venv_context, "pip", ["install", "-e", f"{_ROOT_DIR}/pygen"]) + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/generator/requirements.txt"]) + python_run(venv_context, "pip", ["install", "-e", f"{_ROOT_DIR}/generator"]) if __name__ == "__main__": diff --git a/packages/typespec-python/scripts/prepare.py b/packages/typespec-python/scripts/prepare.py index fe29b6da616..6ba7db0d5b0 100644 --- a/packages/typespec-python/scripts/prepare.py +++ b/packages/typespec-python/scripts/prepare.py @@ -20,7 +20,7 @@ def main(): - venv_path = _ROOT_DIR / "pygen" / "venv" + venv_path = _ROOT_DIR / "generator" / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done @@ -28,7 +28,7 @@ def main(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) try: - python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/pygen/dev_requirements.txt"]) + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/generator/dev_requirements.txt"]) except FileNotFoundError as e: raise ValueError(e.filename) diff --git a/packages/typespec-python/scripts/run_tsp.py b/packages/typespec-python/scripts/run_tsp.py index 5365d316bb8..c9d360706fe 100644 --- a/packages/typespec-python/scripts/run_tsp.py +++ b/packages/typespec-python/scripts/run_tsp.py @@ -14,7 +14,7 @@ _LOGGER = logging.getLogger(__name__) if __name__ == "__main__": - venv_path = _ROOT_DIR / "pygen" / "venv" + venv_path = _ROOT_DIR / "generator" / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done @@ -34,7 +34,7 @@ breakpoint() # pylint: disable=undefined-variable # run m2r - python_run(venv_context, "pygen.m2r.__init__", command=sys.argv[1:]) - python_run(venv_context, "pygen.preprocess.__init__", command=sys.argv[1:]) - python_run(venv_context, "pygen.codegen.__init__", command=sys.argv[1:]) - python_run(venv_context, "pygen.black.__init__", command=sys.argv[1:]) + python_run(venv_context, "m2r.__init__", command=sys.argv[1:]) + python_run(venv_context, "preprocess.__init__", command=sys.argv[1:]) + python_run(venv_context, "codegen.__init__", command=sys.argv[1:]) + python_run(venv_context, "black.__init__", command=sys.argv[1:]) diff --git a/packages/typespec-python/scripts/venvtools.py b/packages/typespec-python/scripts/venvtools.py index ecddd2fc0bf..690f09e174e 100644 --- a/packages/typespec-python/scripts/venvtools.py +++ b/packages/typespec-python/scripts/venvtools.py @@ -73,7 +73,7 @@ def python_run(venv_context, module, command=None, *, additional_dir="."): print("Executing: {}".format(" ".join(cmd_line))) subprocess.run( cmd_line, - cwd=_ROOT_DIR / "pygen" / additional_dir, + cwd=_ROOT_DIR / "generator" / "pygen" / additional_dir, check=True, ) except subprocess.CalledProcessError as err: From 2716cf3283e040e02c0889e1ba6afaffd8cd9056 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 6 Jun 2024 18:07:44 -0400 Subject: [PATCH 65/75] keep trying with setup.py file --- packages/autorest.python/generator/LICENSE | 21 + packages/autorest.python/generator/README.md | 1 + .../generator/dev_requirements.txt | 4 + .../generator/pygen/__init__.py | 107 + .../generator/pygen/_version.py | 7 + .../generator/pygen/black/__init__.py | 0 .../generator/pygen/codegen/__init__.py | 334 +++ .../generator/pygen/codegen/_utils.py | 16 + .../pygen/codegen/models/__init__.py | 202 ++ .../generator/pygen/codegen/models/base.py | 188 ++ .../pygen/codegen/models/base_builder.py | 119 + .../generator/pygen/codegen/models/client.py | 422 ++++ .../pygen/codegen/models/code_model.py | 245 ++ .../pygen/codegen/models/combined_type.py | 153 ++ .../pygen/codegen/models/constant_type.py | 134 ++ .../pygen/codegen/models/credential_types.py | 223 ++ .../pygen/codegen/models/dictionary_type.py | 131 ++ .../pygen/codegen/models/enum_type.py | 246 ++ .../generator/pygen/codegen/models/imports.py | 291 +++ .../pygen/codegen/models/list_type.py | 147 ++ .../pygen/codegen/models/lro_operation.py | 143 ++ .../codegen/models/lro_paging_operation.py | 32 + .../pygen/codegen/models/model_type.py | 350 +++ .../pygen/codegen/models/operation.py | 510 +++++ .../pygen/codegen/models/operation_group.py | 184 ++ .../pygen/codegen/models/paging_operation.py | 156 ++ .../pygen/codegen/models/parameter.py | 402 ++++ .../pygen/codegen/models/parameter_list.py | 390 ++++ .../pygen/codegen/models/primitive_types.py | 640 ++++++ .../pygen/codegen/models/property.py | 182 ++ .../pygen/codegen/models/request_builder.py | 189 ++ .../models/request_builder_parameter.py | 115 + .../pygen/codegen/models/response.py | 348 +++ .../generator/pygen/codegen/models/utils.py | 23 + .../pygen/codegen/serializers/__init__.py | 570 +++++ .../codegen/serializers/base_serializer.py | 21 + .../codegen/serializers/builder_serializer.py | 1453 ++++++++++++ .../codegen/serializers/client_serializer.py | 295 +++ .../codegen/serializers/enum_serializer.py | 15 + .../codegen/serializers/general_serializer.py | 212 ++ .../codegen/serializers/import_serializer.py | 127 ++ .../serializers/metadata_serializer.py | 198 ++ .../serializers/model_init_serializer.py | 33 + .../codegen/serializers/model_serializer.py | 287 +++ .../operation_groups_serializer.py | 89 + .../serializers/operations_init_serializer.py | 44 + .../serializers/parameter_serializer.py | 221 ++ .../codegen/serializers/patch_serializer.py | 19 + .../request_builders_serializer.py | 52 + .../codegen/serializers/sample_serializer.py | 163 ++ .../codegen/serializers/test_serializer.py | 263 +++ .../codegen/serializers/types_serializer.py | 31 + .../pygen/codegen/serializers/utils.py | 68 + .../pygen/codegen/templates/client.py.jinja2 | 37 + .../templates/client_container.py.jinja2 | 12 + .../pygen/codegen/templates/config.py.jinja2 | 73 + .../templates/config_container.py.jinja2 | 16 + .../codegen/templates/conftest.py.jinja2 | 28 + .../pygen/codegen/templates/enum.py.jinja2 | 13 + .../templates/enum_container.py.jinja2 | 10 + .../pygen/codegen/templates/init.py.jinja2 | 24 + .../pygen/codegen/templates/keywords.jinja2 | 19 + .../codegen/templates/lro_operation.py.jinja2 | 16 + .../templates/lro_paging_operation.py.jinja2 | 18 + .../pygen/codegen/templates/macros.jinja2 | 12 + .../codegen/templates/metadata.json.jinja2 | 167 ++ .../codegen/templates/model_base.py.jinja2 | 898 ++++++++ .../templates/model_container.py.jinja2 | 13 + .../codegen/templates/model_dpg.py.jinja2 | 90 + .../codegen/templates/model_init.py.jinja2 | 28 + .../codegen/templates/model_msrest.py.jinja2 | 92 + .../codegen/templates/operation.py.jinja2 | 21 + .../templates/operation_group.py.jinja2 | 75 + .../operation_groups_container.py.jinja2 | 20 + .../codegen/templates/operation_tools.jinja2 | 74 + .../operations_folder_init.py.jinja2 | 17 + .../packaging_templates/CHANGELOG.md.jinja2 | 6 + .../packaging_templates/LICENSE.jinja2 | 21 + .../packaging_templates/MANIFEST.in.jinja2 | 8 + .../packaging_templates/README.md.jinja2 | 107 + .../dev_requirements.txt.jinja2 | 9 + .../packaging_templates/setup.py.jinja2 | 110 + .../templates/paging_operation.py.jinja2 | 21 + .../pygen/codegen/templates/patch.py.jinja2 | 19 + .../codegen/templates/pkgutil_init.py.jinja2 | 1 + .../templates/request_builder.py.jinja2 | 28 + .../templates/request_builders.py.jinja2 | 10 + .../codegen/templates/rest_init.py.jinja2 | 12 + .../pygen/codegen/templates/sample.py.jinja2 | 44 + .../codegen/templates/serialization.py.jinja2 | 2004 +++++++++++++++++ .../pygen/codegen/templates/test.py.jinja2 | 26 + .../codegen/templates/testpreparer.py.jinja2 | 26 + .../pygen/codegen/templates/types.py.jinja2 | 8 + .../codegen/templates/validation.py.jinja2 | 38 + .../pygen/codegen/templates/vendor.py.jinja2 | 98 + .../pygen/codegen/templates/version.py.jinja2 | 4 + .../generator/pygen/m2r/__init__.py | 0 .../generator/pygen/postprocess/__init__.py | 183 ++ .../generator/pygen/postprocess/get_all.py | 19 + .../generator/pygen/postprocess/venvtools.py | 77 + .../generator/pygen/preprocess/__init__.py | 483 ++++ .../generator/pygen/preprocess/helpers.py | 27 + .../pygen/preprocess/python_mappings.py | 222 ++ .../autorest.python/generator/pygen/utils.py | 149 ++ .../generator/requirements.txt | 12 + packages/autorest.python/generator/setup.py | 55 + .../typespec-python/generator/pygen/black.py | 71 + .../typespec-python/generator/pygen/m2r.py | 65 + packages/typespec-python/scripts/run_tsp.py | 4 +- 109 files changed, 16554 insertions(+), 2 deletions(-) create mode 100644 packages/autorest.python/generator/LICENSE create mode 100644 packages/autorest.python/generator/README.md create mode 100644 packages/autorest.python/generator/dev_requirements.txt create mode 100644 packages/autorest.python/generator/pygen/__init__.py create mode 100644 packages/autorest.python/generator/pygen/_version.py rename packages/{typespec-python => autorest.python}/generator/pygen/black/__init__.py (100%) create mode 100644 packages/autorest.python/generator/pygen/codegen/__init__.py create mode 100644 packages/autorest.python/generator/pygen/codegen/_utils.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/__init__.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/base.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/base_builder.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/client.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/code_model.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/combined_type.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/constant_type.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/credential_types.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/dictionary_type.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/enum_type.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/imports.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/list_type.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/lro_operation.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/lro_paging_operation.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/model_type.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/operation.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/operation_group.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/paging_operation.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/parameter.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/parameter_list.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/primitive_types.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/property.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/request_builder.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/request_builder_parameter.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/response.py create mode 100644 packages/autorest.python/generator/pygen/codegen/models/utils.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/__init__.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/base_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/builder_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/client_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/enum_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/general_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/import_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/metadata_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/model_init_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/model_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/operation_groups_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/operations_init_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/parameter_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/patch_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/request_builders_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/sample_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/test_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/types_serializer.py create mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/utils.py create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/client.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/client_container.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/config.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/config_container.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/conftest.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/enum.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/enum_container.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/init.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/keywords.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/lro_operation.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/macros.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/metadata.json.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/model_base.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/model_container.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/model_dpg.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/model_init.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/model_msrest.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/operation.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/operation_group.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/operation_tools.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/paging_operation.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/patch.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/request_builder.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/request_builders.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/rest_init.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/sample.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/serialization.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/test.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/testpreparer.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/types.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/validation.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/vendor.py.jinja2 create mode 100644 packages/autorest.python/generator/pygen/codegen/templates/version.py.jinja2 rename packages/{typespec-python => autorest.python}/generator/pygen/m2r/__init__.py (100%) create mode 100644 packages/autorest.python/generator/pygen/postprocess/__init__.py create mode 100644 packages/autorest.python/generator/pygen/postprocess/get_all.py create mode 100644 packages/autorest.python/generator/pygen/postprocess/venvtools.py create mode 100644 packages/autorest.python/generator/pygen/preprocess/__init__.py create mode 100644 packages/autorest.python/generator/pygen/preprocess/helpers.py create mode 100644 packages/autorest.python/generator/pygen/preprocess/python_mappings.py create mode 100644 packages/autorest.python/generator/pygen/utils.py create mode 100644 packages/autorest.python/generator/requirements.txt create mode 100644 packages/autorest.python/generator/setup.py create mode 100644 packages/typespec-python/generator/pygen/black.py create mode 100644 packages/typespec-python/generator/pygen/m2r.py diff --git a/packages/autorest.python/generator/LICENSE b/packages/autorest.python/generator/LICENSE new file mode 100644 index 00000000000..21071075c24 --- /dev/null +++ b/packages/autorest.python/generator/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/packages/autorest.python/generator/README.md b/packages/autorest.python/generator/README.md new file mode 100644 index 00000000000..3d8a65a8919 --- /dev/null +++ b/packages/autorest.python/generator/README.md @@ -0,0 +1 @@ +# Core Library for Python Generation diff --git a/packages/autorest.python/generator/dev_requirements.txt b/packages/autorest.python/generator/dev_requirements.txt new file mode 100644 index 00000000000..32f2ff33251 --- /dev/null +++ b/packages/autorest.python/generator/dev_requirements.txt @@ -0,0 +1,4 @@ +-r ../../../eng/requirements.txt +-r ../../../eng/dev_requirements.txt +ptvsd==4.3.2 +types-PyYAML==6.0.12.8 diff --git a/packages/autorest.python/generator/pygen/__init__.py b/packages/autorest.python/generator/pygen/__init__.py new file mode 100644 index 00000000000..6050cbec00f --- /dev/null +++ b/packages/autorest.python/generator/pygen/__init__.py @@ -0,0 +1,107 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from pathlib import Path +import json +from abc import ABC, abstractmethod +from typing import Any, Dict, Union, List + +import yaml + +from ._version import VERSION + + +__version__ = VERSION +_LOGGER = logging.getLogger(__name__) + + +class ReaderAndWriter: + def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None: + self.output_folder = Path(output_folder) + self._list_file: List[str] = [] + try: + with open( + Path(self.output_folder) / Path("..") / Path("python.json"), + "r", + encoding="utf-8-sig", + ) as fd: + python_json = json.load(fd) + except Exception: # pylint: disable=broad-except + python_json = {} + self.options = kwargs + if python_json: + _LOGGER.warning("Loading python.json file. This behavior will be depreacted") + self.options.update(python_json) + + def read_file(self, path: Union[str, Path]) -> str: + """Directly reading from disk""" + # make path relative to output folder + try: + with open(self.output_folder / Path(path), "r", encoding="utf-8-sig") as fd: + return fd.read() + except FileNotFoundError: + return "" + + def write_file(self, filename: Union[str, Path], file_content: str) -> None: + """Directly writing to disk""" + file_folder = Path(filename).parent + if not Path.is_dir(self.output_folder / file_folder): + Path.mkdir(self.output_folder / file_folder, parents=True) + with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd: + fd.write(file_content) + + def list_file(self) -> List[str]: + return [str(f) for f in self.output_folder.glob("**/*") if f.is_file()] + + +class Plugin(ReaderAndWriter, ABC): + """A base class for autorest plugin. + + :param autorestapi: An autorest API instance + """ + + @abstractmethod + def process(self) -> bool: + """The plugin process. + + :rtype: bool + :returns: True if everything's ok, False optherwise + :raises Exception: Could raise any exception, stacktrace will be sent to autorest API + """ + raise NotImplementedError() + + +class YamlUpdatePlugin(Plugin): + """A plugin that update the YAML as input.""" + + def get_yaml(self) -> Dict[str, Any]: + # cadl file doesn't have to be relative to output folder + with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd: + return yaml.safe_load(fd.read()) + + def write_yaml(self, yaml_string: str) -> None: + with open(self.options["cadl_file"], "w", encoding="utf-8-sig") as fd: + fd.write(yaml_string) + + def process(self) -> bool: + # List the input file, should be only one + yaml_data = self.get_yaml() + + self.update_yaml(yaml_data) + + yaml_string = yaml.safe_dump(yaml_data) + + self.write_yaml(yaml_string) + return True + + @abstractmethod + def update_yaml(self, yaml_data: Dict[str, Any]) -> None: + """The code-model-v4-no-tags yaml model tree. + + :rtype: updated yaml + :raises Exception: Could raise any exception, stacktrace will be sent to autorest API + """ + raise NotImplementedError() diff --git a/packages/autorest.python/generator/pygen/_version.py b/packages/autorest.python/generator/pygen/_version.py new file mode 100644 index 00000000000..aadc0f31ff8 --- /dev/null +++ b/packages/autorest.python/generator/pygen/_version.py @@ -0,0 +1,7 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +VERSION = "0.1.0" diff --git a/packages/typespec-python/generator/pygen/black/__init__.py b/packages/autorest.python/generator/pygen/black/__init__.py similarity index 100% rename from packages/typespec-python/generator/pygen/black/__init__.py rename to packages/autorest.python/generator/pygen/black/__init__.py diff --git a/packages/autorest.python/generator/pygen/codegen/__init__.py b/packages/autorest.python/generator/pygen/codegen/__init__.py new file mode 100644 index 00000000000..ea4716f86e6 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/__init__.py @@ -0,0 +1,334 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from typing import Dict, Any, Union, Optional +from pathlib import Path +import yaml + + +from .. import Plugin +from ..utils import parse_args +from .models.code_model import CodeModel +from .serializers import JinjaSerializer +from ._utils import DEFAULT_HEADER_TEXT, VALID_PACKAGE_MODE, TYPESPEC_PACKAGE_MODE + + +def _default_pprint(package_name: str) -> str: + return " ".join([i.capitalize() for i in package_name.split("-")]) + + +_LOGGER = logging.getLogger(__name__) + + +class OptionsRetriever: + OPTIONS_TO_DEFAULT = { + "azure-arm": False, + "flavor": "azure", # need to default to azure in shared code so we don't break swagger generation + "no-async": False, + "low-level-client": False, + "version-tolerant": True, + "keep-version-file": False, + "no-namespace-folders": False, + "basic-setup-py": False, + "client-side-validation": False, + "multiapi": False, + "polymorphic-examples": 5, + "generate-sample": False, + "generate-test": False, + "from-typespec": False, + "emit-cross-language-definition-file": False, + } + + @property + def is_azure_flavor(self) -> bool: + return self.flavor == "azure" + + def __init__(self, options: Dict[str, Any]) -> None: + self.options = options + + def __getattr__(self, prop: str) -> Any: + key = prop.replace("_", "-") + return self.options.get(key, self.OPTIONS_TO_DEFAULT.get(key)) + + @property + def company_name(self) -> str: + return self.options.get("company-name", "Microsoft" if self.is_azure_flavor else "") + + @property + def license_header(self) -> str: + license_header = self.options.get( + "header-text", + (DEFAULT_HEADER_TEXT.format(company_name=self.company_name) if self.company_name else ""), + ) + if license_header: + license_header = license_header.replace("\n", "\n# ") + license_header = ( + "# --------------------------------------------------------------------------\n# " + license_header + ) + license_header += "\n# --------------------------------------------------------------------------" + return license_header + + @property + def show_operations(self) -> bool: + return self.options.get("show-operations", not self.low_level_client) + + @property + def _models_mode_default(self) -> str: + models_mode_default = "none" if self.low_level_client or self.version_tolerant else "msrest" + if self.options.get("cadl_file") is not None: + models_mode_default = "dpg" + return models_mode_default + + @property + def original_models_mode(self) -> str: + return self.options.get("models-mode", self._models_mode_default) + + @property + def models_mode(self) -> Union[str, bool]: + # switch to falsy value for easier code writing + return False if self.original_models_mode == "none" else self.original_models_mode + + @property + def tracing(self) -> bool: + return self.options.get( + "tracing", + self.show_operations and self.is_azure_flavor, + ) + + @property + def show_send_request(self) -> bool: + return self.options.get( + "show-send-request", + self._low_level_or_version_tolerant, + ) + + @property + def _low_level_or_version_tolerant(self) -> bool: + return self.low_level_client or self.version_tolerant + + @property + def only_path_and_body_params_positional(self) -> bool: + return self.options.get( + "only-path-and-body-params-positional", + self._low_level_or_version_tolerant, + ) + + @property + def combine_operation_files(self) -> bool: + return self.options.get( + "combine-operation-files", + self.version_tolerant, + ) + + @property + def package_pprint_name(self) -> str: + return self.options.get("package-pprint-name") or _default_pprint(str(self.package_name)) + + @property + def default_optional_constants_to_none(self) -> bool: + return self.options.get( + "default-optional-constants-to-none", + self._low_level_or_version_tolerant, + ) + + @property + def builders_visibility(self) -> str: + builders_visibility = self.options.get("builders-visibility") + if builders_visibility is None: + return "public" if self.low_level_client else "embedded" + return builders_visibility.lower() + + @property + def head_as_boolean(self) -> bool: + head_as_boolean = self.options.get("head-as-boolean", True) + # Force some options in ARM MODE + return True if self.azure_arm else head_as_boolean + + @property + def package_mode(self) -> str: + return self.options.get("packaging-files-dir") or self.options.get("package-mode", "") + + @property + def packaging_files_config(self) -> Optional[Dict[str, Any]]: + packaging_files_config = self.options.get("packaging-files-config") + if packaging_files_config is None: + return None + # packaging-files-config is either a string or a dict + # if it's a string, we can split on the comma to get the dict + # otherwise we just return + try: + return {k.strip(): v.strip() for k, v in [i.split(":") for i in packaging_files_config.split("|")]} + except AttributeError: + return packaging_files_config + + +class CodeGenerator(Plugin): + def __init__(self, *args, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.options_retriever = OptionsRetriever(self.options) + + def _validate_code_model_options(self) -> None: + if self.options_retriever.builders_visibility not in [ + "public", + "hidden", + "embedded", + ]: + raise ValueError("The value of --builders-visibility must be either 'public', 'hidden', or 'embedded'") + + if self.options_retriever.original_models_mode not in ["msrest", "dpg", "none"]: + raise ValueError( + "--models-mode can only be 'msrest', 'dpg' or 'none'. " + "Pass in 'msrest' if you want msrest models, or " + "'none' if you don't want any." + ) + + if not self.options_retriever.show_operations and self.options_retriever.builders_visibility == "embedded": + raise ValueError( + "Can not embed builders without operations. " + "Either set --show-operations to True, or change the value of --builders-visibility " + "to 'public' or 'hidden'." + ) + + if self.options_retriever.basic_setup_py and not self.options_retriever.package_version: + raise ValueError("--basic-setup-py must be used with --package-version") + + if self.options_retriever.package_mode and not self.options_retriever.package_version: + raise ValueError("--package-mode must be used with --package-version") + + if not self.options_retriever.show_operations and self.options_retriever.combine_operation_files: + raise ValueError( + "Can not combine operation files if you are not showing operations. " + "If you want operation files, pass in flag --show-operations" + ) + + if self.options_retriever.package_mode: + if ( + ( + self.options_retriever.package_mode not in TYPESPEC_PACKAGE_MODE + and self.options_retriever.from_typespec + ) + or ( + self.options_retriever.package_mode not in VALID_PACKAGE_MODE + and not self.options_retriever.from_typespec + ) + ) and not Path(self.options_retriever.package_mode).exists(): + raise ValueError( + f"--package-mode can only be {' or '.join(TYPESPEC_PACKAGE_MODE)} or directory which contains template files" # pylint: disable=line-too-long + ) + + if self.options_retriever.multiapi and self.options_retriever.version_tolerant: + raise ValueError( + "Can not currently generate version tolerant multiapi SDKs. " + "We are working on creating a new multiapi SDK for version tolerant and it is not available yet." + ) + + if self.options_retriever.client_side_validation and self.options_retriever.version_tolerant: + raise ValueError("Can not generate version tolerant with --client-side-validation. ") + + if not (self.options_retriever.azure_arm or self.options_retriever.version_tolerant): + _LOGGER.warning( + "You are generating with options that would not allow the SDK to be shipped as an official Azure SDK. " + "Please read https://aka.ms/azsdk/dpcodegen for more details." + ) + + if not self.options_retriever.is_azure_flavor and self.options_retriever.tracing: + raise ValueError("Can only have tracing turned on for Azure SDKs.") + + @staticmethod + def remove_cloud_errors(yaml_data: Dict[str, Any]) -> None: + for client in yaml_data["clients"]: + for group in client["operationGroups"]: + for operation in group["operations"]: + if not operation.get("exceptions"): + continue + i = 0 + while i < len(operation["exceptions"]): + exception = operation["exceptions"][i] + if ( + exception.get("schema") + and exception["schema"]["language"]["default"]["name"] == "CloudError" + ): + del operation["exceptions"][i] + i -= 1 + i += 1 + if yaml_data.get("schemas") and yaml_data["schemas"].get("objects"): + for i in range(len(yaml_data["schemas"]["objects"])): + obj_schema = yaml_data["schemas"]["objects"][i] + if obj_schema["language"]["default"]["name"] == "CloudError": + del yaml_data["schemas"]["objects"][i] + break + + def _build_code_model_options(self) -> Dict[str, Any]: + flags = [ + "azure_arm", + "head_as_boolean", + "license_header", + "keep_version_file", + "no_async", + "no_namespace_folders", + "basic_setup_py", + "package_name", + "package_version", + "client_side_validation", + "tracing", + "multiapi", + "polymorphic_examples", + "models_mode", + "builders_visibility", + "show_operations", + "show_send_request", + "only_path_and_body_params_positional", + "version_tolerant", + "low_level_client", + "combine_operation_files", + "package_mode", + "package_pprint_name", + "packaging_files_config", + "default_optional_constants_to_none", + "generate_sample", + "generate_test", + "default_api_version", + "from_typespec", + "flavor", + "company_name", + "emit_cross_language_definition_file", + ] + return {f: getattr(self.options_retriever, f) for f in flags} + + def get_yaml(self) -> Dict[str, Any]: + # cadl file doesn't have to be relative to output folder + with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd: + return yaml.safe_load(fd.read()) + + def get_serializer(self, code_model: CodeModel): + return JinjaSerializer(code_model, output_folder=self.output_folder) + + def process(self) -> bool: + # List the input file, should be only one + self._validate_code_model_options() + options = self._build_code_model_options() + yaml_data = self.get_yaml() + + if self.options_retriever.azure_arm: + self.remove_cloud_errors(yaml_data) + + code_model = CodeModel(yaml_data=yaml_data, options=options) + if not self.options_retriever.is_azure_flavor and any(client.lro_operations for client in code_model.clients): + raise ValueError("Only support LROs for Azure SDKs") + serializer = self.get_serializer(code_model) + serializer.serialize() + + return True + + +if __name__ == "__main__": + # CADL pipeline will call this + parsed_args, unknown_args = parse_args() + CodeGenerator( + output_folder=parsed_args.output_folder, + cadl_file=parsed_args.cadl_file, + **unknown_args, + ).process() diff --git a/packages/autorest.python/generator/pygen/codegen/_utils.py b/packages/autorest.python/generator/pygen/codegen/_utils.py new file mode 100644 index 00000000000..38d881e8197 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/_utils.py @@ -0,0 +1,16 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +DEFAULT_HEADER_TEXT = ( + "Copyright (c) {company_name} Corporation. All rights reserved.\n" + "Licensed under the MIT License. See License.txt in the project root for license information.\n" + "Code generated by {company_name} (R) Python Code Generator.\n" + "Changes may cause incorrect behavior and will be lost if the code is regenerated." +) + +SWAGGER_PACKAGE_MODE = ["mgmtplane", "dataplane"] # for backward compatibility +TYPESPEC_PACKAGE_MODE = ["azure-mgmt", "azure-dataplane", "generic"] +VALID_PACKAGE_MODE = SWAGGER_PACKAGE_MODE + TYPESPEC_PACKAGE_MODE diff --git a/packages/autorest.python/generator/pygen/codegen/models/__init__.py b/packages/autorest.python/generator/pygen/codegen/models/__init__.py new file mode 100644 index 00000000000..7f9a8e0507c --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/__init__.py @@ -0,0 +1,202 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from typing import Any, Dict, Union, Optional +from .base import BaseModel +from .base_builder import BaseBuilder, ParameterListType +from .code_model import CodeModel +from .client import Client +from .model_type import ModelType, JSONModelType, DPGModelType, MsrestModelType +from .dictionary_type import DictionaryType +from .list_type import ListType +from .combined_type import CombinedType +from .primitive_types import ( + ByteArraySchema, + DateType, + DatetimeType, + DurationType, + IntegerType, + FloatType, + StringType, + TimeType, + AnyType, + PrimitiveType, + BinaryType, + BooleanType, + AnyObjectType, + UnixTimeType, + SdkCoreType, + DecimalType, +) +from .enum_type import EnumType, EnumValue +from .base import BaseType +from .constant_type import ConstantType +from .imports import FileImport, ImportType, TypingSection +from .lro_operation import LROOperation +from .paging_operation import PagingOperation +from .parameter import ( + Parameter, + ParameterMethodLocation, + ParameterLocation, + BodyParameter, + ParameterDelimeter, + ClientParameter, + ConfigParameter, +) +from .operation import Operation +from .property import Property +from .operation_group import OperationGroup +from .response import Response +from .parameter_list import ( + ParameterList, + ClientGlobalParameterList, + ConfigGlobalParameterList, +) +from .request_builder import ( + RequestBuilder, + OverloadedRequestBuilder, + RequestBuilderBase, +) +from .lro_paging_operation import LROPagingOperation +from .request_builder_parameter import ( + RequestBuilderParameter, + RequestBuilderBodyParameter, +) +from .credential_types import ( + TokenCredentialType, + KeyCredentialType, + ARMChallengeAuthenticationPolicyType, + BearerTokenCredentialPolicyType, + KeyCredentialPolicyType, + CredentialType, +) + +__all__ = [ + "KeyCredentialPolicyType", + "AnyType", + "BaseModel", + "BaseType", + "CodeModel", + "Client", + "ConstantType", + "ModelType", + "DictionaryType", + "ListType", + "EnumType", + "EnumValue", + "FileImport", + "ImportType", + "TypingSection", + "PrimitiveType", + "LROOperation", + "Operation", + "PagingOperation", + "Parameter", + "ParameterList", + "OperationGroup", + "Property", + "RequestBuilder", + "Response", + "TokenCredentialType", + "LROPagingOperation", + "BaseBuilder", + "RequestBuilderParameter", + "BinaryType", + "ClientGlobalParameterList", + "ConfigGlobalParameterList", + "ParameterMethodLocation", + "ParameterLocation", + "OverloadedRequestBuilder", + "RequestBuilderBase", + "BodyParameter", + "RequestBuilderBodyParameter", + "ParameterDelimeter", + "CredentialType", + "ClientParameter", + "ConfigParameter", + "ParameterListType", +] + +TYPE_TO_OBJECT = { + "integer": IntegerType, + "float": FloatType, + "decimal": DecimalType, + "string": StringType, + "list": ListType, + "dict": DictionaryType, + "constant": ConstantType, + "enum": EnumType, + "enumvalue": EnumValue, + "binary": BinaryType, + "any": AnyType, + "utcDateTime": DatetimeType, + "offsetDateTime": DatetimeType, + "plainTime": TimeType, + "duration": DurationType, + "plainDate": DateType, + "bytes": ByteArraySchema, + "boolean": BooleanType, + "combined": CombinedType, + "OAuth2": TokenCredentialType, + "Key": KeyCredentialType, + "ARMChallengeAuthenticationPolicy": ARMChallengeAuthenticationPolicyType, + "BearerTokenCredentialPolicy": BearerTokenCredentialPolicyType, + "KeyCredentialPolicy": KeyCredentialPolicyType, + "any-object": AnyObjectType, + "unixtime": UnixTimeType, + "credential": StringType, + "sdkcore": SdkCoreType, +} +_LOGGER = logging.getLogger(__name__) + + +def build_type(yaml_data: Dict[str, Any], code_model: CodeModel) -> BaseType: + yaml_id = id(yaml_data) + try: + return code_model.lookup_type(yaml_id) + except KeyError: + # Not created yet, let's create it and add it to the index + pass + response: Optional[BaseType] = None + if yaml_data["type"] == "model": + # need to special case model to avoid recursion + if yaml_data["base"] == "json" or not code_model.options["models_mode"]: + model_type = JSONModelType + elif yaml_data["base"] == "dpg": + model_type = DPGModelType # type: ignore + else: + model_type = MsrestModelType # type: ignore + response = model_type(yaml_data, code_model) + code_model.types_map[yaml_id] = response + response.fill_instance_from_yaml(yaml_data, code_model) + elif yaml_data["type"] == "enum": + # avoid recursion because we add the parent enum type to the enum value + response = EnumType( + yaml_data, + code_model, + values=[], + value_type=build_type(yaml_data["valueType"], code_model), + ) + code_model.types_map[yaml_id] = response + response.fill_instance_from_yaml(yaml_data, code_model) + else: + object_type = yaml_data.get("type") + if object_type not in TYPE_TO_OBJECT: + _LOGGER.warning( + 'Unrecognized definition type "%s" is found, falling back it as "string"! ', + yaml_data["type"], + ) + object_type = "string" + response = TYPE_TO_OBJECT[object_type].from_yaml(yaml_data, code_model) # type: ignore + if response is None: + raise ValueError("response can not be None") + code_model.types_map[yaml_id] = response + return response + + +RequestBuilderType = Union[RequestBuilder, OverloadedRequestBuilder] +ParameterType = Union[Parameter, RequestBuilderParameter, ClientParameter, ConfigParameter] +OperationType = Union[Operation, LROOperation, PagingOperation, LROPagingOperation] diff --git a/packages/autorest.python/generator/pygen/codegen/models/base.py b/packages/autorest.python/generator/pygen/codegen/models/base.py new file mode 100644 index 00000000000..7a44fb0b283 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/base.py @@ -0,0 +1,188 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict, TYPE_CHECKING, List, Optional +from abc import ABC, abstractmethod +from .imports import FileImport + + +if TYPE_CHECKING: + from .code_model import CodeModel + from .model_type import ModelType + + +class BaseModel: + """This is the base class for model representations that are based on some YAML data. + + :param yaml_data: the yaml data for this schema + :type yaml_data: dict[str, Any] + """ + + def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + self.yaml_data = yaml_data + self.code_model = code_model + + @property + def id(self) -> int: + return id(self.yaml_data) + + def __repr__(self): + return f"<{self.__class__.__name__}>" + + +class BaseType(BaseModel, ABC): # pylint: disable=too-many-public-methods + """This is the base class for all types. + + :param yaml_data: the yaml data for this schema + :type yaml_data: dict[str, Any] + """ + + def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + super().__init__(yaml_data, code_model) + self.type = yaml_data["type"] # the type discriminator + self.api_versions: List[str] = yaml_data.get("apiVersions", []) # api versions this type is in. + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "BaseType": + return cls(yaml_data=yaml_data, code_model=code_model) + + def imports(self, **kwargs) -> FileImport: # pylint: disable=unused-argument + return FileImport(self.code_model) + + def imports_for_multiapi(self, **kwargs: Any) -> FileImport: + return self.imports(**kwargs) + + def imports_for_sample(self) -> FileImport: + return self.imports() + + @staticmethod + def serialize_sample_value(value: Any) -> str: + return repr(value) + + @property + def xml_metadata(self) -> Dict[str, Any]: + """XML metadata for the type, if the type has it.""" + return self.yaml_data.get("xmlMetadata", {}) + + @property + def is_xml(self) -> bool: + """Whether the type is an XML type or not. Most likely not.""" + return bool(self.xml_metadata) + + @property + def xml_serialization_ctxt(self) -> Optional[str]: + """Return the serialization context in case this schema is used in an operation.""" + attrs_list = [] + if self.xml_metadata.get("name"): + attrs_list.append(f"'name': '{self.xml_metadata['name']}'") + if self.xml_metadata.get("attribute", False): + attrs_list.append("'attr': True") + if self.xml_metadata.get("prefix", False): + attrs_list.append(f"'prefix': '{self.xml_metadata['prefix']}'") + if self.xml_metadata.get("namespace", False): + attrs_list.append(f"'ns': '{self.xml_metadata['namespace']}'") + if self.xml_metadata.get("text"): + attrs_list.append("'text': True") + return ", ".join(attrs_list) + + @property + def serialization_type(self) -> str: + """The tag recognized by 'msrest' as a serialization/deserialization. + + 'str', 'int', 'float', 'bool' or + https://github.com/Azure/msrest-for-python/blob/b505e3627b547bd8fdc38327e86c70bdb16df061/msrest/serialization.py#L407-L416 + + or the object schema name (e.g. DotSalmon). + + If list: '[str]' + If dict: '{str}' + """ + raise NotImplementedError() + + @property + def msrest_deserialization_key(self) -> str: + return self.serialization_type + + @property + def client_default_value(self) -> Any: + """Whether there's a client default value for this type""" + return self.yaml_data.get("clientDefaultValue") + + @abstractmethod + def description(self, *, is_operation_file: bool) -> str: + """The description""" + + @abstractmethod + def docstring_text(self, **kwargs: Any) -> str: + """The names used in rtype documentation""" + + @abstractmethod + def docstring_type(self, **kwargs: Any) -> str: + """The python type used for RST syntax input. + + Special case for enum, for instance: 'str or ~namespace.EnumName' + """ + + @abstractmethod + def type_annotation(self, **kwargs: Any) -> str: + """The python type used for type annotation + + Special case for enum, for instance: Union[str, "EnumName"] + """ + + @property + def validation(self) -> Optional[Dict[str, Any]]: + """Whether there's any validation constraints on this type. + + Even though we generate validation maps if there are validation constraints, + only SDKs with client-side-validate=true (0.001% libraries, if any) actually raise in this case. + """ + return None + + def get_declaration(self, value: Any) -> str: + """Return the current value from YAML as a Python string that represents the constant. + + Example, if schema is "bytearray" and value is "foo", + should return bytearray("foo", encoding="utf-8") + as a string. + + This is important for constant serialization. + + By default, return value, since it works sometimes (integer) + """ + return str(value) + + @abstractmethod + def get_json_template_representation( + self, + *, + optional: bool = True, + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + """Template of what this schema would look like as JSON input""" + + def get_polymorphic_subtypes( + self, polymorphic_subtypes: List["ModelType"] # pylint: disable=unused-argument + ) -> None: + return None + + @property + @abstractmethod + def instance_check_template(self) -> str: + """Template of what an instance check of a variable for this type would look like""" + + @property + def serialization_constraints(self) -> List[str]: + """Whether there are any serialization constraints when serializing this type.""" + return [] + + @property + def type_description(self) -> str: + return self.type_annotation() + + @property + def is_form_data(self) -> bool: + return False diff --git a/packages/autorest.python/generator/pygen/codegen/models/base_builder.py b/packages/autorest.python/generator/pygen/codegen/models/base_builder.py new file mode 100644 index 00000000000..7bd5b2f4279 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/base_builder.py @@ -0,0 +1,119 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from typing import ( + List, + Dict, + Any, + Generic, + TypeVar, + Optional, + Union, + TYPE_CHECKING, + cast, +) +from abc import abstractmethod + +from .base import BaseModel +from .parameter_list import ( + ParameterList, + RequestBuilderParameterList, + OverloadedRequestBuilderParameterList, +) + +ParameterListType = TypeVar( + "ParameterListType", + bound=Union[ + ParameterList, + RequestBuilderParameterList, + OverloadedRequestBuilderParameterList, + ], +) +if TYPE_CHECKING: + from .code_model import CodeModel + from .client import Client + from .operation import Operation + from .request_builder import RequestBuilder + + +OverloadListType = TypeVar("OverloadListType", bound=Union[List["Operation"], List["RequestBuilder"]]) + +_LOGGER = logging.getLogger(__name__) + + +class BaseBuilder( + Generic[ParameterListType, OverloadListType], BaseModel +): # pylint: disable=too-many-instance-attributes + """Base class for Operations and Request Builders""" + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + client: "Client", + name: str, + parameters: ParameterListType, + *, + overloads: Optional[OverloadListType] = None, + ) -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.client = client + self.name = name + self._description: str = yaml_data.get("description", "") + self.parameters = parameters + self.overloads = overloads or cast(OverloadListType, []) + self._summary: str = yaml_data.get("summary", "") + self.want_tracing: bool = yaml_data.get("wantTracing", True) + self.group_name: str = yaml_data["groupName"] # either operationGroup or client I am on + self.is_overload: bool = yaml_data["isOverload"] + self.api_versions: List[str] = yaml_data["apiVersions"] + self.added_on: Optional[str] = yaml_data.get("addedOn") + self.external_docs: Optional[Dict[str, Any]] = yaml_data.get("externalDocs") + + if code_model.options["version_tolerant"] and yaml_data.get("abstract"): + _LOGGER.warning( + 'Not going to generate operation "%s" because we are unable to generate this ' + "type of operation right now. " + 'Please write your own custom operation in the "_patch.py" file ' + "following https://aka.ms/azsdk/python/dpcodegen/python/customize", + name, + ) + self.abstract = True + else: + self.abstract = False + + @property + def summary(self) -> Optional[str]: + if self.abstract: + return None + return self._summary + + @property + def pylint_disable(self) -> str: + return "" + + @abstractmethod + def response_type_annotation(self, **kwargs) -> str: ... + + @abstractmethod + def response_docstring_text(self, **kwargs) -> str: ... + + @abstractmethod + def response_docstring_type(self, **kwargs) -> str: ... + + @property + def description(self) -> str: + if self.abstract: + return ( + f'You need to write a custom operation for "{self.name}". Please refer to ' + "https://aka.ms/azsdk/python/dpcodegen/python/customize to learn how to customize." + ) + return self._description or self.name + + def method_signature(self, async_mode: bool) -> List[str]: + if self.abstract: + return ["*args,", "**kwargs"] + return self.parameters.method_signature(async_mode) diff --git a/packages/autorest.python/generator/pygen/codegen/models/client.py b/packages/autorest.python/generator/pygen/codegen/models/client.py new file mode 100644 index 00000000000..953ad7d63d8 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/client.py @@ -0,0 +1,422 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict, TYPE_CHECKING, TypeVar, Generic, Union, List, Optional + +from .base import BaseModel +from .parameter_list import ClientGlobalParameterList, ConfigGlobalParameterList +from .imports import FileImport, ImportType, TypingSection, MsrestImportType +from .utils import add_to_pylint_disable, NAME_LENGTH_LIMIT +from .operation_group import OperationGroup +from .request_builder import ( + RequestBuilder, + OverloadedRequestBuilder, + get_request_builder, +) +from .parameter import Parameter, ParameterMethodLocation +from .lro_operation import LROOperation +from .lro_paging_operation import LROPagingOperation + +ParameterListType = TypeVar( + "ParameterListType", + bound=Union[ClientGlobalParameterList, ConfigGlobalParameterList], +) + +if TYPE_CHECKING: + from .code_model import CodeModel + from . import OperationType + + +class _ClientConfigBase(Generic[ParameterListType], BaseModel): + """The service client base. Shared across our Client and Config type""" + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + parameters: ParameterListType, + ): + super().__init__(yaml_data, code_model) + self.parameters = parameters + self.url: str = self.yaml_data["url"] # the base endpoint of the client. Can be parameterized or not + self.legacy_filename: str = self.yaml_data.get("legacyFilename", "client") + + @property + def description(self) -> str: + return self.yaml_data["description"] + + @property + def name(self) -> str: + return self.yaml_data["name"] + + +class Client(_ClientConfigBase[ClientGlobalParameterList]): + """Model representing our service client""" + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + parameters: ClientGlobalParameterList, + *, + is_subclient: bool = False, + ): + super().__init__(yaml_data, code_model, parameters) + self.operation_groups: List[OperationGroup] = [] + self.config = Config.from_yaml(yaml_data, self.code_model) + self.is_subclient = is_subclient + self.request_builders = self._build_request_builders() + if self.code_model.options["show_operations"]: + self.operation_groups = [ + OperationGroup.from_yaml(op_group, code_model, self) + for op_group in self.yaml_data.get("operationGroups", []) + ] + self.link_lro_initial_operations() + self.request_id_header_name = self.yaml_data.get("requestIdHeaderName", None) + self.has_etag: bool = yaml_data.get("hasEtag", False) + + def _build_request_builders( + self, + ) -> List[Union[RequestBuilder, OverloadedRequestBuilder]]: + request_builders: List[Union[RequestBuilder, OverloadedRequestBuilder]] = [] + + def add_og_request_builder(og: Dict[str, Any]): + for operation_yaml in og["operations"]: + request_builder = get_request_builder( + operation_yaml, + code_model=self.code_model, + client=self, + ) + if operation_yaml.get("isLroInitialOperation"): + # we want to change the name + request_builder.name = request_builder.get_name( + request_builder.yaml_data["name"][1 : -len("_initial")], + request_builder.yaml_data, + request_builder.code_model, + request_builder.client, + ) + if request_builder.overloads: + request_builders.extend(request_builder.overloads) + request_builders.append(request_builder) + if operation_yaml.get("nextOperation"): + # i am a paging operation and i have a next operation. + # Make sure to include my next operation + request_builders.append( + get_request_builder( + operation_yaml["nextOperation"], + code_model=self.code_model, + client=self, + ) + ) + + queue = self.yaml_data["operationGroups"].copy() + while queue: + now = queue.pop(0) + add_og_request_builder(now) + if now.get("operationGroups"): + queue.extend(now["operationGroups"]) + + return request_builders + + def pipeline_class(self, async_mode: bool) -> str: + if self.code_model.options["azure_arm"]: + if async_mode: + return "AsyncARMPipelineClient" + return "ARMPipelineClient" + if async_mode: + return "AsyncPipelineClient" + return "PipelineClient" + + @property + def credential(self) -> Optional[Parameter]: + """The credential param, if one exists""" + return self.parameters.credential + + @property + def send_request_name(self) -> str: + """Name of the send request function""" + return "send_request" if self.code_model.options["show_send_request"] else "_send_request" + + @property + def has_parameterized_host(self) -> bool: + """Whether the base url is parameterized or not""" + return not any(p for p in self.parameters if p.is_host) + + @property + def pylint_disable(self) -> str: + retval = add_to_pylint_disable("", "client-accepts-api-version-keyword") + if len(self.operation_groups) > 6: + retval = add_to_pylint_disable(retval, "too-many-instance-attributes") + if len(self.name) > NAME_LENGTH_LIMIT: + retval = add_to_pylint_disable(retval, "name-too-long") + return retval + + @property + def url_pylint_disable(self) -> str: + # if the url is too long + retval = "" + if len(self.url) > 85: + retval = add_to_pylint_disable(retval, "line-too-long") + return retval + + @property + def filename(self) -> str: + """Name of the file for the client""" + if self.code_model.options["version_tolerant"] or self.code_model.options["low_level_client"]: + return "_client" + return f"_{self.legacy_filename}" + + def lookup_request_builder(self, request_builder_id: int) -> Union[RequestBuilder, OverloadedRequestBuilder]: + """Find the request builder based off of id""" + try: + return next(rb for rb in self.request_builders if id(rb.yaml_data) == request_builder_id) + except StopIteration as exc: + raise KeyError(f"No request builder with id {request_builder_id} found.") from exc + + def lookup_operation(self, operation_id: int) -> "OperationType": + try: + return next(o for og in self.operation_groups for o in og.operations if id(o.yaml_data) == operation_id) + except StopIteration as exc: + raise KeyError(f"No operation with id {operation_id} found.") from exc + + def _imports_shared(self, async_mode: bool) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) + if self.code_model.options["azure_arm"]: + file_import.add_submodule_import("azure.mgmt.core", self.pipeline_class(async_mode), ImportType.SDKCORE) + else: + file_import.add_submodule_import( + "" if self.code_model.is_azure_flavor else "runtime", + self.pipeline_class(async_mode), + ImportType.SDKCORE, + ) + + for gp in self.parameters: + if gp.method_location == ParameterMethodLocation.KWARG: + continue + file_import.merge( + gp.imports( + async_mode, + relative_path=".." if async_mode else ".", + operation=True, + ) + ) + file_import.add_submodule_import( + "._configuration", + f"{self.name}Configuration", + ImportType.LOCAL, + ) + file_import.add_msrest_import( + relative_path=".." if async_mode else ".", + msrest_import_type=MsrestImportType.SerializerDeserializer, + typing_section=TypingSection.REGULAR, + ) + file_import.add_submodule_import( + "pipeline" if self.code_model.is_azure_flavor else "runtime", + "policies", + ImportType.SDKCORE, + ) + if self.code_model.options["azure_arm"]: + async_prefix = "Async" if async_mode else "" + file_import.add_submodule_import( + "azure.mgmt.core.policies", + f"{async_prefix}ARMAutoResourceProviderRegistrationPolicy", + ImportType.SDKCORE, + ) + return file_import + + @property + def has_mixin(self) -> bool: + """Do we want a mixin ABC class for typing purposes?""" + return any(og for og in self.operation_groups if og.is_mixin) + + @property + def lro_operations(self) -> List["OperationType"]: + """all LRO operations in this SDK?""" + return [operation for operation_group in self.operation_groups for operation in operation_group.lro_operations] + + @property + def has_public_lro_operations(self) -> bool: + """Are there any public LRO operations in this SDK?""" + return any(not operation.internal for operation in self.lro_operations) + + @property + def has_operations(self) -> bool: + return any(operation_group.has_operations for operation_group in self.operation_groups) + + def link_lro_initial_operations(self) -> None: + """Link each LRO operation to its initial operation""" + for operation_group in self.operation_groups: + for operation in operation_group.operations: + if isinstance(operation, (LROOperation, LROPagingOperation)): + operation.initial_operation = self.lookup_operation(id(operation.yaml_data["initialOperation"])) + + @property + def has_abstract_operations(self) -> bool: + """Whether there is abstract operation in any operation group.""" + return any(og.has_abstract_operations for og in self.operation_groups) + + @property + def has_non_abstract_operations(self) -> bool: + """Whether there is non-abstract operation in any operation group.""" + return any(og.has_non_abstract_operations for og in self.operation_groups) + + def imports(self, async_mode: bool) -> FileImport: + file_import = self._imports_shared(async_mode) + if async_mode: + file_import.add_submodule_import("typing", "Awaitable", ImportType.STDLIB) + file_import.add_submodule_import( + "rest", + "AsyncHttpResponse", + ImportType.SDKCORE, + TypingSection.CONDITIONAL, + ) + else: + file_import.add_submodule_import( + "rest", + "HttpResponse", + ImportType.SDKCORE, + TypingSection.CONDITIONAL, + ) + file_import.add_submodule_import( + "rest", + "HttpRequest", + ImportType.SDKCORE, + TypingSection.CONDITIONAL, + ) + for og in self.operation_groups: + file_import.add_submodule_import( + f".{self.code_model.operations_folder_name}", + og.class_name, + ImportType.LOCAL, + ) + + if self.code_model.model_types and self.code_model.options["models_mode"] == "msrest": + path_to_models = ".." if async_mode else "." + file_import.add_submodule_import(path_to_models, "models", ImportType.LOCAL, alias="_models") + elif self.code_model.options["models_mode"] == "msrest": + # in this case, we have client_models = {} in the service client, which needs a type annotation + # this import will always be commented, so will always add it to the typing section + file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) + file_import.add_submodule_import("copy", "deepcopy", ImportType.STDLIB) + return file_import + + def imports_for_multiapi(self, async_mode: bool) -> FileImport: + file_import = self._imports_shared(async_mode) + file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB, TypingSection.CONDITIONAL) + try: + mixin_operation = next(og for og in self.operation_groups if og.is_mixin) + file_import.add_submodule_import("._operations_mixin", mixin_operation.class_name, ImportType.LOCAL) + except StopIteration: + pass + file_import.add_submodule_import("azure.profiles", "KnownProfiles", import_type=ImportType.SDKCORE) + file_import.add_submodule_import("azure.profiles", "ProfileDefinition", import_type=ImportType.SDKCORE) + file_import.add_submodule_import( + "azure.profiles.multiapiclient", + "MultiApiClientMixin", + import_type=ImportType.SDKCORE, + ) + return file_import + + @classmethod + def from_yaml( + cls, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + *, + is_subclient: bool = False, + ) -> "Client": + return cls( + yaml_data=yaml_data, + code_model=code_model, + parameters=ClientGlobalParameterList.from_yaml(yaml_data, code_model), + is_subclient=is_subclient, + ) + + +class Config(_ClientConfigBase[ConfigGlobalParameterList]): + """Model representing our Config type.""" + + @property + def pylint_disable(self) -> str: + retval = add_to_pylint_disable("", "too-many-instance-attributes") + if len(self.name) + len("Configuration") > NAME_LENGTH_LIMIT: + retval = add_to_pylint_disable(retval, "name-too-long") + return retval + + @property + def description(self) -> str: + return ( + f"Configuration for {self.yaml_data['name']}.\n\n." + "Note that all parameters used to create this instance are saved as instance attributes." + ) + + @property + def sdk_moniker(self) -> str: + package_name = self.code_model.options["package_name"] + if package_name and package_name.startswith("azure-"): + package_name = package_name[len("azure-") :] + return package_name if package_name else self.yaml_data["name"].lower() + + @property + def name(self) -> str: + return f"{super().name}Configuration" + + def _imports_shared(self, async_mode: bool) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_submodule_import( + "pipeline" if self.code_model.is_azure_flavor else "runtime", + "policies", + ImportType.SDKCORE, + ) + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) + if self.code_model.options["package_version"]: + file_import.add_submodule_import(".._version" if async_mode else "._version", "VERSION", ImportType.LOCAL) + if self.code_model.options["azure_arm"]: + policy = "AsyncARMChallengeAuthenticationPolicy" if async_mode else "ARMChallengeAuthenticationPolicy" + file_import.add_submodule_import("azure.mgmt.core.policies", "ARMHttpLoggingPolicy", ImportType.SDKCORE) + file_import.add_submodule_import("azure.mgmt.core.policies", policy, ImportType.SDKCORE) + + return file_import + + def imports(self, async_mode: bool) -> FileImport: + file_import = self._imports_shared(async_mode) + for gp in self.parameters: + if gp.method_location == ParameterMethodLocation.KWARG and gp not in self.parameters.kwargs_to_pop: + continue + file_import.merge( + gp.imports( + async_mode=async_mode, + relative_path=".." if async_mode else ".", + operation=True, + ) + ) + return file_import + + def imports_for_multiapi(self, async_mode: bool) -> FileImport: + file_import = self._imports_shared(async_mode) + for gp in self.parameters: + if ( + gp.method_location == ParameterMethodLocation.KWARG + and gp not in self.parameters.kwargs_to_pop + and gp.client_name == "api_version" + ): + continue + file_import.merge( + gp.imports_for_multiapi( + async_mode=async_mode, + relative_path=".." if async_mode else ".", + operation=True, + ) + ) + return file_import + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "Config": + return cls( + yaml_data=yaml_data, + code_model=code_model, + parameters=ConfigGlobalParameterList.from_yaml(yaml_data, code_model), + ) diff --git a/packages/autorest.python/generator/pygen/codegen/models/code_model.py b/packages/autorest.python/generator/pygen/codegen/models/code_model.py new file mode 100644 index 00000000000..c94ec77c2b7 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/code_model.py @@ -0,0 +1,245 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import List, Dict, Any, Set, Union, Literal + +from .base import BaseType +from .enum_type import EnumType +from .model_type import ModelType +from .combined_type import CombinedType +from .client import Client +from .request_builder import RequestBuilder, OverloadedRequestBuilder + + +def _is_legacy(options) -> bool: + return not (options.get("version_tolerant") or options.get("low_level_client")) + + +class CodeModel: # pylint: disable=too-many-public-methods, disable=too-many-instance-attributes + """Top level code model + + :param options: Options of the code model. I.e., whether this is for management generation + :type options: dict[str, bool] + :param str module_name: The module name for the client. Is in snake case. + :param str class_name: The class name for the client. Is in pascal case. + :param str description: The description of the client + :param str namespace: The namespace of our module + :param schemas: The list of schemas we are going to serialize in the models files. Maps their yaml + id to our created ModelType. + :type schemas: dict[int, ~autorest.models.ModelType] + :param sorted_schemas: Our schemas in order by inheritance and alphabet + :type sorted_schemas: list[~autorest.models.ModelType] + :param enums: The enums, if any, we are going to serialize. Maps their yaml id to our created EnumType. + :type enums: Dict[int, ~autorest.models.EnumType] + :param primitives: List of schemas we've created that are not EnumSchemas or ObjectSchemas. Maps their + yaml id to our created schemas. + :type primitives: Dict[int, ~autorest.models.BaseType] + :param package_dependency: All the dependencies needed in setup.py + :type package_dependency: Dict[str, str] + """ + + def __init__( + self, + yaml_data: Dict[str, Any], + options: Dict[str, Any], + *, + is_subnamespace: bool = False, + ) -> None: + self.yaml_data = yaml_data + self.options = options + self.namespace = self.yaml_data["namespace"] + self.types_map: Dict[int, BaseType] = {} # map yaml id to schema + self._model_types: List[ModelType] = [] + from . import build_type + + for type_yaml in yaml_data.get("types", []): + build_type(yaml_data=type_yaml, code_model=self) + self.clients: List[Client] = [ + Client.from_yaml(client_yaml_data, self) for client_yaml_data in yaml_data["clients"] + ] + self.subnamespace_to_clients: Dict[str, List[Client]] = { + subnamespace: [Client.from_yaml(client_yaml, self, is_subclient=True) for client_yaml in client_yamls] + for subnamespace, client_yamls in yaml_data.get("subnamespaceToClients", {}).items() + } + if self.options["models_mode"] and self.model_types: + self.sort_model_types() + self.is_subnamespace = is_subnamespace + self.named_unions: List[CombinedType] = [ + t for t in self.types_map.values() if isinstance(t, CombinedType) and t.name + ] + self.cross_language_package_id = self.yaml_data.get("crossLanguagePackageId") + self.for_test: bool = False + + @property + def has_form_data(self) -> bool: + return any(og.has_form_data_body for client in self.clients for og in client.operation_groups) + + @property + def has_etag(self) -> bool: + return any(client.has_etag for client in self.clients) + + @property + def has_operations(self) -> bool: + if any(c for c in self.clients if c.has_operations): + return True + return any(c for clients in self.subnamespace_to_clients.values() for c in clients if c.has_operations) + + @property + def has_non_abstract_operations(self) -> bool: + return any(c for c in self.clients if c.has_non_abstract_operations) or any( + c for cs in self.subnamespace_to_clients.values() for c in cs if c.has_non_abstract_operations + ) + + def lookup_request_builder(self, request_builder_id: int) -> Union[RequestBuilder, OverloadedRequestBuilder]: + """Find the request builder based off of id""" + for client in self.clients: + try: + return client.lookup_request_builder(request_builder_id) + except KeyError: + pass + raise KeyError(f"No request builder with id {request_builder_id} found.") + + @property + def is_azure_flavor(self) -> bool: + return self.options["flavor"] == "azure" + + @property + def rest_layer_name(self) -> str: + """If we have a separate rest layer, what is its name?""" + return "rest" if self.options["builders_visibility"] == "public" else "_rest" + + @property + def client_filename(self) -> str: + return self.clients[0].filename + + def need_vendored_code(self, async_mode: bool) -> bool: + """Whether we need to vendor code in the _vendor.py file for this SDK""" + if self.has_abstract_operations: + return True + if async_mode: + return self.need_mixin_abc + return self.need_mixin_abc or self.has_etag or self.has_form_data + + @property + def need_mixin_abc(self) -> bool: + return any(c for c in self.clients if c.has_mixin) + + @property + def has_abstract_operations(self) -> bool: + return any(c for c in self.clients if c.has_abstract_operations) + + @property + def operations_folder_name(self) -> str: + """Get the name of the operations folder that holds operations.""" + name = "operations" + if self.options["version_tolerant"] and not any( + og for client in self.clients for og in client.operation_groups if not og.is_mixin + ): + name = f"_{name}" + return name + + @property + def description(self) -> str: + return self.clients[0].description + + def lookup_type(self, schema_id: int) -> BaseType: + """Looks to see if the schema has already been created. + + :param int schema_id: The yaml id of the schema + :return: If created, we return the created schema, otherwise, we throw. + :rtype: ~autorest.models.BaseType + :raises: KeyError if schema is not found + """ + try: + return next(type for id, type in self.types_map.items() if id == schema_id) + except StopIteration as exc: + raise KeyError(f"Couldn't find schema with id {schema_id}") from exc + + @property + def model_types(self) -> List[ModelType]: + """All of the model types in this class""" + if not self._model_types: + self._model_types = [ + t + for t in self.types_map.values() + if isinstance(t, ModelType) and not (self.options["models_mode"] == "dpg" and t.page_result_model) + ] + return self._model_types + + @model_types.setter + def model_types(self, val: List[ModelType]) -> None: + self._model_types = val + + @property + def public_model_types(self) -> List[ModelType]: + return [m for m in self.model_types if not m.internal and not m.base == "json"] + + @property + def enums(self) -> List[EnumType]: + """All of the enums""" + return [t for t in self.types_map.values() if isinstance(t, EnumType)] + + @property + def core_library(self) -> Literal["azure.core", "corehttp"]: + return "azure.core" if self.is_azure_flavor else "corehttp" + + def _sort_model_types_helper( + self, + current: ModelType, + seen_schema_names: Set[str], + seen_schema_yaml_ids: Set[int], + ): + if current.id in seen_schema_yaml_ids: + return [] + if current.name in seen_schema_names: + raise ValueError(f"We have already generated a schema with name {current.name}") + ancestors = [current] + if current.parents: + for parent in current.parents: + if parent.id in seen_schema_yaml_ids: + continue + seen_schema_names.add(current.name) + seen_schema_yaml_ids.add(current.id) + ancestors = self._sort_model_types_helper(parent, seen_schema_names, seen_schema_yaml_ids) + ancestors + seen_schema_names.add(current.name) + seen_schema_yaml_ids.add(current.id) + return ancestors + + def sort_model_types(self) -> None: + """Sorts the final object schemas by inheritance and by alphabetical order. + + :return: None + :rtype: None + """ + seen_schema_names: Set[str] = set() + seen_schema_yaml_ids: Set[int] = set() + sorted_object_schemas: List[ModelType] = [] + for schema in sorted(self.model_types, key=lambda x: x.name.lower()): + sorted_object_schemas.extend(self._sort_model_types_helper(schema, seen_schema_names, seen_schema_yaml_ids)) + self.model_types = sorted_object_schemas + + @property + def models_filename(self) -> str: + """Get the names of the model file(s)""" + if self.is_legacy: + return "_models_py3" + return "_models" + + @property + def enums_filename(self) -> str: + """The name of the enums file""" + if self.is_legacy: + return f"_{self.clients[0].legacy_filename}_enums" + return "_enums" + + @property + def is_legacy(self) -> bool: + return _is_legacy(self.options) + + @property + def need_typing_extensions(self) -> bool: + if self.options["models_mode"] == "dpg": + return True + return False diff --git a/packages/autorest.python/generator/pygen/codegen/models/combined_type.py b/packages/autorest.python/generator/pygen/codegen/models/combined_type.py new file mode 100644 index 00000000000..8b6fc261c26 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/combined_type.py @@ -0,0 +1,153 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict, List, Optional, TYPE_CHECKING, Type, Tuple, Union +import re +from .imports import FileImport, ImportType, TypingSection +from .base import BaseType +from .model_type import ModelType + +if TYPE_CHECKING: + from .code_model import CodeModel + + +class CombinedType(BaseType): + """A type that consists of multiple different types. + + Used by body parameters that have multiple types, i.e. one that can be + a stream body or a JSON body. + """ + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + types: List[BaseType], + ) -> None: + super().__init__(yaml_data, code_model) + self.types = types # the types that this type is combining + self.name = yaml_data.get("name") + self._is_union_of_literals = all(i.type == "constant" for i in self.types) + + @property + def serialization_type(self) -> str: + """The tag recognized by 'msrest' as a serialization/deserialization. + + 'str', 'int', 'float', 'bool' or + https://github.com/Azure/msrest-for-python/blob/b505e3627b547bd8fdc38327e86c70bdb16df061/msrest/serialization.py#L407-L416 + + or the object schema name (e.g. DotSalmon). + + If list: '[str]' + If dict: '{str}' + """ + if not all(t for t in self.types if t.type == "constant"): + raise ValueError("Shouldn't get serialization type of a combinedtype") + return self.types[0].serialization_type + + @property + def client_default_value(self) -> Any: + return self.yaml_data.get("clientDefaultValue") + + def description(self, *, is_operation_file: bool) -> str: # pylint: disable=unused-argument + if len(self.types) == 2: + return f"Is either a {self.types[0].type_description} type or a {self.types[1].type_description} type." + return f"Is one of the following types: {', '.join([t.type_description for t in self.types])}" + + def docstring_text(self, **kwargs: Any) -> str: + return " or ".join(t.docstring_text(**kwargs) for t in self.types) + + def docstring_type(self, **kwargs: Any) -> str: + return " or ".join(t.docstring_type(**kwargs) for t in self.types) + + def type_annotation(self, **kwargs: Any) -> str: + if self.name: + return f'"_types.{self.name}"' + return self.type_definition(**kwargs) + + def type_definition(self, **kwargs: Any) -> str: + """The python type used for type annotation + + Special case for enum, for instance: Union[str, "EnumName"] + """ + # remove duplicates + inside_types = list(dict.fromkeys([type.type_annotation(**kwargs) for type in self.types])) + if len(inside_types) == 1: + return inside_types[0] + if self._is_union_of_literals: + parsed_values = [] + for entry in inside_types: + match = re.search(r"Literal\[(.*)\]", entry) + if match is not None: + parsed_values.append(match.group(1)) + join_string = ", ".join(parsed_values) + return f"Literal[{join_string}]" + + # If the inside types has been a Union, peel first and then re-union + pattern = re.compile(r"Union\[.*\]") + return f'Union[{", ".join(map(lambda x: x[6: -1] if pattern.match(x) else x, inside_types))}]' + + @property + def is_form_data(self) -> bool: + return any(t.is_form_data for t in self.types) + + def get_json_template_representation( + self, + *, + optional: bool = True, + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + return self.types[0].get_json_template_representation( + optional=optional, + client_default_value_declaration=client_default_value_declaration, + description=description, + ) + + def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: + raise ValueError("You shouldn't get polymorphic subtypes of multiple types") + + @property + def instance_check_template(self) -> str: + """Template of what an instance check of a variable for this type would look like""" + raise ValueError("You shouldn't do instance checks on a multiple type") + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + if self.name and not kwargs.get("is_types_file"): + file_import.add_submodule_import( + kwargs.pop("relative_path"), + "_types", + ImportType.LOCAL, + TypingSection.TYPING, + ) + return file_import + for type in self.types: + file_import.merge(type.imports(**kwargs)) + if not self._is_union_of_literals: + file_import.add_submodule_import("typing", "Union", ImportType.STDLIB) + return file_import + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "BaseType": + from . import build_type + + return cls( + yaml_data, + code_model, + [build_type(t, code_model) for t in yaml_data["types"]], + ) + + def target_model_subtype( + self, + target_types: Union[ + Tuple[Type[ModelType]], + Tuple[Type[ModelType], Type[ModelType]], + ], + ) -> Optional[ModelType]: + for sub_t in self.types: + if isinstance(sub_t, target_types): + return sub_t # type: ignore + return None diff --git a/packages/autorest.python/generator/pygen/codegen/models/constant_type.py b/packages/autorest.python/generator/pygen/codegen/models/constant_type.py new file mode 100644 index 00000000000..16be177de4c --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/constant_type.py @@ -0,0 +1,134 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from typing import Dict, Any, Optional, TYPE_CHECKING, Union +from .base import BaseType +from .imports import FileImport, ImportType, TypingSection +from .primitive_types import IntegerType, BinaryType, StringType, BooleanType +from .utils import add_to_description + +if TYPE_CHECKING: + from .code_model import CodeModel + +_LOGGER = logging.getLogger(__name__) + + +class ConstantType(BaseType): + """Schema for constants that will be serialized. + + :param yaml_data: the yaml data for this schema + :type yaml_data: dict[str, Any] + :param str value: The actual value of this constant. + :param schema: The schema for the value of this constant. + :type schema: ~autorest.models.PrimitiveType + """ + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + value_type: BaseType, + value: Optional[Union[str, int, float]], + ) -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.value_type = value_type + self.value = value + + def get_declaration(self, value=None): + if value and value != self.value: + _LOGGER.warning( + "Passed in value of %s differs from constant value of %s. Choosing constant value", + str(value), + str(self.value), + ) + if self.value is None: + return "None" + return self.value_type.get_declaration(self.value) + + def description(self, *, is_operation_file: bool) -> str: + if is_operation_file: + return "" + return add_to_description( + self.yaml_data.get("description", ""), + f"Default value is {self.get_declaration()}.", + ) + + @property + def serialization_type(self) -> str: + """Returns the serialization value for msrest. + + :return: The serialization value for msrest + :rtype: str + """ + return self.value_type.serialization_type + + def docstring_text(self, **kwargs: Any) -> str: + return "constant" + + def docstring_type(self, **kwargs: Any) -> str: + """The python type used for RST syntax input and type annotation. + + :param str namespace: Optional. The namespace for the models. + """ + return self.value_type.docstring_type(**kwargs) + + def type_annotation(self, **kwargs: Any) -> str: + return f"Literal[{self.get_declaration()}]" if self._is_literal else self.value_type.type_annotation(**kwargs) + + @property + def _is_literal(self) -> bool: + return isinstance(self.value_type, (IntegerType, BinaryType, StringType, BooleanType)) + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "ConstantType": + """Constructs a ConstantType from yaml data. + + :param yaml_data: the yaml data from which we will construct this schema + :type yaml_data: dict[str, Any] + + :return: A created ConstantType + :rtype: ~autorest.models.ConstantType + """ + from . import build_type + + return cls( + yaml_data=yaml_data, + code_model=code_model, + value_type=build_type(yaml_data["valueType"], code_model), + value=yaml_data["value"], + ) + + def get_json_template_representation( + self, + *, + optional: bool = True, + # pylint: disable=unused-argument + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + return self.value_type.get_json_template_representation( + optional=optional, + client_default_value_declaration=self.get_declaration(), + description=description, + ) + + def _imports_shared(self, **kwargs: Any): + file_import = super().imports(**kwargs) + file_import.merge(self.value_type.imports(**kwargs)) + return file_import + + def imports_for_multiapi(self, **kwargs: Any) -> FileImport: + return self._imports_shared(**kwargs) + + def imports(self, **kwargs: Any) -> FileImport: + file_import = self._imports_shared(**kwargs) + if self._is_literal: + file_import.add_submodule_import("typing", "Literal", ImportType.STDLIB, TypingSection.REGULAR) + return file_import + + @property + def instance_check_template(self) -> str: + return self.value_type.instance_check_template diff --git a/packages/autorest.python/generator/pygen/codegen/models/credential_types.py b/packages/autorest.python/generator/pygen/codegen/models/credential_types.py new file mode 100644 index 00000000000..261cad0e228 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/credential_types.py @@ -0,0 +1,223 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from abc import abstractmethod +from typing import ( + Optional, + Any, + Dict, + TYPE_CHECKING, + List, + Generic, + TypeVar, + Union, + cast, +) + +from .imports import FileImport, ImportType, TypingSection +from .base import BaseType + +if TYPE_CHECKING: + from .code_model import CodeModel + + +class _CredentialPolicyBaseType: + """Base class for our different credential policy types. + + Inherited by our BearerTokenCredentialPolicy and KeyCredentialPolicy types. + """ + + def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + self.yaml_data = yaml_data + self.code_model = code_model + + @abstractmethod + def call(self, async_mode: bool) -> str: + """ + How to call this credential policy. Used to initialize the credential policy in the config file. + """ + + +class BearerTokenCredentialPolicyType(_CredentialPolicyBaseType): + """Credential policy type representing BearerTokenCredentialPolicy""" + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + credential_scopes: List[str], + ) -> None: + super().__init__(yaml_data, code_model) + self.credential_scopes = credential_scopes + + def call(self, async_mode: bool) -> str: + policy_name = f"{'Async' if async_mode else ''}BearerTokenCredentialPolicy" + return f"policies.{policy_name}(self.credential, *self.credential_scopes, **kwargs)" + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "BearerTokenCredentialPolicyType": + return cls(yaml_data, code_model, yaml_data["credentialScopes"]) + + +class ARMChallengeAuthenticationPolicyType(BearerTokenCredentialPolicyType): + """Credential policy type representing ARMChallengeAuthenticationPolicy""" + + def call(self, async_mode: bool) -> str: + policy_name = f"{'Async' if async_mode else ''}ARMChallengeAuthenticationPolicy" + return f"{policy_name}(self.credential, *self.credential_scopes, **kwargs)" + + +class KeyCredentialPolicyType(_CredentialPolicyBaseType): + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + key: str, + scheme: Optional[str] = None, + ) -> None: + super().__init__(yaml_data, code_model) + self.key = key + self.scheme = scheme + + @property + def credential_name(self) -> str: + return "AzureKeyCredential" if self.code_model.is_azure_flavor else "ServiceKeyCredential" + + def call(self, async_mode: bool) -> str: + params = f'"{self.key}", ' + if self.scheme: + params += f'prefix="{self.scheme}", ' + return f"policies.{self.credential_name}Policy(self.credential, {params}**kwargs)" + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "KeyCredentialPolicyType": + return cls(yaml_data, code_model, yaml_data["key"], yaml_data.get("scheme", None)) + + +CredentialPolicyType = TypeVar( + "CredentialPolicyType", + bound=Union[ + BearerTokenCredentialPolicyType, + ARMChallengeAuthenticationPolicyType, + KeyCredentialPolicyType, + ], +) + + +class CredentialType(Generic[CredentialPolicyType], BaseType): # pylint:disable=abstract-method + """Store info about the type of the credential. Can be either an KeyCredential or a TokenCredential""" + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + policy: CredentialPolicyType, + ) -> None: + super().__init__(yaml_data, code_model) + self.policy = policy + + def description(self, *, is_operation_file: bool) -> str: # pylint: disable=unused-argument + return "" + + def get_json_template_representation( + self, + *, + optional: bool = True, + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + raise TypeError("You should not try to get a JSON template representation of a CredentialSchema") + + def docstring_text(self, **kwargs: Any) -> str: + return "credential" + + @property + def serialization_type(self) -> str: + return self.docstring_type() + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "CredentialType": + from . import build_type + + return cls( + yaml_data, + code_model, + policy=cast(CredentialPolicyType, build_type(yaml_data["policy"], code_model)), + ) + + +class TokenCredentialType( + CredentialType[ # pylint: disable=unsubscriptable-object + Union[BearerTokenCredentialPolicyType, ARMChallengeAuthenticationPolicyType] + ] +): + """Type of a token credential. Used by BearerAuth and ARMChallenge policies""" + + def type_annotation(self, **kwargs: Any) -> str: + if kwargs.get("async_mode"): + return '"AsyncTokenCredential"' + return '"TokenCredential"' + + @property + def type_description(self) -> str: + return "TokenCredential" + + @property + def credentials_subfolder(self) -> str: + return "credentials_async" if self.code_model.is_azure_flavor else "credentials" + + def docstring_type(self, **kwargs: Any) -> str: + if kwargs.get("async_mode"): + return f"~{self.code_model.core_library}.{self.credentials_subfolder}.AsyncTokenCredential" + return f"~{self.code_model.core_library}.credentials.TokenCredential" + + def imports(self, **kwargs: Any) -> FileImport: + file_import = super().imports(**kwargs) + if kwargs.get("async_mode"): + file_import.add_submodule_import( + self.credentials_subfolder, + "AsyncTokenCredential", + ImportType.SDKCORE, + typing_section=TypingSection.TYPING, + ) + else: + file_import.add_submodule_import( + "credentials", + "TokenCredential", + ImportType.SDKCORE, + typing_section=TypingSection.TYPING, + ) + return file_import + + @property + def instance_check_template(self) -> str: + return "hasattr({}, 'get_token')" + + +class KeyCredentialType( + # pylint: disable=unsubscriptable-object + CredentialType[KeyCredentialPolicyType] +): + """Type for an KeyCredential""" + + def docstring_type(self, **kwargs: Any) -> str: # pylint: disable=unused-argument + return f"~{self.code_model.core_library}.credentials.{self.policy.credential_name}" + + def type_annotation(self, **kwargs: Any) -> str: # pylint: disable=unused-argument + return self.policy.credential_name + + @property + def instance_check_template(self) -> str: + return "isinstance({}, " + f"{self.policy.credential_name})" + + def imports(self, **kwargs: Any) -> FileImport: # pylint: disable=unused-argument + file_import = super().imports(**kwargs) + file_import.add_submodule_import( + "credentials", + self.policy.credential_name, + ImportType.SDKCORE, + typing_section=TypingSection.CONDITIONAL, + ) + return file_import diff --git a/packages/autorest.python/generator/pygen/codegen/models/dictionary_type.py b/packages/autorest.python/generator/pygen/codegen/models/dictionary_type.py new file mode 100644 index 00000000000..bf00b1c2317 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/dictionary_type.py @@ -0,0 +1,131 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict, Optional, TYPE_CHECKING, List +from .base import BaseType +from .imports import FileImport, ImportType, TypingSection + +if TYPE_CHECKING: + from .code_model import CodeModel + from .model_type import ModelType + + +class DictionaryType(BaseType): + """Schema for dictionaries that will be serialized. + + :param yaml_data: the yaml data for this schema + :type yaml_data: dict[str, Any] + :param element_type: The type of the value for the dictionary + :type element_type: ~autorest.models.BaseType + """ + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + element_type: BaseType, + ) -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.element_type = element_type + + @property + def encode(self) -> Optional[str]: + return self.element_type.encode if hasattr(self.element_type, "encode") else None # type: ignore + + @property + def serialization_type(self) -> str: + """Returns the serialization value for msrest. + + :return: The serialization value for msrest + :rtype: str + """ + return f"{{{self.element_type.serialization_type}}}" + + def type_annotation(self, **kwargs: Any) -> str: + """The python type used for type annotation + + :return: The type annotation for this schema + :rtype: str + """ + return f"Dict[str, {self.element_type.type_annotation(**kwargs)}]" + + def description(self, *, is_operation_file: bool) -> str: + return "" if is_operation_file else self.yaml_data.get("description", "") + + def docstring_text(self, **kwargs: Any) -> str: + return f"dict mapping str to {self.element_type.docstring_text(**kwargs)}" + + @property + def xml_serialization_ctxt(self) -> Optional[str]: + """No serialization ctxt for dictionaries""" + return None + + def docstring_type(self, **kwargs: Any) -> str: + """The python type used for RST syntax input and type annotation. + + :param str namespace: Optional. The namespace for the models. + """ + return f"dict[str, {self.element_type.docstring_type(**kwargs)}]" + + def get_json_template_representation( + self, + *, + optional: bool = True, + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + return { + '"str"': self.element_type.get_json_template_representation( + optional=optional, + client_default_value_declaration=client_default_value_declaration, + description=description, + ) + } + + def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: + from .model_type import ModelType + + if isinstance(self.element_type, ModelType): + is_polymorphic_subtype = ( + self.element_type.discriminator_value and not self.element_type.discriminated_subtypes + ) + if self.element_type.name not in (m.name for m in polymorphic_subtypes) and is_polymorphic_subtype: + polymorphic_subtypes.append(self.element_type) + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "DictionaryType": + """Constructs a DictionaryType from yaml data. + + :param yaml_data: the yaml data from which we will construct this schema + :type yaml_data: dict[str, Any] + + :return: A created DictionaryType + :rtype: ~autorest.models.DictionaryType + """ + element_schema: Dict[str, Any] = yaml_data["elementType"] + + from . import build_type # pylint: disable=import-outside-toplevel + + element_type = build_type(yaml_data=element_schema, code_model=code_model) + + return cls( + yaml_data=yaml_data, + code_model=code_model, + element_type=element_type, + ) + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB, TypingSection.CONDITIONAL) + file_import.merge(self.element_type.imports(**kwargs)) + return file_import + + @property + def instance_check_template(self) -> str: + return "isinstance({}, dict)" + + @property + def type_description(self) -> str: + return f"{{str: {self.element_type.type_description}}}" diff --git a/packages/autorest.python/generator/pygen/codegen/models/enum_type.py b/packages/autorest.python/generator/pygen/codegen/models/enum_type.py new file mode 100644 index 00000000000..6e9c74531c7 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/enum_type.py @@ -0,0 +1,246 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict, List, TYPE_CHECKING, Optional, cast + +from .base import BaseType +from .imports import FileImport, ImportType, TypingSection + +if TYPE_CHECKING: + from .code_model import CodeModel + + +class EnumValue(BaseType): + """Model containing necessary information for a single value of an enum. + + :param str name: The name of this enum value + :param str value: The value of this enum value + :param str description: Optional. The description for this enum value + """ + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + enum_type: "EnumType", + value_type: BaseType, + ) -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.name: str = self.yaml_data["name"] + self.value: str = self.yaml_data["value"] + self.enum_type = enum_type + self.value_type = value_type + + def description(self, *, is_operation_file: bool) -> str: + return self.yaml_data.get("description", "") + + def type_annotation(self, **kwargs: Any) -> str: + """The python type used for type annotation""" + return f"Literal[{self.enum_type.name}.{self.name}]" + + def get_declaration(self, value=None): + return self.enum_type.name + "." + self.name + + def docstring_text(self, **kwargs: Any) -> str: + return self.enum_type.name + "." + self.name + + def docstring_type(self, **kwargs: Any) -> str: + """The python type used for RST syntax input and type annotation.""" + + type_annotation = self.value_type.type_annotation(**kwargs) + enum_type_annotation = f"{self.code_model.namespace}.models.{self.name}" + return f"{type_annotation} or ~{enum_type_annotation}" + + def get_json_template_representation( + self, + *, + optional: bool = True, + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + # for better display effect, use the only value instead of var type + return self.value_type.get_json_template_representation( + optional=optional, + client_default_value_declaration=client_default_value_declaration, + description=description, + ) + + @property + def serialization_type(self) -> str: + return self.value_type.serialization_type + + @property + def instance_check_template(self) -> str: + return self.value_type.instance_check_template + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.merge(self.value_type.imports(**kwargs)) + file_import.add_submodule_import("typing", "Literal", ImportType.STDLIB, TypingSection.REGULAR) + file_import.add_submodule_import("._enums", self.enum_type.name, ImportType.LOCAL, TypingSection.REGULAR) + + return file_import + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "EnumValue": + """Constructs an EnumValue from yaml data. + + :param yaml_data: the yaml data from which we will construct this object + :type yaml_data: dict[str, Any] + + :return: A created EnumValue + :rtype: ~autorest.models.EnumValue + """ + from . import build_type + + return cls( + yaml_data=yaml_data, + code_model=code_model, + enum_type=cast(EnumType, build_type(yaml_data["enumType"], code_model)), + value_type=build_type(yaml_data["valueType"], code_model), + ) + + +class EnumType(BaseType): + """Schema for enums that will be serialized. + + :param yaml_data: the yaml data for this schema + :type yaml_data: dict[str, Any] + :param str description: The description of this enum + :param str name: The name of the enum. + :type element_type: ~autorest.models.PrimitiveType + :param values: List of the values for this enum + :type values: list[~autorest.models.EnumValue] + """ + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + values: List["EnumValue"], + value_type: BaseType, + ) -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.name: str = yaml_data["name"][0].upper() + yaml_data["name"][1:] + self.values = values + self.value_type = value_type + self.internal: bool = self.yaml_data.get("internal", False) + self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") + + def __lt__(self, other): + return self.name.lower() < other.name.lower() + + @property + def serialization_type(self) -> str: + """Returns the serialization value for msrest. + + :return: The serialization value for msrest + :rtype: str + """ + return self.value_type.serialization_type + + def description(self, *, is_operation_file: bool) -> str: # pylint: disable=unused-argument + possible_values = [self.get_declaration(v.value) for v in self.values] + if not possible_values: + return "" + if len(possible_values) == 1: + return possible_values[0] + if len(possible_values) == 2: + possible_values_str = " and ".join(possible_values) + else: + possible_values_str = ( + ", ".join(possible_values[: len(possible_values) - 1]) + f", and {possible_values[-1]}" + ) + + enum_description = f"Known values are: {possible_values_str}." + return enum_description + + def type_annotation(self, **kwargs: Any) -> str: + """The python type used for type annotation + + :return: The type annotation for this schema + :rtype: str + """ + if self.code_model.options["models_mode"]: + module_name = "_models." if kwargs.get("need_module_name", True) else "" + file_name = f"{self.code_model.enums_filename}." if self.internal else "" + model_name = module_name + file_name + self.name + # we don't need quoted annotation in operation files, and need it in model folder files. + if not kwargs.get("is_operation_file", False): + model_name = f'"{model_name}"' + + return f"Union[{self.value_type.type_annotation(**kwargs)}, {model_name}]" + return self.value_type.type_annotation(**kwargs) + + def get_declaration(self, value: Any) -> str: + return self.value_type.get_declaration(value) + + def docstring_text(self, **kwargs: Any) -> str: + if self.code_model.options["models_mode"]: + return self.name + return self.value_type.type_annotation(**kwargs) + + def docstring_type(self, **kwargs: Any) -> str: + """The python type used for RST syntax input and type annotation.""" + if self.code_model.options["models_mode"]: + type_annotation = self.value_type.type_annotation(**kwargs) + enum_type_annotation = f"{self.code_model.namespace}.models.{self.name}" + return f"{type_annotation} or ~{enum_type_annotation}" + return self.value_type.type_annotation(**kwargs) + + def get_json_template_representation( + self, + *, + optional: bool = True, + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + # for better display effect, use the only value instead of var type + return self.value_type.get_json_template_representation( + optional=optional, + client_default_value_declaration=client_default_value_declaration, + description=description, + ) + + @property + def instance_check_template(self) -> str: + return self.value_type.instance_check_template + + def fill_instance_from_yaml(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + for value in yaml_data["values"]: + self.values.append(EnumValue.from_yaml(value, code_model)) + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "EnumType": + raise ValueError( + "You shouldn't call from_yaml for EnumType to avoid recursion. " + "Please initial a blank EnumType, then call .fill_instance_from_yaml on the created type." + ) + + def imports(self, **kwargs: Any) -> FileImport: + operation = kwargs.pop("operation", False) + file_import = FileImport(self.code_model) + if self.code_model.options["models_mode"]: + file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) + if not operation: + file_import.add_submodule_import( + "..", + "models", + ImportType.LOCAL, + TypingSection.TYPING, + alias="_models", + ) + file_import.merge(self.value_type.imports(operation=operation, **kwargs)) + relative_path = kwargs.pop("relative_path", None) + if self.code_model.options["models_mode"] and relative_path: + # add import for enums in operations file + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias="_models", + typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), + ) + return file_import diff --git a/packages/autorest.python/generator/pygen/codegen/models/imports.py b/packages/autorest.python/generator/pygen/codegen/models/imports.py new file mode 100644 index 00000000000..8e21e4b9c1b --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/imports.py @@ -0,0 +1,291 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from enum import Enum, auto +from typing import Dict, List, Optional, Tuple, Union, Set, TYPE_CHECKING + +if TYPE_CHECKING: + from .code_model import CodeModel + + +class ImportType(str, Enum): + """ + Ordering of these enum matters. We order import groupings in a file based off of this ordering. + """ + + STDLIB = "stdlib" + THIRDPARTY = "thirdparty" + SDKCORE = "sdkcore" + LOCAL = "local" + BYVERSION = "by_version" + + +class TypingSection(str, Enum): + REGULAR = "regular" # this import is always a typing import + CONDITIONAL = "conditional" # is a typing import when we're dealing with files that py2 will use, else regular + TYPING = "typing" # never a typing import + + +class MsrestImportType(Enum): + Module = auto() # import _serialization.py or msrest.serialization as Module + Serializer = auto() # from _serialization.py or msrest.serialization import Serializer + SerializerDeserializer = auto() # from _serialization.py or msrest.serialization import Serializer and Deserializer + + +class ImportModel: + def __init__( + self, + typing_section: TypingSection, + import_type: ImportType, + module_name: str, + *, + submodule_name: Optional[str] = None, + alias: Optional[str] = None, + version_modules: Optional[Tuple[Tuple[Tuple[int, int], str, Optional[str]]]] = None, + ): + self.typing_section = typing_section + self.import_type = import_type + self.module_name = module_name + self.submodule_name = submodule_name + self.alias = alias + # version_modules: this field is for imports submodule from specified module by python version. + # It's a list of "python version, module_name, comments". + # The python version is in form of (major, minor), for instance (3, 9) stands for py3.9. + self.version_modules = version_modules + + def __eq__(self, other): + try: + return ( + self.typing_section == other.typing_section + and self.import_type == other.import_type + and self.module_name == other.module_name + and self.submodule_name == other.submodule_name + and self.alias == other.alias + ) + except AttributeError: + return False + + def __hash__(self) -> int: + retval: int = 0 + for attr in dir(self): + if attr[0] != "_": + retval += hash(getattr(self, attr)) + return retval + + +class TypeDefinition: + def __init__( + self, + sync_definition: str, + async_definition: str, + ): + self.sync_definition = sync_definition + self.async_definition = async_definition + + +class FileImport: + def __init__(self, code_model: "CodeModel") -> None: + self.imports: List[ImportModel] = [] + self.code_model = code_model + # has sync and async type definitions + self.type_definitions: Dict[str, TypeDefinition] = {} + self.core_library = self.code_model.core_library + + def _append_import(self, import_model: ImportModel) -> None: + if import_model.import_type == ImportType.SDKCORE: + mod_name = import_model.module_name + core_libraries = [ + self.code_model.core_library, + "azure", + "msrest", + ] + if all(l not in mod_name for l in core_libraries): + # this is to make sure we don't tack on core libraries when we don't need to + import_model.module_name = f"{self.code_model.core_library}{'.' if mod_name else ''}{mod_name}" + if not any( + i + for i in self.imports + if all(getattr(i, attr) == getattr(import_model, attr) for attr in dir(i) if attr[0] != "_") + ): + self.imports.append(import_model) + + def get_imports_from_section(self, typing_section: TypingSection) -> List[ImportModel]: + return [i for i in self.imports if i.typing_section == typing_section] + + def add_submodule_import( + self, + module_name: str, + submodule_name: str, + import_type: ImportType, + typing_section: TypingSection = TypingSection.REGULAR, + alias: Optional[str] = None, + version_modules: Optional[Tuple[Tuple[Tuple[int, int], str, Optional[str]]]] = None, + ) -> None: + """Add an import to this import block.""" + self._append_import( + ImportModel( + typing_section=typing_section, + import_type=import_type, + module_name=module_name, + submodule_name=submodule_name, + alias=alias, + version_modules=version_modules, + ) + ) + + def add_import( + self, + module_name: str, + import_type: ImportType, + typing_section: TypingSection = TypingSection.REGULAR, + alias: Optional[str] = None, + ) -> None: + # Implementation detail: a regular import is just a "from" with no from + self._append_import( + ImportModel( + typing_section=typing_section, + import_type=import_type, + module_name=module_name, + alias=alias, + ) + ) + + def define_mypy_type( + self, + type_name: str, + type_value: str, + async_type_value: Optional[str] = None, + ): + self.type_definitions[type_name] = TypeDefinition(type_value, async_type_value or type_value) + + def merge(self, file_import: "FileImport") -> None: + """Merge the given file import format.""" + for i in file_import.imports: + self._append_import(i) + self.type_definitions.update(file_import.type_definitions) + + def add_mutable_mapping_import(self) -> None: + self.add_import("sys", ImportType.STDLIB) + self.add_submodule_import( + "typing", + "MutableMapping", + ImportType.BYVERSION, + TypingSection.REGULAR, + None, + (((3, 9), "collections.abc", None),), + ) + + def define_mutable_mapping_type(self) -> None: + """Helper function for defining the mutable mapping type""" + self.add_mutable_mapping_import() + self.define_mypy_type( + "JSON", + "MutableMapping[str, Any] # pylint: disable=unsubscriptable-object", + ) + self.add_submodule_import("typing", "Any", ImportType.STDLIB) + + def to_dict( + self, + ) -> Dict[ + TypingSection, + Dict[ + ImportType, + Dict[ + str, + Set[ + Optional[ + Union[ + str, + Tuple[str, str], + Tuple[ + str, + Optional[str], + Tuple[Tuple[Tuple[int, int], str, Optional[str]]], + ], + ] + ] + ], + ], + ], + ]: + retval: Dict[ + TypingSection, + Dict[ + ImportType, + Dict[ + str, + Set[ + Optional[ + Union[ + str, + Tuple[str, str], + Tuple[ + str, + Optional[str], + Tuple[Tuple[Tuple[int, int], str, Optional[str]]], + ], + ] + ] + ], + ], + ], + ] = {} + for i in self.imports: + name_import: Optional[ + Union[ + str, + Tuple[str, str], + Tuple[ + str, + Optional[str], + Tuple[Tuple[Tuple[int, int], str, Optional[str]]], + ], + ] + ] = None + if i.submodule_name: + if i.version_modules: + name_import = (i.submodule_name, i.alias, i.version_modules) + elif i.alias: + name_import = (i.submodule_name, i.alias) + else: + name_import = i.submodule_name + retval.setdefault(i.typing_section, {}).setdefault(i.import_type, {}).setdefault(i.module_name, set()).add( + name_import + ) + return retval + + def add_msrest_import( + self, + *, + relative_path: str, + msrest_import_type: MsrestImportType, + typing_section: TypingSection, + ): + if self.code_model.options["client_side_validation"]: + if msrest_import_type == MsrestImportType.Module: + self.add_import("msrest.serialization", ImportType.SDKCORE, typing_section) + else: + self.add_submodule_import("msrest", "Serializer", ImportType.THIRDPARTY, typing_section) + if msrest_import_type == MsrestImportType.SerializerDeserializer: + self.add_submodule_import("msrest", "Deserializer", ImportType.THIRDPARTY, typing_section) + else: + if self.code_model.options["multiapi"]: + relative_path += "." + if msrest_import_type == MsrestImportType.Module: + self.add_submodule_import(relative_path, "_serialization", ImportType.LOCAL, typing_section) + else: + self.add_submodule_import( + f"{relative_path}_serialization", + "Serializer", + ImportType.LOCAL, + typing_section, + ) + if msrest_import_type == MsrestImportType.SerializerDeserializer: + self.add_submodule_import( + f"{relative_path}_serialization", + "Deserializer", + ImportType.LOCAL, + typing_section, + ) diff --git a/packages/autorest.python/generator/pygen/codegen/models/list_type.py b/packages/autorest.python/generator/pygen/codegen/models/list_type.py new file mode 100644 index 00000000000..79a2745086f --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/list_type.py @@ -0,0 +1,147 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict, Optional, Union, TYPE_CHECKING, List +from .base import BaseType +from .imports import FileImport, ImportType, TypingSection + +if TYPE_CHECKING: + from .code_model import CodeModel + from .model_type import ModelType + + +class ListType(BaseType): + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + element_type: BaseType, + ) -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.element_type = element_type + self.max_items: Optional[int] = yaml_data.get("maxItems") + self.min_items: Optional[int] = yaml_data.get("minItems") + self.unique_items: bool = yaml_data.get("uniqueItems", False) + + @property + def encode(self) -> Optional[str]: + return self.element_type.encode if hasattr(self.element_type, "encode") else None # type: ignore + + @property + def serialization_type(self) -> str: + return f"[{self.element_type.serialization_type}]" + + def type_annotation(self, **kwargs: Any) -> str: + if ( + self.code_model.options["version_tolerant"] + and self.element_type.is_xml + and not self.code_model.options["models_mode"] + ): + # this means we're version tolerant XML, we just return the XML element + return self.element_type.type_annotation(**kwargs) + return f"List[{self.element_type.type_annotation(**kwargs)}]" + + def description(self, *, is_operation_file: bool) -> str: + return "" if is_operation_file else self.yaml_data.get("description", "") + + @property + def xml_serialization_ctxt(self) -> Optional[str]: + attrs_list = [] + base_xml_map = super().xml_serialization_ctxt + if base_xml_map: + attrs_list.append(base_xml_map) + + # Attribute at the list level + if self.xml_metadata.get("wrapped", False): + attrs_list.append("'wrapped': True") + + # Attributes of the items + item_xml_metadata = self.element_type.xml_metadata + if item_xml_metadata.get("name"): + attrs_list.append(f"'itemsName': '{item_xml_metadata['name']}'") + if item_xml_metadata.get("prefix", False): + attrs_list.append(f"'itemsPrefix': '{item_xml_metadata['prefix']}'") + if item_xml_metadata.get("namespace", False): + attrs_list.append(f"'itemsNs': '{item_xml_metadata['namespace']}'") + + return ", ".join(attrs_list) + + def docstring_type(self, **kwargs: Any) -> str: + if self.code_model.options["version_tolerant"] and self.element_type.xml_metadata: + # this means we're version tolerant XML, we just return the XML element + return self.element_type.docstring_type(**kwargs) + return f"list[{self.element_type.docstring_type(**kwargs)}]" + + def docstring_text(self, **kwargs: Any) -> str: + if self.code_model.options["version_tolerant"] and self.element_type.xml_metadata: + # this means we're version tolerant XML, we just return the XML element + return self.element_type.docstring_text(**kwargs) + return f"list of {self.element_type.docstring_text(**kwargs)}" + + @property + def validation(self) -> Optional[Dict[str, Union[bool, int, str]]]: + validation: Dict[str, Union[bool, int, str]] = {} + if self.max_items: + validation["max_items"] = self.max_items + validation["min_items"] = self.min_items or 0 + if self.min_items: + validation["min_items"] = self.min_items + if self.unique_items: + validation["unique"] = True + return validation or None + + def get_json_template_representation( + self, + *, + optional: bool = True, + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + return [ + self.element_type.get_json_template_representation( + optional=optional, + client_default_value_declaration=client_default_value_declaration, + description=description, + ) + ] + + def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: + from .model_type import ModelType + + if isinstance(self.element_type, ModelType): + is_polymorphic_subtype = ( + self.element_type.discriminator_value and not self.element_type.discriminated_subtypes + ) + if self.element_type.name not in (m.name for m in polymorphic_subtypes) and is_polymorphic_subtype: + polymorphic_subtypes.append(self.element_type) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, list)" + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "ListType": + from . import build_type + + return cls( + yaml_data=yaml_data, + code_model=code_model, + element_type=build_type(yaml_data=yaml_data["elementType"], code_model=code_model), + ) + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + if not ( + self.code_model.options["version_tolerant"] + and self.element_type.is_xml + and not self.code_model.options["models_mode"] + ): + file_import.add_submodule_import("typing", "List", ImportType.STDLIB, TypingSection.CONDITIONAL) + file_import.merge(self.element_type.imports(**kwargs)) + return file_import + + @property + def type_description(self) -> str: + return f"[{self.element_type.type_description}]" diff --git a/packages/autorest.python/generator/pygen/codegen/models/lro_operation.py b/packages/autorest.python/generator/pygen/codegen/models/lro_operation.py new file mode 100644 index 00000000000..2e127df9d3b --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/lro_operation.py @@ -0,0 +1,143 @@ +# pylint: disable=multiple-statements +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict, Optional, List, TYPE_CHECKING, TypeVar, Union +from .imports import FileImport +from .operation import OperationBase, Operation +from .response import LROPagingResponse, LROResponse, Response +from .imports import ImportType, TypingSection +from .request_builder import RequestBuilder +from .parameter_list import ParameterList + +if TYPE_CHECKING: + from .code_model import CodeModel + from .client import Client + from . import OperationType + +LROResponseType = TypeVar("LROResponseType", bound=Union[LROResponse, LROPagingResponse]) + + +class LROOperationBase(OperationBase[LROResponseType]): + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + client: "Client", + name: str, + request_builder: RequestBuilder, + parameters: ParameterList, + responses: List[LROResponseType], + exceptions: List[Response], + *, + overloads: Optional[List[Operation]] = None, + ) -> None: + super().__init__( + code_model=code_model, + client=client, + yaml_data=yaml_data, + name=name, + request_builder=request_builder, + parameters=parameters, + responses=responses, + exceptions=exceptions, + overloads=overloads, + ) + if not self.name.lstrip("_").startswith("begin"): + self.name = ("_begin" if self.internal else "begin_") + self.name + self.lro_options: Dict[str, Any] = self.yaml_data.get("lroOptions", {}) + self._initial_operation: Optional["OperationType"] = None + + @property + def initial_operation(self) -> "OperationType": + if not self._initial_operation: + raise ValueError("You need to first call client.link_lro_initial_operations before accessing") + return self._initial_operation + + @initial_operation.setter + def initial_operation(self, val: "OperationType") -> None: + self._initial_operation = val + + @property + def operation_type(self) -> str: + return "lro" + + @property + def has_optional_return_type(self) -> bool: + return False + + @property + def lro_response(self) -> Optional[LROResponseType]: + responses_with_bodies = [r for r in self.responses if r.type] + num_response_schemas = {id(r.type.yaml_data) for r in responses_with_bodies if r.type} + response = None + if len(num_response_schemas) > 1: + # choose the response that has a status code of 200 + try: + response = next(r for r in responses_with_bodies if 200 in r.status_codes) + except StopIteration as exc: + raise ValueError( + "Your swagger is invalid because you have multiple response schemas for LRO" + + f" method {self.name} and none of them have a 200 status code." + ) from exc + + elif num_response_schemas: + response = responses_with_bodies[0] + return response + + def response_type_annotation(self, **kwargs) -> str: + lro_response = self.lro_response or next(iter(self.responses), None) + if lro_response: + return lro_response.type_annotation(**kwargs) + return "None" + + def cls_type_annotation(self, *, async_mode: bool) -> str: + """We don't want the poller to show up in ClsType, so we call super() on resposne type annotation""" + return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode)}]" + + def get_poller_with_response_type(self, async_mode: bool) -> str: + return self.response_type_annotation(async_mode=async_mode) + + def get_poller(self, async_mode: bool) -> str: + return self.responses[0].get_poller(async_mode) + + def get_polling_method(self, async_mode: bool) -> str: + return self.responses[0].get_polling_method(async_mode) + + def get_base_polling_method(self, async_mode: bool) -> str: + return self.responses[0].get_base_polling_method(async_mode) + + def get_base_polling_method_path(self, async_mode: bool) -> str: + return self.responses[0].get_base_polling_method_path(async_mode) + + def get_no_polling_method(self, async_mode: bool) -> str: + return self.responses[0].get_no_polling_method(async_mode) + + def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: + file_import = super().imports(async_mode, **kwargs) + if self.abstract: + return file_import + if async_mode and self.code_model.options["tracing"] and self.want_tracing: + file_import.add_submodule_import( + "azure.core.tracing.decorator_async", + "distributed_trace_async", + ImportType.SDKCORE, + ) + if ( + self.code_model.options["models_mode"] == "dpg" + and self.lro_response + and self.lro_response.type + and self.lro_response.type.type == "model" + ): + # used in the case if initial operation returns none + # but final call returns a model + relative_path = "..." if async_mode else ".." + file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) + file_import.add_submodule_import("typing", "cast", ImportType.STDLIB) + return file_import + + +class LROOperation(LROOperationBase[LROResponse]): ... diff --git a/packages/autorest.python/generator/pygen/codegen/models/lro_paging_operation.py b/packages/autorest.python/generator/pygen/codegen/models/lro_paging_operation.py new file mode 100644 index 00000000000..42c885f308f --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/lro_paging_operation.py @@ -0,0 +1,32 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any +from .imports import FileImport +from .lro_operation import LROOperationBase +from .paging_operation import PagingOperationBase +from .response import LROPagingResponse, Response + + +class LROPagingOperation(LROOperationBase[LROPagingResponse], PagingOperationBase[LROPagingResponse]): + @property + def success_status_codes(self): + """The list of all successfull status code.""" + return [200] + + @property + def operation_type(self) -> str: + return "lropaging" + + def cls_type_annotation(self, *, async_mode: bool) -> str: + return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode)}]" # pylint: disable=no-member + + def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: + lro_imports = LROOperationBase.imports(self, async_mode, **kwargs) + paging_imports = PagingOperationBase.imports(self, async_mode, **kwargs) + + file_import = lro_imports + file_import.merge(paging_imports) + return file_import diff --git a/packages/autorest.python/generator/pygen/codegen/models/model_type.py b/packages/autorest.python/generator/pygen/codegen/models/model_type.py new file mode 100644 index 00000000000..84bae17609a --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/model_type.py @@ -0,0 +1,350 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from collections import OrderedDict +from typing import Any, Dict, List, Optional, TYPE_CHECKING, cast +import sys +from .utils import ( + add_to_pylint_disable, + NAME_LENGTH_LIMIT, +) +from .base import BaseType +from .constant_type import ConstantType +from .property import Property +from .imports import FileImport, ImportType, TypingSection + +if sys.version_info >= (3, 8): + from typing import Literal # pylint: disable=no-name-in-module, ungrouped-imports +else: + from typing_extensions import Literal # type: ignore # pylint: disable=ungrouped-imports + +if TYPE_CHECKING: + from .code_model import CodeModel + + +def _get_properties(type: "ModelType", properties: List[Property]) -> List[Property]: + for parent in type.parents: + # here we're adding the properties from our parents + + # need to make sure that the properties we choose from our parent also don't contain + # any of our own properties + property_names = set([p.client_name for p in properties] + [p.client_name for p in type.properties]) + chosen_parent_properties = [p for p in parent.properties if p.client_name not in property_names] + properties = _get_properties(parent, chosen_parent_properties) + properties + return properties + + +class ModelType( # pylint: disable=abstract-method + BaseType +): # pylint: disable=too-many-instance-attributes, too-many-public-methods + """Represents a class ready to be serialized in Python. + + :param str name: The name of the class. + :param str description: The description of the class. + :param properties: the optional properties of the class. + :type properties: dict(str, str) + """ + + base: Literal["msrest", "dpg", "json"] + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + *, + properties: Optional[List[Property]] = None, + parents: Optional[List["ModelType"]] = None, + discriminated_subtypes: Optional[Dict[str, "ModelType"]] = None, + ) -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.name: str = self.yaml_data["name"] + self.max_properties: Optional[int] = self.yaml_data.get("maxProperties") + self.min_properties: Optional[int] = self.yaml_data.get("minProperties") + self.properties = properties or [] + self.parents = parents or [] + self.discriminated_subtypes = discriminated_subtypes or {} + self.discriminator_value: Optional[str] = self.yaml_data.get("discriminatorValue") + self._created_json_template_representation = False + self._got_polymorphic_subtypes = False + self.internal: bool = self.yaml_data.get("internal", False) + self.snake_case_name: str = self.yaml_data["snakeCaseName"] + self.page_result_model: bool = self.yaml_data.get("pageResultModel", False) + self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") + + @property + def flattened_property(self) -> Optional[Property]: + try: + return next(p for p in self.properties if p.flatten) + except StopIteration: + return None + + @property + def flattened_items(self) -> List[str]: + return [ + item.client_name + for prop in self.properties + if isinstance(prop.type, ModelType) and prop.flatten + for item in prop.type.properties + ] + + @property + def is_form_data(self) -> bool: + return any(p.is_multipart_file_input for p in self.properties) + + @property + def is_xml(self) -> bool: + return self.yaml_data.get("isXml", False) + + @property + def msrest_deserialization_key(self) -> str: + return self.name + + @property + def is_polymorphic(self) -> bool: + return any(p.is_polymorphic for p in self.properties) + + def description(self, *, is_operation_file: bool = False) -> str: + return "" if is_operation_file else self.yaml_data.get("description", self.name) + + def get_declaration(self, value: Any) -> str: + return f"{self.name}()" + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self.name}>" + + @property + def xml_serialization_ctxt(self) -> Optional[str]: + # object schema contains _xml_map, they don't need serialization context + return "" + + @property + def xml_map_content(self) -> Optional[str]: + # This is NOT an error on the super call, we use the serialization context for "xml_map", + # but we don't want to write a serialization context for an object. + return super().xml_serialization_ctxt + + @property + def discriminated_subtypes_name_mapping(self) -> Dict[str, str]: + return {k: v.name for k, v in self.discriminated_subtypes.items()} + + def get_json_template_representation( + self, + *, + optional: bool = True, + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + if self._created_json_template_representation: + return "..." # do this to avoid loop + self._created_json_template_representation = True + if self.discriminated_subtypes: + # we will instead print the discriminated subtypes + self._created_json_template_representation = False + return f'"{self.snake_case_name}"' if self.code_model.for_test else self.snake_case_name + + # don't add additional properties, because there's not really a concept of + # additional properties in the template + representation = { + f'"{prop.wire_name}"': prop.get_json_template_representation( + optional=optional, + client_default_value_declaration=client_default_value_declaration, + description=description, + ) + for prop in [ + p for p in self.properties if not (p.is_discriminator or p.client_name == "additional_properties") + ] + } + if self.discriminator and self.discriminator_value: + representation[f'"{self.discriminator.wire_name}"'] = f'"{self.discriminator_value}"' + + # once we've finished, we want to reset created_json_template_representation to false + # so we can call it again + self._created_json_template_representation = False + optional_keys = [f'"{p.wire_name}"' for p in self.properties if getattr(p, "optional", False)] + return OrderedDict( + sorted( + representation.items(), + key=lambda item: f"{1 if item[0] in optional_keys else 0}{item[0]}", + ) + ) + + def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: + is_polymorphic_subtype = self.discriminator_value and not self.discriminated_subtypes + if self._got_polymorphic_subtypes: + return + self._got_polymorphic_subtypes = True + if self.name not in (m.name for m in polymorphic_subtypes) and is_polymorphic_subtype: + polymorphic_subtypes.append(self) + for discriminated_subtype in self.discriminated_subtypes.values(): + discriminated_subtype.get_polymorphic_subtypes(polymorphic_subtypes) + for property in self.properties: + property.get_polymorphic_subtypes(polymorphic_subtypes) + self._got_polymorphic_subtypes = False + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "ModelType": + raise ValueError( + "You shouldn't call from_yaml for ModelType to avoid recursion. " + "Please initial a blank ModelType, then call .fill_instance_from_yaml on the created type." + ) + + def fill_instance_from_yaml(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + from . import build_type + + self.parents = [cast(ModelType, build_type(bm, code_model)) for bm in yaml_data.get("parents", [])] + properties = [Property.from_yaml(p, code_model) for p in yaml_data["properties"]] + self.properties = _get_properties(self, properties) + # checking to see if this is a polymorphic class + self.discriminated_subtypes = { + k: cast(ModelType, build_type(v, code_model)) + for k, v in self.yaml_data.get("discriminatedSubtypes", {}).items() + } + + @property + def has_readonly_or_constant_property(self) -> bool: + return any(x.readonly or x.constant or x.visibility == ["read"] for x in self.properties) + + @property + def discriminator(self) -> Optional[Property]: + try: + return next(p for p in self.properties if p.is_discriminator) + except StopIteration: + return None + + @property + def discriminator_property(self) -> Optional[Property]: + try: + return next( + p + for p in self.properties + if p.is_discriminator and isinstance(p.type, ConstantType) and p.type.value == self.discriminator_value + ) + except StopIteration: + return None + + @property + def pylint_disable(self) -> str: + retval: str = "" + if len(self.properties) > 10: + retval = add_to_pylint_disable(retval, "too-many-instance-attributes") + if len(self.name) > NAME_LENGTH_LIMIT: + retval = add_to_pylint_disable(retval, "name-too-long") + return retval + + @property + def init_pylint_disable(self) -> str: + retval: str = "" + if len(self.properties) > 23: + retval = add_to_pylint_disable(retval, "too-many-locals") + return retval + + +class JSONModelType(ModelType): + base = "json" + + def type_annotation(self, **kwargs: Any) -> str: + return "ET.Element" if self.is_xml else "JSON" + + @property + def serialization_type(self) -> str: + return "object" + + def docstring_type(self, **kwargs: Any) -> str: + return "ET.Element" if self.is_xml else "JSON" + + def docstring_text(self, **kwargs: Any) -> str: + return "XML Element" if self.is_xml else "JSON object" + + @property + def instance_check_template(self) -> str: + return "isinstance({}, MutableMapping)" + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) + file_import.define_mutable_mapping_type() + if self.is_xml: + file_import.add_submodule_import("xml.etree", "ElementTree", ImportType.STDLIB, alias="ET") + return file_import + + +class GeneratedModelType(ModelType): # pylint: disable=abstract-method + def type_annotation(self, **kwargs: Any) -> str: + is_operation_file = kwargs.pop("is_operation_file", False) + skip_quote = kwargs.get("skip_quote", False) + module_name = "_models." if kwargs.get("need_module_name", True) else "" + file_name = f"{self.code_model.models_filename}." if self.internal else "" + retval = module_name + file_name + self.name + return retval if is_operation_file or skip_quote else f'"{retval}"' + + def docstring_type(self, **kwargs: Any) -> str: + return f"~{self.code_model.namespace}.models.{self.type_annotation(need_module_name=False, skip_quote=True)}" + + def docstring_text(self, **kwargs: Any) -> str: + return self.name + + @property + def type_description(self) -> str: + return self.name + + def imports(self, **kwargs: Any) -> FileImport: + file_import = super().imports(**kwargs) + relative_path = kwargs.pop("relative_path", None) + if relative_path: + # add import for models in operations or _types file + file_import.add_submodule_import( + relative_path, + "models", + ImportType.LOCAL, + alias="_models", + typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), + ) + if self.is_form_data: + file_import.add_submodule_import( + relative_path, + "_model_base", + ImportType.LOCAL, + typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), + ) + return file_import + + +class MsrestModelType(GeneratedModelType): + base = "msrest" + + @property + def serialization_type(self) -> str: + return self.type_annotation(skip_quote=True) if self.internal else self.name + + @property + def instance_check_template(self) -> str: + return "isinstance({}, msrest.Model)" + + def imports(self, **kwargs: Any) -> FileImport: + file_import = super().imports(**kwargs) + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) + return file_import + + +class DPGModelType(GeneratedModelType): + base = "dpg" + + @property + def serialization_type(self) -> str: + return ( + self.type_annotation(skip_quote=True) + if self.internal + else self.type_annotation(need_module_name=False, skip_quote=True) + ) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, _model_base.Model)" + + def imports(self, **kwargs: Any) -> FileImport: + file_import = super().imports(**kwargs) + if self.flattened_property: + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) + return file_import diff --git a/packages/autorest.python/generator/pygen/codegen/models/operation.py b/packages/autorest.python/generator/pygen/codegen/models/operation.py new file mode 100644 index 00000000000..c121649ca88 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/operation.py @@ -0,0 +1,510 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from itertools import chain +from typing import ( + Dict, + List, + Any, + Optional, + Union, + TYPE_CHECKING, + Generic, + TypeVar, + cast, + Sequence, +) + +from .request_builder_parameter import RequestBuilderParameter + +from .utils import OrderedSet, add_to_pylint_disable, NAME_LENGTH_LIMIT +from .base_builder import BaseBuilder +from .imports import FileImport, ImportType, TypingSection +from .response import ( + Response, + PagingResponse, + LROResponse, + LROPagingResponse, + get_response, +) +from .parameter import ( + BodyParameter, + Parameter, + ParameterLocation, +) +from .parameter_list import ParameterList +from .model_type import ModelType +from .base import BaseType +from .request_builder import OverloadedRequestBuilder, RequestBuilder + +if TYPE_CHECKING: + from .code_model import CodeModel + from .client import Client + from . import OperationType + +ResponseType = TypeVar( + "ResponseType", + bound=Union[Response, PagingResponse, LROResponse, LROPagingResponse], +) + + +def is_internal(target: Optional[BaseType]) -> bool: + return isinstance(target, ModelType) and target.base == "dpg" and target.internal + + +class OperationBase( # pylint: disable=too-many-public-methods,too-many-instance-attributes + Generic[ResponseType], BaseBuilder[ParameterList, List["Operation"]] +): + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + client: "Client", + name: str, + request_builder: Union[RequestBuilder, OverloadedRequestBuilder], + parameters: ParameterList, + responses: List[ResponseType], + exceptions: List[Response], + *, + overloads: Optional[List["Operation"]] = None, + ) -> None: + super().__init__( + code_model=code_model, + client=client, + yaml_data=yaml_data, + name=name, + parameters=parameters, + overloads=overloads, + ) + self.overloads: List["Operation"] = overloads or [] + self.responses = responses + self.request_builder = request_builder + self.deprecated = False + self.exceptions = exceptions + self.is_lro_initial_operation: bool = self.yaml_data.get("isLroInitialOperation", False) + self.include_documentation: bool = not self.is_lro_initial_operation + self.internal: bool = self.yaml_data.get("internal", False) + if self.internal: + self.name = "_" + self.name + self.has_etag: bool = self.yaml_data.get("hasEtag", False) + self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") + + @property + def has_form_data_body(self): + return self.parameters.has_form_data_body + + @property + def expose_stream_keyword(self) -> bool: + return self.yaml_data.get("exposeStreamKeyword", False) + + @property + def operation_type(self) -> str: + return "operation" + + @property + def has_optional_return_type(self) -> bool: + """Has optional return type if there are multiple successful response types where some have + bodies and some are None + """ + # means if we have at least one successful response with a body and one without + successful_response_with_body = any(r for r in self.responses if r.type) + successful_response_without_body = any(r for r in self.responses if not r.type) + return successful_response_with_body and successful_response_without_body + + def response_type_annotation(self, **kwargs) -> str: + if self.code_model.options["head_as_boolean"] and self.request_builder.method.lower() == "head": + return "bool" + response_type_annotations: OrderedSet[str] = { + response.type_annotation(**kwargs): None for response in self.responses if response.type + } + response_str = ", ".join(response_type_annotations.keys()) + if len(response_type_annotations) > 1: + return f"Union[{response_str}]" + if self.has_optional_return_type: + return f"Optional[{response_str}]" + if self.responses: + return self.responses[0].type_annotation(**kwargs) + return "None" + + @property + def pylint_disable(self) -> str: + retval: str = "" + if self.response_type_annotation(async_mode=False) == "None": + # doesn't matter if it's async or not + retval = add_to_pylint_disable(retval, "inconsistent-return-statements") + try: + if any(is_internal(r.type) for r in self.responses) or is_internal(self.parameters.body_parameter.type): + retval = add_to_pylint_disable(retval, "protected-access") + except ValueError: + pass + if len(self.name) > NAME_LENGTH_LIMIT: + retval = add_to_pylint_disable(retval, "name-too-long") + return retval + + def cls_type_annotation(self, *, async_mode: bool) -> str: + if self.request_builder.method.lower() == "head" and self.code_model.options["head_as_boolean"]: + return "ClsType[None]" + return f"ClsType[{self.response_type_annotation(async_mode=async_mode)}]" + + def _response_docstring_helper(self, attr_name: str, **kwargs: Any) -> str: + responses_with_body = [r for r in self.responses if r.type] + if self.request_builder.method.lower() == "head" and self.code_model.options["head_as_boolean"]: + return "bool" + if responses_with_body: + response_docstring_values: OrderedSet[str] = { + getattr(response, attr_name)(**kwargs): None for response in responses_with_body + } + retval = " or ".join(response_docstring_values.keys()) + if self.has_optional_return_type: + retval += " or None" + return retval + if self.responses: + return getattr(self.responses[0], attr_name)(**kwargs) + return "None" + + def response_docstring_text(self, **kwargs) -> str: + retval = self._response_docstring_helper("docstring_text", **kwargs) + if not self.code_model.options["version_tolerant"]: + retval += " or the result of cls(response)" + if self.code_model.options["models_mode"] == "dpg" and any( + isinstance(r.type, ModelType) for r in self.responses + ): + r = next(r for r in self.responses if isinstance(r.type, ModelType)) + item_type = getattr(r, "item_type", getattr(r, "type")) + if item_type: + type_name = item_type.docstring_text(**kwargs) + retval += f". The {type_name} is compatible with MutableMapping" + return retval + + def response_docstring_type(self, **kwargs) -> str: + return self._response_docstring_helper("docstring_type", **kwargs) + + @property + def has_response_body(self) -> bool: + """Tell if at least one response has a body.""" + return any(response.type for response in self.responses) + + @property + def any_response_has_headers(self) -> bool: + return any(response.headers for response in self.responses) + + @property + def default_error_deserialization(self) -> Optional[str]: + default_exceptions = [e for e in self.exceptions if "default" in e.status_codes and e.type] + if not default_exceptions: + return None + excep_schema = default_exceptions[0].type + if isinstance(excep_schema, ModelType): + return excep_schema.type_annotation(skip_quote=True) + # in this case, it's just an AnyType + return "'object'" + + @property + def non_default_errors(self) -> List[Response]: + return [e for e in self.exceptions if "default" not in e.status_codes] + + @property + def non_default_error_status_codes(self) -> List[Union[str, int]]: + """Actually returns all of the status codes from exceptions (besides default)""" + return list(chain.from_iterable([error.status_codes for error in self.non_default_errors])) + + def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: # pylint: disable=unused-argument + file_import = FileImport(self.code_model) + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) + + response_types = [r.type_annotation(async_mode=async_mode, operation=self) for r in self.responses if r.type] + if len(set(response_types)) > 1: + file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) + if self.added_on: + file_import.add_submodule_import( + f"{'.' if async_mode else ''}.._validation", + "api_version_validation", + ImportType.LOCAL, + ) + return file_import + + def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport: + if self.abstract: + return FileImport(self.code_model) + file_import = self._imports_shared(async_mode, **kwargs) + for param in self.parameters.method: + file_import.merge( + param.imports_for_multiapi( + async_mode, + operation=self, + **kwargs, + ) + ) + for response in self.responses: + file_import.merge(response.imports_for_multiapi(async_mode=async_mode, operation=self, **kwargs)) + if self.code_model.options["models_mode"]: + for exception in self.exceptions: + file_import.merge(exception.imports_for_multiapi(async_mode=async_mode, operation=self, **kwargs)) + return file_import + + @staticmethod + def has_kwargs_to_pop_with_default( + kwargs_to_pop: List[ + Union[ + Parameter, + RequestBuilderParameter, + BodyParameter, + ] + ], + location: ParameterLocation, + ) -> bool: + return any( + (kwarg.client_default_value or kwarg.optional) and kwarg.location == location for kwarg in kwargs_to_pop + ) + + @property + def need_validation(self) -> bool: + """Whether we need parameter / operation validation. For API version.""" + return bool(self.added_on) or any(p for p in self.parameters if p.added_on) + + def get_request_builder_import( + self, + request_builder: Union[RequestBuilder, OverloadedRequestBuilder], + async_mode: bool, + ) -> FileImport: + """Helper method to get a request builder import.""" + file_import = FileImport(self.code_model) + if self.code_model.options["builders_visibility"] != "embedded": + group_name = request_builder.group_name + rest_import_path = "..." if async_mode else ".." + if group_name: + file_import.add_submodule_import( + f"{rest_import_path}{self.code_model.rest_layer_name}", + group_name, + import_type=ImportType.LOCAL, + alias=f"rest_{group_name}", + ) + else: + file_import.add_submodule_import( + rest_import_path, + self.code_model.rest_layer_name, + import_type=ImportType.LOCAL, + alias="rest", + ) + if self.code_model.options["builders_visibility"] == "embedded" and async_mode: + file_import.add_submodule_import( + f"...{self.code_model.operations_folder_name}.{self.filename}", + request_builder.name, + import_type=ImportType.LOCAL, + ) + return file_import + + def imports( # pylint: disable=too-many-branches, disable=too-many-statements + self, async_mode: bool, **kwargs: Any + ) -> FileImport: + if self.abstract: + return FileImport(self.code_model) + file_import = self._imports_shared(async_mode, **kwargs) + + for param in self.parameters.method: + file_import.merge( + param.imports( + async_mode, + operation=self, + **kwargs, + ) + ) + for response in self.responses: + file_import.merge(response.imports(async_mode=async_mode, operation=self, **kwargs)) + if self.code_model.options["models_mode"]: + for exception in self.exceptions: + file_import.merge(exception.imports(async_mode=async_mode, **kwargs)) + + if self.parameters.has_body and self.parameters.body_parameter.flattened: + file_import.merge(self.parameters.body_parameter.type.imports(operation=self, **kwargs)) + if not async_mode: + for param in self.parameters.headers: + if param.wire_name.lower() == "repeatability-request-id": + file_import.add_import("uuid", ImportType.STDLIB) + elif param.wire_name.lower() == "repeatability-first-sent": + file_import.add_import("datetime", ImportType.STDLIB) + + # Exceptions + errors = [ + "map_error", + "HttpResponseError", + "ClientAuthenticationError", + "ResourceNotFoundError", + "ResourceExistsError", + "ResourceNotModifiedError", + ] + for error in errors: + file_import.add_submodule_import("exceptions", error, ImportType.SDKCORE) + if self.code_model.options["azure_arm"]: + file_import.add_submodule_import("azure.mgmt.core.exceptions", "ARMErrorFormat", ImportType.SDKCORE) + file_import.add_submodule_import( + "typing", + "Type", + ImportType.STDLIB, + ) + file_import.add_mutable_mapping_import() + if self.non_default_error_status_codes: + file_import.add_submodule_import( + "typing", + "cast", + ImportType.STDLIB, + ) + + if self.has_kwargs_to_pop_with_default( + self.parameters.kwargs_to_pop, ParameterLocation.HEADER # type: ignore + ) or self.has_kwargs_to_pop_with_default( + self.parameters.kwargs_to_pop, ParameterLocation.QUERY # type: ignore + ): + file_import.add_submodule_import( + "utils", + "case_insensitive_dict", + ImportType.SDKCORE, + ) + if self.deprecated: + file_import.add_import("warnings", ImportType.STDLIB) + + relative_path = "..." if async_mode else ".." + if self.has_etag: + file_import.add_submodule_import( + "exceptions", + "ResourceModifiedError", + ImportType.SDKCORE, + ) + if not async_mode: + file_import.add_submodule_import(f"{relative_path}_vendor", "prep_if_match", ImportType.LOCAL) + file_import.add_submodule_import(f"{relative_path}_vendor", "prep_if_none_match", ImportType.LOCAL) + if async_mode: + file_import.add_submodule_import( + "rest", + "AsyncHttpResponse", + ImportType.SDKCORE, + ) + else: + file_import.add_submodule_import( + "rest", + "HttpResponse", + ImportType.SDKCORE, + ) + if self.code_model.options["builders_visibility"] == "embedded" and not async_mode: + file_import.merge(self.request_builder.imports()) + file_import.add_submodule_import( + f"{'' if self.code_model.is_azure_flavor else 'runtime.'}pipeline", + "PipelineResponse", + ImportType.SDKCORE, + ) + file_import.add_submodule_import("rest", "HttpRequest", ImportType.SDKCORE) + file_import.add_submodule_import("typing", "Callable", ImportType.STDLIB, TypingSection.CONDITIONAL) + file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB, TypingSection.CONDITIONAL) + file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB, TypingSection.CONDITIONAL) + file_import.add_submodule_import("typing", "TypeVar", ImportType.STDLIB, TypingSection.CONDITIONAL) + if self.code_model.options["tracing"] and self.want_tracing and not async_mode: + file_import.add_submodule_import( + "azure.core.tracing.decorator", + "distributed_trace", + ImportType.SDKCORE, + ) + file_import.merge(self.get_request_builder_import(self.request_builder, async_mode)) + if self.overloads: + file_import.add_submodule_import("typing", "overload", ImportType.STDLIB) + if self.non_default_errors and self.code_model.options["models_mode"] == "dpg": + file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + return file_import + + def get_response_from_status(self, status_code: Optional[Union[str, int]]) -> ResponseType: + try: + return next(r for r in self.responses if status_code in r.status_codes) + except StopIteration as exc: + raise ValueError(f"Incorrect status code {status_code}, operation {self.name}") from exc + + @property + def success_status_codes(self) -> Sequence[Union[str, int]]: + """The list of all successfull status code.""" + return sorted([code for response in self.responses for code in response.status_codes]) + + @property + def filename(self) -> str: + basename = self.group_name + if basename == "": + # in a mixin + basename = self.code_model.clients[0].legacy_filename + + if basename == "operations" or self.code_model.options["combine_operation_files"]: + return "_operations" + return f"_{basename}_operations" + + @property + def has_stream_response(self) -> bool: + return any(r.is_stream_response for r in self.responses) + + @classmethod + def get_request_builder(cls, yaml_data: Dict[str, Any], client: "Client"): + return client.lookup_request_builder(id(yaml_data)) + + @classmethod + def from_yaml( + cls, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + client: "Client", + ): + name = yaml_data["name"] + request_builder = cls.get_request_builder(yaml_data, client) + responses = [cast(ResponseType, get_response(r, code_model)) for r in yaml_data["responses"]] + exceptions = [Response.from_yaml(e, code_model) for e in yaml_data["exceptions"]] + parameter_list = ParameterList.from_yaml(yaml_data, code_model) + overloads = [cls.from_yaml(overload, code_model, client) for overload in yaml_data.get("overloads", [])] + + return cls( + yaml_data=yaml_data, + code_model=code_model, + client=client, + request_builder=request_builder, + name=name, + parameters=parameter_list, + overloads=overloads, + responses=responses, + exceptions=exceptions, + ) + + +class Operation(OperationBase[Response]): + def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: + file_import = super().imports(async_mode, **kwargs) + if self.abstract: + return file_import + if async_mode and self.code_model.options["tracing"] and self.want_tracing: + file_import.add_submodule_import( + "azure.core.tracing.decorator_async", + "distributed_trace_async", + ImportType.SDKCORE, + ) + if self.has_response_body and not self.has_optional_return_type and not self.code_model.options["models_mode"]: + file_import.add_submodule_import("typing", "cast", ImportType.STDLIB) + relative_path = "..." if async_mode else ".." + if self.code_model.options["models_mode"] == "dpg": + if self.parameters.has_body: + if not self.has_form_data_body: + file_import.add_submodule_import( + f"{relative_path}_model_base", + "SdkJSONEncoder", + ImportType.LOCAL, + ) + file_import.add_import("json", ImportType.STDLIB) + if self.default_error_deserialization or any(r.type for r in self.responses): + file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + + return file_import + + +def get_operation(yaml_data: Dict[str, Any], code_model: "CodeModel", client: "Client") -> "OperationType": + if yaml_data["discriminator"] == "lropaging": + from .lro_paging_operation import LROPagingOperation as OperationCls + elif yaml_data["discriminator"] == "lro": + from .lro_operation import LROOperation as OperationCls # type: ignore + elif yaml_data["discriminator"] == "paging": + from .paging_operation import PagingOperation as OperationCls # type: ignore + else: + from . import Operation as OperationCls # type: ignore + return OperationCls.from_yaml(yaml_data, code_model, client) diff --git a/packages/autorest.python/generator/pygen/codegen/models/operation_group.py b/packages/autorest.python/generator/pygen/codegen/models/operation_group.py new file mode 100644 index 00000000000..23b6f4daaba --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/operation_group.py @@ -0,0 +1,184 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Dict, List, Any, TYPE_CHECKING + +from .utils import OrderedSet + +from .base import BaseModel +from .operation import get_operation +from .imports import FileImport, ImportType, TypingSection +from .utils import add_to_pylint_disable, NAME_LENGTH_LIMIT +from .lro_operation import LROOperation +from .lro_paging_operation import LROPagingOperation + +if TYPE_CHECKING: + from .code_model import CodeModel + from .client import Client + from . import OperationType + + +class OperationGroup(BaseModel): + """Represent an operation group.""" + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + client: "Client", + operations: List["OperationType"], + api_versions: List[str], + ) -> None: + super().__init__(yaml_data, code_model) + self.client = client + self.class_name: str = yaml_data["className"] + self.identify_name: str = yaml_data["identifyName"] + self.property_name: str = yaml_data["propertyName"] + self.operations = operations + self.api_versions = api_versions + self.operation_groups: List[OperationGroup] = [] + if self.code_model.options["show_operations"]: + self.operation_groups = [ + OperationGroup.from_yaml(op_group, code_model, client) + for op_group in self.yaml_data.get("operationGroups", []) + ] + self.link_lro_initial_operations() + + @property + def has_abstract_operations(self) -> bool: + return any(o for o in self.operations if o.abstract) or any( + operation_group.has_abstract_operations for operation_group in self.operation_groups + ) + + @property + def has_non_abstract_operations(self) -> bool: + return any(o for o in self.operations if not o.abstract) or any( + operation_group.has_non_abstract_operations for operation_group in self.operation_groups + ) + + @property + def base_class(self) -> str: + base_classes: List[str] = [] + if self.is_mixin: + base_classes.append(f"{self.client.name}MixinABC") + return ", ".join(base_classes) + + def imports_for_multiapi(self, async_mode: bool) -> FileImport: + file_import = FileImport(self.code_model) + relative_path = ".." if async_mode else "." + for operation in self.operations: + file_import.merge(operation.imports_for_multiapi(async_mode, relative_path=relative_path)) + if (self.code_model.model_types or self.code_model.enums) and self.code_model.options[ + "models_mode" + ] == "msrest": + file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") + return file_import + + @property + def pylint_disable(self) -> str: + retval: str = "" + if self.has_abstract_operations: + retval = add_to_pylint_disable(retval, "abstract-class-instantiated") + if len(self.operations) > 20: + retval = add_to_pylint_disable(retval, "too-many-public-methods") + if len(self.class_name) > NAME_LENGTH_LIMIT: + retval = add_to_pylint_disable(retval, "name-too-long") + if len(self.operation_groups) > 6: + retval = add_to_pylint_disable(retval, "too-many-instance-attributes") + return retval + + @property + def need_validation(self) -> bool: + """Whether any of its operations need validation""" + return any(o for o in self.operations if o.need_validation) + + def imports(self, async_mode: bool) -> FileImport: + file_import = FileImport(self.code_model) + + relative_path = ("..." if async_mode else "..") + ("." if self.client.is_subclient else "") + for operation in self.operations: + file_import.merge(operation.imports(async_mode, relative_path=relative_path)) + if not self.code_model.options["combine_operation_files"]: + for og in self.operation_groups: + file_import.add_submodule_import( + ".", + og.class_name, + ImportType.LOCAL, + ) + # for multiapi + if ( + (self.code_model.public_model_types) + and self.code_model.options["models_mode"] == "msrest" + and not self.is_mixin + ): + file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") + if self.code_model.need_mixin_abc: + file_import.add_submodule_import(".._vendor", f"{self.client.name}MixinABC", ImportType.LOCAL) + if self.has_abstract_operations: + file_import.add_submodule_import(".._vendor", "raise_if_not_implemented", ImportType.LOCAL) + if all(o.abstract for o in self.operations): + return file_import + file_import.add_submodule_import("typing", "TypeVar", ImportType.STDLIB, TypingSection.CONDITIONAL) + file_import.define_mypy_type("T", "TypeVar('T')") + type_value = "Optional[Callable[[PipelineResponse[HttpRequest, {}HttpResponse], T, Dict[str, Any]], Any]]" + file_import.define_mypy_type("ClsType", type_value.format(""), type_value.format("Async")) + return file_import + + @property + def filename(self) -> str: + return self.operations[0].filename + + @property + def is_mixin(self) -> bool: + """The operation group with no name is the direct client methods.""" + return self.identify_name == "" + + def link_lro_initial_operations(self) -> None: + """Link each LRO operation to its initial operation""" + for operation_group in self.operation_groups: + for operation in operation_group.operations: + if isinstance(operation, (LROOperation, LROPagingOperation)): + operation.initial_operation = self.lookup_operation(id(operation.yaml_data["initialOperation"])) + + def lookup_operation(self, operation_id: int) -> "OperationType": + try: + return next(o for og in self.operation_groups for o in og.operations if id(o.yaml_data) == operation_id) + except StopIteration as exc: + raise KeyError(f"No operation with id {operation_id} found.") from exc + + @property + def lro_operations(self) -> List["OperationType"]: + return [operation for operation in self.operations if operation.operation_type in ("lro", "lropaging")] + [ + operation for operation_group in self.operation_groups for operation in operation_group.lro_operations + ] + + @property + def has_operations(self) -> bool: + return any(operation_group.has_operations for operation_group in self.operation_groups) or bool(self.operations) + + @property + def has_form_data_body(self) -> bool: + operations = self.operations + [o for og in self.operation_groups for o in og.operations] + return any(operation.has_form_data_body for operation in operations) + + @classmethod + def from_yaml( + cls, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + client: "Client", + ) -> "OperationGroup": + operations = [get_operation(o, code_model, client) for o in yaml_data["operations"]] + api_versions: OrderedSet[str] = {} + for operation in operations: + for api_version in operation.api_versions: + api_versions[api_version] = None + return cls( + yaml_data=yaml_data, + code_model=code_model, + client=client, + operations=operations, + api_versions=list(api_versions.keys()), + ) diff --git a/packages/autorest.python/generator/pygen/codegen/models/paging_operation.py b/packages/autorest.python/generator/pygen/codegen/models/paging_operation.py new file mode 100644 index 00000000000..554b72d4431 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/paging_operation.py @@ -0,0 +1,156 @@ +# pylint: disable=multiple-statements +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Dict, List, Any, Optional, Union, TYPE_CHECKING, cast, TypeVar + +from .operation import Operation, OperationBase +from .response import PagingResponse, LROPagingResponse, Response +from .request_builder import ( + OverloadedRequestBuilder, + RequestBuilder, + get_request_builder, +) +from .imports import ImportType, FileImport, TypingSection +from .parameter_list import ParameterList +from .model_type import ModelType +from .list_type import ListType + +if TYPE_CHECKING: + from .code_model import CodeModel + from .client import Client + +PagingResponseType = TypeVar("PagingResponseType", bound=Union[PagingResponse, LROPagingResponse]) + + +class PagingOperationBase(OperationBase[PagingResponseType]): + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + client: "Client", + name: str, + request_builder: RequestBuilder, + parameters: ParameterList, + responses: List[PagingResponseType], + exceptions: List[Response], + *, + overloads: Optional[List[Operation]] = None, + override_success_response_to_200: bool = False, + ) -> None: + super().__init__( + code_model=code_model, + client=client, + yaml_data=yaml_data, + name=name, + request_builder=request_builder, + parameters=parameters, + responses=responses, + exceptions=exceptions, + overloads=overloads, + ) + self.next_request_builder: Optional[Union[RequestBuilder, OverloadedRequestBuilder]] = ( + get_request_builder(self.yaml_data["nextOperation"], code_model, client) + if self.yaml_data.get("nextOperation") + else None + ) + self.override_success_response_to_200 = override_success_response_to_200 + self.pager_sync: str = yaml_data.get("pagerSync") or f"{self.code_model.core_library}.paging.ItemPaged" + self.pager_async: str = yaml_data.get("pagerAsync") or f"{self.code_model.core_library}.paging.AsyncItemPaged" + + def _get_attr_name(self, wire_name: str) -> str: + response_type = self.responses[0].type + if not response_type: + raise ValueError(f"Can't find a matching property in response for {wire_name}") + if response_type.type == "list": + response_type = cast(ListType, response_type).element_type + try: + return next(p.client_name for p in cast(ModelType, response_type).properties if p.wire_name == wire_name) + except StopIteration as exc: + raise ValueError(f"Can't find a matching property in response for {wire_name}") from exc + + def get_pager(self, async_mode: bool) -> str: + return self.responses[0].get_pager(async_mode) + + @property + def continuation_token_name(self) -> Optional[str]: + wire_name = self.yaml_data.get("continuationTokenName") + if not wire_name: + # That's an ok scenario, it just means no next page possible + return None + if self.code_model.options["models_mode"] == "msrest": + return self._get_attr_name(wire_name) + return wire_name + + @property + def item_name(self) -> str: + wire_name = self.yaml_data["itemName"] + if self.code_model.options["models_mode"] == "msrest": + # we don't use the paging model for dpg + return self._get_attr_name(wire_name) + return wire_name + + @property + def item_type(self) -> ModelType: + try: + item_type_yaml = self.yaml_data["itemType"] + except KeyError as e: + raise ValueError("Only call this for DPG paging model deserialization") from e + return cast(ModelType, self.code_model.types_map[id(item_type_yaml)]) + + @property + def operation_type(self) -> str: + return "paging" + + def cls_type_annotation(self, *, async_mode: bool) -> str: + return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode)}]" + + def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: + file_import = super()._imports_shared(async_mode, **kwargs) + if async_mode: + file_import.add_submodule_import("typing", "AsyncIterable", ImportType.STDLIB, TypingSection.CONDITIONAL) + else: + file_import.add_submodule_import("typing", "Iterable", ImportType.STDLIB, TypingSection.CONDITIONAL) + if ( + self.next_request_builder + and self.code_model.options["builders_visibility"] == "embedded" + and not async_mode + ): + file_import.merge(self.next_request_builder.imports()) + return file_import + + @property + def has_optional_return_type(self) -> bool: + return False + + def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: + if self.abstract: + return FileImport(self.code_model) + file_import = self._imports_shared(async_mode, **kwargs) + file_import.merge(super().imports(async_mode, **kwargs)) + if self.code_model.options["tracing"] and self.want_tracing: + file_import.add_submodule_import( + "azure.core.tracing.decorator", + "distributed_trace", + ImportType.SDKCORE, + ) + if self.next_request_builder: + file_import.merge(self.get_request_builder_import(self.next_request_builder, async_mode)) + elif any(p.is_api_version for p in self.client.parameters): + file_import.add_import("urllib.parse", ImportType.STDLIB) + file_import.add_submodule_import( + "utils", + "case_insensitive_dict", + ImportType.SDKCORE, + ) + if self.code_model.options["models_mode"] == "dpg": + relative_path = "..." if async_mode else ".." + file_import.merge(self.item_type.imports(**kwargs)) + if self.default_error_deserialization or any(r.type for r in self.responses): + file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) + return file_import + + +class PagingOperation(PagingOperationBase[PagingResponse]): ... diff --git a/packages/autorest.python/generator/pygen/codegen/models/parameter.py b/packages/autorest.python/generator/pygen/codegen/models/parameter.py new file mode 100644 index 00000000000..296c8386008 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/parameter.py @@ -0,0 +1,402 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import abc +from enum import Enum + +from typing import ( + Dict, + Any, + TYPE_CHECKING, + List, + Optional, + TypeVar, + Union, +) + +from .imports import FileImport, ImportType, TypingSection +from .base import BaseModel +from .base import BaseType +from .constant_type import ConstantType +from .utils import add_to_description +from .combined_type import CombinedType +from .model_type import JSONModelType + +if TYPE_CHECKING: + from .code_model import CodeModel + from .request_builder_parameter import RequestBuilderBodyParameter + + +class ParameterLocation(str, Enum): + HEADER = "header" + PATH = "path" + ENDPOINT_PATH = "endpointPath" + QUERY = "query" + BODY = "body" + OTHER = "other" + + +class ParameterMethodLocation(str, Enum): + POSITIONAL = "positional" + KEYWORD_ONLY = "keywordOnly" + KWARG = "kwarg" + + +class ParameterDelimeter(str, Enum): + SPACE = "space" + PIPE = "pipe" + TAB = "tab" + COMMA = "comma" + + +class _ParameterBase(BaseModel, abc.ABC): # pylint: disable=too-many-instance-attributes + """Base class for all parameters""" + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + type: BaseType, + ) -> None: + super().__init__(yaml_data, code_model) + self.wire_name: str = yaml_data.get("wireName", "") + self.client_name: str = self.yaml_data["clientName"] + self.optional: bool = self.yaml_data["optional"] + self.implementation: str = yaml_data.get("implementation", None) + self.location: ParameterLocation = self.yaml_data["location"] + self.client_default_value = self.yaml_data.get("clientDefaultValue", None) + self.in_docstring = self.yaml_data.get("inDocstring", True) + self.type = type + if self.client_default_value is None: + self.client_default_value = self.type.client_default_value + # name of grouper if it is grouped by another parameter + self.grouped_by: Optional[str] = self.yaml_data.get("groupedBy") + # property matching property name to parameter name for grouping params + # and flattened body params + self.property_to_parameter_name: Optional[Dict[str, str]] = self.yaml_data.get("propertyToParameterName") + self.flattened: bool = self.yaml_data.get("flattened", False) + self.in_flattened_body: bool = self.yaml_data.get("inFlattenedBody", False) + self.grouper: bool = self.yaml_data.get("grouper", False) + self.check_client_input: bool = self.yaml_data.get("checkClientInput", False) + self.added_on: Optional[str] = self.yaml_data.get("addedOn") + self.is_api_version: bool = self.yaml_data.get("isApiVersion", False) + self.in_overload: bool = self.yaml_data.get("inOverload", False) + self.default_to_unset_sentinel: bool = self.yaml_data.get("defaultToUnsetSentinel", False) + self.hide_in_method: bool = self.yaml_data.get("hideInMethod", False) + + def get_declaration(self, value: Any = None) -> Any: + return self.type.get_declaration(value) + + @property + def hide_in_operation_signature(self) -> bool: + return False + + @property + def constant(self) -> bool: + """Returns whether a parameter is a constant or not. + Checking to see if it's required, because if not, we don't consider it + a constant because it can have a value of None. + """ + return (not self.optional or self.is_api_version) and isinstance(self.type, ConstantType) + + @property + def description(self) -> str: + base_description = self.yaml_data["description"] + type_description = self.type.description(is_operation_file=True) + if type_description: + base_description = add_to_description(base_description, type_description) + if self.optional and isinstance(self.type, ConstantType): + base_description = add_to_description( + base_description, + f"Known values are {self.get_declaration()} and None.", + ) + if not (self.optional or self.client_default_value): + base_description = add_to_description(base_description, "Required.") + if self.client_default_value is not None: + base_description = add_to_description( + base_description, + f"Default value is {self.client_default_value_declaration}.", + ) + if self.optional and self.client_default_value is None: + base_description = add_to_description( + base_description, + f"Default value is {self.client_default_value_declaration}.", + ) + if self.constant: + base_description = add_to_description( + base_description, + "Note that overriding this default value may result in unsupported behavior.", + ) + return base_description + + @property + def client_default_value_declaration(self): + """Declaration of parameter's client default value""" + if self.client_default_value is None: + return None + return self.get_declaration(self.client_default_value) + + def type_annotation(self, **kwargs: Any) -> str: + kwargs["is_operation_file"] = True + # special logic for api-version parameter + if self.is_api_version: + type_annot = "str" + else: + type_annot = self.type.type_annotation(**kwargs) + if self.optional and self.client_default_value is None: + return f"Optional[{type_annot}]" + return type_annot + + def docstring_text(self, **kwargs: Any) -> str: + return self.type.docstring_text(**kwargs) + + def docstring_type(self, **kwargs: Any) -> str: + return self.type.docstring_type(**kwargs) + + @property + def serialization_type(self) -> str: + return self.type.serialization_type + + def _imports_shared(self, async_mode: bool, **_: Any) -> FileImport: + file_import = FileImport(self.code_model) + if self.optional and self.client_default_value is None: + file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) + if self.added_on and self.implementation != "Client": + file_import.add_submodule_import( + f"{'.' if async_mode else ''}.._validation", + "api_version_validation", + ImportType.LOCAL, + ) + if isinstance(self.type, CombinedType) and self.type.name: + file_import.add_submodule_import( + "..." if async_mode else "..", + "_types", + ImportType.LOCAL, + TypingSection.TYPING, + ) + return file_import + + def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: + file_import = self._imports_shared(async_mode, **kwargs) + # special logic for api-version parameter + if not self.is_api_version: + file_import.merge(self.type.imports(async_mode=async_mode, **kwargs)) + if self.default_to_unset_sentinel: + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) + file_import.define_mypy_type( + "_Unset: Any", + "object()", + ) + return file_import + + def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport: + file_import = self._imports_shared(async_mode, **kwargs) + file_import.merge(self.type.imports_for_multiapi(async_mode=async_mode, **kwargs)) + return file_import + + @property + def method_location(self) -> ParameterMethodLocation: + raise NotImplementedError("Please implement in children") + + @property + def description_keyword(self) -> str: + return "param" if self.method_location == ParameterMethodLocation.POSITIONAL else "keyword" + + @property + def docstring_type_keyword(self) -> str: + return "type" if self.method_location == ParameterMethodLocation.POSITIONAL else "paramtype" + + @property + @abc.abstractmethod + def in_method_signature(self) -> bool: ... + + def method_signature(self, async_mode: bool) -> str: + type_annot = self.type_annotation(async_mode=async_mode) + if self.client_default_value is not None or self.optional: + return f"{self.client_name}: {type_annot} = {self.client_default_value_declaration}," + if self.default_to_unset_sentinel: + return f"{self.client_name}: {type_annot} = _Unset," + return f"{self.client_name}: {type_annot}," + + +class BodyParameter(_ParameterBase): + """Body parameter.""" + + @property + def entries(self) -> List["BodyParameter"]: + return [BodyParameter.from_yaml(e, self.code_model) for e in self.yaml_data.get("entries", [])] + + @property + def is_form_data(self) -> bool: + # hacky, but rn in legacy, there is no formdata model type, it's just a dict + # with all of the entries splatted out + return self.type.is_form_data or bool(self.entries) + + @property + def is_partial_body(self) -> bool: + """Whether it's part of a bigger body parameter, i.e. a MultipartBodyParameter""" + return self.yaml_data.get("isPartialBody", False) + + @property + def method_location(self) -> ParameterMethodLocation: + return ParameterMethodLocation.KWARG if self.constant else ParameterMethodLocation.POSITIONAL + + @property + def in_method_signature(self) -> bool: + if self.yaml_data.get("entries"): + # Right now, only legacy generates with multipart bodies and entries + # and legacy generates with the multipart body arguments splatted out + return False + return not (self.flattened or self.grouped_by) + + @property + def content_types(self) -> List[str]: + return self.yaml_data["contentTypes"] + + @property + def default_content_type(self) -> str: + return self.yaml_data["defaultContentType"] + + @property + def has_json_model_type(self) -> bool: + if isinstance(self.type, CombinedType): + return self.type.target_model_subtype((JSONModelType,)) is not None + return isinstance(self.type, JSONModelType) + + def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: + file_import = super().imports(async_mode, **kwargs) + if self.is_form_data: + relative_path = "..." if async_mode else ".." + file_import.add_submodule_import( + f"{relative_path}_vendor", + "prepare_multipart_form_data", + ImportType.LOCAL, + ) + file_import.add_submodule_import("typing", "List", ImportType.STDLIB) + return file_import + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "BodyParameter": + return cls( + yaml_data=yaml_data, + code_model=code_model, + type=code_model.lookup_type(id(yaml_data["type"])), + ) + + +EntryBodyParameterType = TypeVar("EntryBodyParameterType", bound=Union[BodyParameter, "RequestBuilderBodyParameter"]) + + +class Parameter(_ParameterBase): + """Basic Parameter class""" + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + type: BaseType, + ) -> None: + super().__init__(yaml_data, code_model, type=type) + + self.skip_url_encoding: bool = self.yaml_data.get("skipUrlEncoding", False) + self.explode: bool = self.yaml_data.get("explode", False) + self.in_overriden: bool = self.yaml_data.get("inOverriden", False) + self.delimiter: Optional[ParameterDelimeter] = self.yaml_data.get("delimiter") + self._default_to_unset_sentinel: bool = False + + @property + def hide_in_operation_signature(self) -> bool: + if self.code_model.options["version_tolerant"] and self.client_name == "maxpagesize": + return True + return False + + @property + def in_method_signature(self) -> bool: + return not (self.wire_name == "Accept" or self.grouped_by or self.flattened) + + @property + def full_client_name(self) -> str: + if self.implementation == "Client": + return f"self._config.{self.client_name}" + return self.client_name + + @property + def xml_serialization_ctxt(self) -> str: + return self.type.xml_serialization_ctxt or "" + + @property + def is_content_type(self) -> bool: + return bool(self.wire_name) and self.wire_name.lower() == "content-type" + + @property + def method_location( # pylint: disable=too-many-return-statements + self, + ) -> ParameterMethodLocation: + if not self.in_method_signature: + raise ValueError(f"Parameter '{self.client_name}' is not in the method.") + if self.code_model.options["models_mode"] == "dpg" and self.in_flattened_body: + return ParameterMethodLocation.KEYWORD_ONLY + if self.grouper: + return ParameterMethodLocation.POSITIONAL + if self.constant and self.wire_name != "Content-Type": + return ParameterMethodLocation.KWARG + if self.is_content_type: + if self.in_overload: + return ParameterMethodLocation.KEYWORD_ONLY + return ParameterMethodLocation.KWARG + query_or_header = self.location in ( + ParameterLocation.HEADER, + ParameterLocation.QUERY, + ) + if self.code_model.options["only_path_and_body_params_positional"] and query_or_header: + return ParameterMethodLocation.KEYWORD_ONLY + return ParameterMethodLocation.POSITIONAL + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel"): + return cls( + yaml_data=yaml_data, + code_model=code_model, + type=code_model.lookup_type(id(yaml_data["type"])), + ) + + +class ClientParameter(Parameter): + """Client parameter""" + + @property + def is_host(self) -> bool: + return self.wire_name == "$host" + + @property + def method_location(self) -> ParameterMethodLocation: + if self.constant: + return ParameterMethodLocation.KWARG + if ( + self.is_host + and (self.code_model.options["version_tolerant"] or self.code_model.options["low_level_client"]) + and not self.code_model.options["azure_arm"] + ): + # this means i am the base url + return ParameterMethodLocation.KEYWORD_ONLY + return ParameterMethodLocation.POSITIONAL + + +class ConfigParameter(Parameter): + """Config Parameter""" + + @property + def in_method_signature(self) -> bool: + return not self.is_host + + @property + def is_host(self) -> bool: + return self.wire_name == "$host" + + @property + def method_location(self) -> ParameterMethodLocation: + if self.constant: + return ParameterMethodLocation.KWARG + return ParameterMethodLocation.POSITIONAL diff --git a/packages/autorest.python/generator/pygen/codegen/models/parameter_list.py b/packages/autorest.python/generator/pygen/codegen/models/parameter_list.py new file mode 100644 index 00000000000..73278584a34 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/parameter_list.py @@ -0,0 +1,390 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + TYPE_CHECKING, + Union, + Generic, + TypeVar, + cast, +) +from abc import abstractmethod +from collections.abc import MutableSequence +from enum import Enum + +from .request_builder_parameter import ( + RequestBuilderBodyParameter, + RequestBuilderParameter, +) +from .parameter import ( + ParameterLocation, + BodyParameter, + Parameter, + ParameterMethodLocation, + ClientParameter, + ConfigParameter, +) + +ParameterType = TypeVar("ParameterType", bound=Union[Parameter, RequestBuilderParameter]) +BodyParameterType = TypeVar("BodyParameterType", bound=Union[BodyParameter, RequestBuilderBodyParameter]) + +if TYPE_CHECKING: + from .code_model import CodeModel + + +class ParameterImplementation(Enum): + METHOD = "method" + CLIENT = "client" + + +_LOGGER = logging.getLogger(__name__) + + +def method_signature_helper(positional: List[str], keyword_only: Optional[List[str]], kwarg_params: List[str]): + keyword_only = keyword_only or [] + return positional + keyword_only + kwarg_params + + +def _sort(params): + return sorted(params, key=lambda x: not (x.client_default_value or x.optional), reverse=True) + + +class _ParameterListBase( + MutableSequence, Generic[ParameterType, BodyParameterType] +): # pylint: disable=too-many-public-methods + """Base class for all of our different ParameterList classes""" + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + parameters: List[ParameterType], + body_parameter: Optional[BodyParameterType] = None, + ) -> None: + self.yaml_data = yaml_data + self.code_model = code_model + self.parameters = parameters or [] + self._body_parameter = body_parameter + + # MutableSequence + + def __getitem__(self, index): + if isinstance(index, str): + raise TypeError(f"{index} is invalid type") + return self.parameters[index] + + def __len__(self) -> int: + return len(self.parameters) + + def __setitem__(self, index, parameter): + self.parameters[index] = parameter + + def __delitem__(self, index): + del self.parameters[index] + + def insert(self, index: int, value: ParameterType) -> None: + self.parameters.insert(index, value) + + # Parameter helpers + + @staticmethod + @abstractmethod + def parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], ParameterType]: + """Callable for creating parameters""" + + @staticmethod + @abstractmethod + def body_parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], BodyParameterType]: + """Callable for creating body parameters""" + + @property + def grouped(self) -> List[Union[ParameterType, BodyParameterType]]: + """All parameters that are inside a parameter group""" + params: List[Union[ParameterType, BodyParameterType]] = [p for p in self.parameters if p.grouped_by] + if self.has_body and self.body_parameter.grouped_by: + params.append(self.body_parameter) + return params + + @property + def has_form_data_body(self): + return self.has_body and self.body_parameter.is_form_data + + @property + def has_body(self) -> bool: + """Whether there is a body parameter in the parameter list""" + return bool(self._body_parameter) + + @property + def path(self) -> List[ParameterType]: + """All path parameters""" + return [p for p in self.parameters if p.location in (ParameterLocation.PATH, ParameterLocation.ENDPOINT_PATH)] + + @property + def query(self) -> List[ParameterType]: + """All query parameters""" + return [p for p in self.parameters if p.location == ParameterLocation.QUERY] + + @property + def headers(self) -> List[ParameterType]: + """All header parameters""" + return [p for p in self.parameters if p.location == ParameterLocation.HEADER] + + @property + def constant(self) -> List[Union[ParameterType, BodyParameterType]]: + """All constant parameters""" + return [p for p in self.parameters if p.constant] + + @property + def positional(self) -> List[Union[ParameterType, BodyParameterType]]: + """All positional parameters""" + return _sort( + [p for p in self.unsorted_method_params if p.method_location == ParameterMethodLocation.POSITIONAL] + ) + + @property + def keyword_only(self) -> List[Union[ParameterType, BodyParameterType]]: + """All keyword only parameters""" + return _sort( + [p for p in self.unsorted_method_params if p.method_location == ParameterMethodLocation.KEYWORD_ONLY] + ) + + @property + def kwarg(self) -> List[Union[ParameterType, BodyParameterType]]: + """All kwargs""" + return _sort([p for p in self.unsorted_method_params if p.method_location == ParameterMethodLocation.KWARG]) + + @property + def body_parameter(self) -> BodyParameterType: + """The body parameter of the parameter list. Will only ever be at most one.""" + if not self._body_parameter: + raise ValueError("There is no body parameter") + return self._body_parameter + + @property + @abstractmethod + def implementation(self) -> str: + """Whether this is a client or a method parameter""" + + @property + def unsorted_method_params(self) -> List[Union[ParameterType, BodyParameterType]]: + """Method params before sorting""" + method_params: List[Union[ParameterType, BodyParameterType]] = [ + p + for p in self.parameters + if p.in_method_signature + and p.implementation == self.implementation + and (self.code_model.is_legacy or not p.hide_in_method) + ] + if self._body_parameter: + if self._body_parameter.in_method_signature: + method_params.append(self._body_parameter) + try: + # i am a multipart body parameter + # Only legacy generates operations with me, so I will follow the legacy rules + # I will splat out my entries as individual entries + method_params.extend(self._body_parameter.entries) # type: ignore + except AttributeError: + pass + return method_params + + @property + def method(self) -> List[Union[ParameterType, BodyParameterType]]: + """Sorted method params. First positional, then keyword only, then kwarg""" + return self.positional + self.keyword_only + self.kwarg + + def method_signature(self, async_mode: bool) -> List[str]: + """Method signature for this parameter list.""" + return method_signature_helper( + positional=self.method_signature_positional(async_mode), + keyword_only=self.method_signature_keyword_only(async_mode), + kwarg_params=self.method_signature_kwargs, + ) + + def method_signature_positional(self, async_mode: bool) -> List[str]: + """Signature for positional parameters""" + return [parameter.method_signature(async_mode) for parameter in self.positional] + + def method_signature_keyword_only(self, async_mode: bool) -> List[str]: + """Signature for keyword only parameters""" + result = [ + parameter.method_signature(async_mode) + for parameter in self.keyword_only + if not parameter.hide_in_operation_signature + ] + return ["*,"] + result if result else [] + + @property + def method_signature_kwargs(self) -> List[str]: + """Signature for kwargs""" + return ["**kwargs: Any"] + + @property + def kwargs_to_pop(self) -> List[Union[ParameterType, BodyParameterType]]: + """Method kwargs we want to pop""" + # don't want to pop bodies unless it's a constant + kwargs_to_pop = self.kwarg + return [k for k in kwargs_to_pop if k.location != ParameterLocation.BODY or k.constant] + + @property + def call(self) -> List[str]: + """How to pass in parameters to call the operation""" + retval = [p.client_name for p in self.method if p.method_location == ParameterMethodLocation.POSITIONAL] + retval.extend( + [ + f"{p.client_name}={p.client_name}" + for p in self.method + if p.method_location == ParameterMethodLocation.KEYWORD_ONLY + ] + ) + retval.append("**kwargs") + return retval + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel"): + parameters = [cls.parameter_creator()(parameter, code_model) for parameter in yaml_data["parameters"]] + body_parameter = None + if yaml_data.get("bodyParameter"): + body_parameter = cls.body_parameter_creator()(yaml_data["bodyParameter"], code_model) + return cls( + yaml_data, + code_model, + parameters=parameters, + body_parameter=body_parameter, + ) + + +class _ParameterList(_ParameterListBase[Parameter, BodyParameter]): # pylint: disable=unsubscriptable-object + """Base Parameter class for the two operation ParameterLists""" + + @staticmethod + def parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], Parameter]: + return Parameter.from_yaml + + @staticmethod + def body_parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], BodyParameter]: + return BodyParameter.from_yaml + + @property + def implementation(self) -> str: + return "Method" + + @property + def path(self) -> List[Parameter]: + return [k for k in super().path if k.location == ParameterLocation.ENDPOINT_PATH] + + +class ParameterList(_ParameterList): + """ParameterList is the parameter list for Operation classes""" + + +class _RequestBuilderParameterList( + _ParameterListBase[RequestBuilderParameter, RequestBuilderBodyParameter] # pylint: disable=unsubscriptable-object +): + """_RequestBuilderParameterList is base parameter list for RequestBuilder classes""" + + @staticmethod + def parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], RequestBuilderParameter]: + return RequestBuilderParameter.from_yaml + + @staticmethod + def body_parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], RequestBuilderBodyParameter]: + return RequestBuilderBodyParameter.from_yaml + + @property + def implementation(self) -> str: + return "Method" + + @property + def unsorted_method_params( + self, + ) -> List[Union[RequestBuilderParameter, RequestBuilderBodyParameter]]: + # don't have access to client params in request builder + retval = [ + p + for p in super().unsorted_method_params + if not (p.location == ParameterLocation.BODY and cast(RequestBuilderBodyParameter, p).is_partial_body) + ] + retval.extend([p for p in self.parameters if p.implementation == "Client" and p.in_method_signature]) + return retval + + @property + def path(self) -> List[RequestBuilderParameter]: + return [p for p in super().path if p.location != ParameterLocation.ENDPOINT_PATH] + + @property + def constant( + self, + ) -> List[Union[RequestBuilderParameter, RequestBuilderBodyParameter]]: + """All constant parameters""" + return [p for p in super().constant if p.location != ParameterLocation.ENDPOINT_PATH] + + +class RequestBuilderParameterList(_RequestBuilderParameterList): + """Parameter list for Request Builder""" + + +class OverloadedRequestBuilderParameterList(_RequestBuilderParameterList): + """Parameter list for OverloadedRequestBuilder""" + + +class _ClientGlobalParameterList(_ParameterListBase[ParameterType, BodyParameter]): # pylint: disable=abstract-method + """Base parameter list for client and config classes""" + + @staticmethod + def body_parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], BodyParameter]: + return BodyParameter.from_yaml + + @property + def implementation(self) -> str: + return "Client" + + @property + def credential(self) -> Optional[ParameterType]: + try: + return next(p for p in self.parameters if p.client_name == "credential") + except StopIteration: + return None + + @property + def path(self) -> List[ParameterType]: + return [p for p in super().path if p.location == ParameterLocation.ENDPOINT_PATH] + + +class ClientGlobalParameterList(_ClientGlobalParameterList[ClientParameter]): + """Parameter list for Client class""" + + @staticmethod + def parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], ClientParameter]: + return ClientParameter.from_yaml + + @property + def path(self) -> List[ClientParameter]: + return [p for p in super().path if not p.is_host] + + @property + def host(self) -> Optional[ClientParameter]: + """Get the host parameter""" + try: + return next(p for p in self.parameters if p.is_host) + except StopIteration: + return None + + +class ConfigGlobalParameterList(_ClientGlobalParameterList[ConfigParameter]): + """Parameter list for config""" + + @staticmethod + def parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], ConfigParameter]: + return ConfigParameter.from_yaml + + @property + def implementation(self) -> str: + return "Client" diff --git a/packages/autorest.python/generator/pygen/codegen/models/primitive_types.py b/packages/autorest.python/generator/pygen/codegen/models/primitive_types.py new file mode 100644 index 00000000000..a4fc0392a6d --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/primitive_types.py @@ -0,0 +1,640 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import datetime +import decimal +from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING + +from .base import BaseType +from .imports import FileImport, ImportType, TypingSection +from .utils import add_to_description + +if TYPE_CHECKING: + from .code_model import CodeModel + + +class RawString(object): + def __init__(self, string: str) -> None: + self.string = string + + def __repr__(self) -> str: + return "r'{}'".format(self.string.replace("'", "\\'")) + + +class PrimitiveType(BaseType): # pylint: disable=abstract-method + def description(self, *, is_operation_file: bool) -> str: # pylint: disable=unused-argument + return "" + + def type_annotation(self, **kwargs: Any) -> str: + return self.docstring_type(**kwargs) + + def docstring_text(self, **kwargs: Any) -> str: + return self.docstring_type(**kwargs) + + def get_json_template_representation( + self, + *, + optional: bool = True, + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + comment = "" + if optional: + comment = add_to_description(comment, "Optional.") + if self.client_default_value is not None: + client_default_value_declaration = client_default_value_declaration or self.get_declaration( + self.client_default_value + ) + if client_default_value_declaration: + comment = add_to_description(comment, f"Default value is {client_default_value_declaration}.") + else: + client_default_value_declaration = self.default_template_representation_declaration + if description: + comment = add_to_description(comment, description) + if comment: + comment = f"# {comment}" + return client_default_value_declaration + ("" if self.code_model.for_test else comment) + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration(self.docstring_type()) + + +class BooleanType(PrimitiveType): + @property + def serialization_type(self) -> str: + return "bool" + + def docstring_type(self, **kwargs: Any) -> str: + return "bool" + + @property + def instance_check_template(self) -> str: + return "isinstance({}, bool)" + + +class BinaryType(PrimitiveType): + def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.type = "IO" + + @property + def serialization_type(self) -> str: + return self.type + + def docstring_type(self, **kwargs: Any) -> str: + return f"{self.type}[bytes]" + + def type_annotation(self, **kwargs: Any) -> str: + return f"{self.type}[bytes]" + + def docstring_text(self, **kwargs: Any) -> str: + return f"{self.type}[bytes]" + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration(b"bytes") + + def imports(self, **kwargs: Any) -> FileImport: + from .combined_type import CombinedType + from .operation import OperationBase + + file_import = FileImport(self.code_model) + file_import.add_submodule_import("typing", "IO", ImportType.STDLIB) + operation = kwargs.get("operation") + if ( + isinstance(operation, OperationBase) + and operation.parameters.has_body + and isinstance(operation.parameters.body_parameter.type, CombinedType) + ): + file_import.add_submodule_import("io", "IOBase", ImportType.STDLIB) + return file_import + + @property + def instance_check_template(self) -> str: + return "isinstance({}, (IOBase, bytes))" + + +class BinaryIteratorType(PrimitiveType): + def _iterator_name(self, **kwargs: Any) -> str: + return "AsyncIterator" if kwargs.pop("async_mode") else "Iterator" + + @property + def serialization_type(self) -> str: + return "IO" + + def docstring_type(self, **kwargs: Any) -> str: + return f"{self._iterator_name(**kwargs)}[bytes]" + + def type_annotation(self, **kwargs: Any) -> str: + return f"{self._iterator_name(**kwargs)}[bytes]" + + def docstring_text(self, **kwargs: Any) -> str: + return f"{self._iterator_name(**kwargs)}[bytes]" + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration(b"bytes") + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_submodule_import("typing", self._iterator_name(**kwargs), ImportType.STDLIB) + return file_import + + @property + def instance_check_template(self) -> str: + return "getattr({}, '__aiter__', None) is not None or getattr({}, '__iter__', None) is not None" + + +class AnyType(PrimitiveType): + @property + def serialization_type(self) -> str: + return "object" + + def docstring_type(self, **kwargs: Any) -> str: + return "any" + + def type_annotation(self, **kwargs: Any) -> str: + return "Any" + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration({}) + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) + return file_import + + @property + def instance_check_template(self) -> str: + raise ValueError("Shouldn't do instance check on an anytype, it can be anything") + + +class AnyObjectType(PrimitiveType): + @property + def serialization_type(self) -> str: + return "object" + + def docstring_type(self, **kwargs: Any) -> str: + return "JSON" + + def type_annotation(self, **kwargs: Any) -> str: + return "JSON" + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration({}) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, MutableMapping)" + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.define_mutable_mapping_type() + return file_import + + @property + def type_description(self) -> str: + return "JSON" + + +class NumberType(PrimitiveType): # pylint: disable=abstract-method + def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.precision: Optional[int] = yaml_data.get("precision") + self.multiple: Optional[int] = yaml_data.get("multipleOf") + self.maximum: Optional[int] = yaml_data.get("maximum") + self.minimum: Optional[int] = yaml_data.get("minimum") + self.exclusive_maximum: Optional[int] = yaml_data.get("exclusiveMaximum") + self.exclusive_minimum: Optional[int] = yaml_data.get("exclusiveMinimum") + + @property + def serialization_constraints(self) -> List[str]: + validation_constraints = [ + (f"maximum_ex={self.maximum}" if self.maximum is not None and self.exclusive_maximum else None), + (f"maximum={self.maximum}" if self.maximum is not None and not self.exclusive_maximum else None), + (f"minimum_ex={self.minimum}" if self.minimum is not None and self.exclusive_minimum else None), + (f"minimum={self.minimum}" if self.minimum is not None and not self.exclusive_minimum else None), + f"multiple={self.multiple}" if self.multiple else None, + ] + return [x for x in validation_constraints if x is not None] + + @property + def validation(self) -> Optional[Dict[str, Union[bool, int, str]]]: + validation: Dict[str, Union[bool, int, str]] = {} + if self.maximum is not None: + if self.exclusive_maximum: + validation["maximum_ex"] = self.maximum + else: + validation["maximum"] = self.maximum + if self.minimum is not None: + if self.exclusive_minimum: + validation["minimum_ex"] = self.minimum + else: + validation["minimum"] = self.minimum + if self.multiple: + validation["multiple"] = self.multiple + return validation or None + + @property + def default_template_representation_declaration(self) -> str: + default_value = 0 if self.docstring_type() == "int" else 0.0 + return self.get_declaration(default_value) + + +class IntegerType(NumberType): + @property + def serialization_type(self) -> str: + return "int" + + def docstring_type(self, **kwargs: Any) -> str: + return "int" + + def type_annotation(self, **kwargs: Any) -> str: + return "int" + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration(0) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, int)" + + +class FloatType(NumberType): + @property + def serialization_type(self) -> str: + return "float" + + def docstring_type(self, **kwargs: Any) -> str: + return "float" + + def type_annotation(self, **kwargs: Any) -> str: + return "float" + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration(0.0) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, float)" + + +class DecimalType(NumberType): + @property + def serialization_type(self) -> str: + return "decimal" + + def docstring_type(self, **kwargs: Any) -> str: + return "~" + self.type_annotation() + + def type_annotation(self, **kwargs: Any) -> str: + return "decimal.Decimal" + + def docstring_text(self, **kwargs: Any) -> str: + return self.type_annotation() + + def get_declaration(self, value: decimal.Decimal) -> str: + return str(value) + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_import("decimal", ImportType.STDLIB) + return file_import + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration(decimal.Decimal("0.0")) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, decimal.Decimal)" + + +class StringType(PrimitiveType): + def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.max_length: Optional[int] = yaml_data.get("maxLength") + self.min_length: Optional[int] = ( + yaml_data.get("minLength", 0) if yaml_data.get("maxLength") else yaml_data.get("minLength") + ) + self.pattern: Optional[str] = yaml_data.get("pattern") + + @property + def serialization_constraints(self) -> List[str]: + validation_constraints = [ + f"max_length={self.max_length}" if self.max_length is not None else None, + f"min_length={self.min_length}" if self.min_length is not None else None, + f"pattern={RawString(self.pattern)}" if self.pattern else None, + ] + return [x for x in validation_constraints if x is not None] + + @property + def validation(self) -> Optional[Dict[str, Union[bool, int, str]]]: + validation: Dict[str, Union[bool, int, str]] = {} + if self.max_length is not None: + validation["max_length"] = self.max_length + if self.min_length is not None: + validation["min_length"] = self.min_length + if self.pattern: + # https://github.com/Azure/autorest.python/issues/407 + validation["pattern"] = RawString(self.pattern) # type: ignore + return validation or None + + def get_declaration(self, value) -> str: + return f'"{value}"' + + @property + def serialization_type(self) -> str: + return "str" + + def docstring_type(self, **kwargs: Any) -> str: + return "str" + + @property + def instance_check_template(self) -> str: + return "isinstance({}, str)" + + +class DatetimeType(PrimitiveType): + def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.encode = ( + "rfc3339" + if yaml_data.get("encode", "date-time") == "date-time" or yaml_data.get("encode", "date-time") == "rfc3339" + else "rfc7231" + ) + + @property + def serialization_type(self) -> str: + formats_to_attribute_type = { + "rfc3339": "iso-8601", + "rfc7231": "rfc-1123", + } + return formats_to_attribute_type[self.encode] + + def docstring_type(self, **kwargs: Any) -> str: + return "~" + self.type_annotation() + + def type_annotation(self, **kwargs: Any) -> str: + return "datetime.datetime" + + def docstring_text(self, **kwargs: Any) -> str: + return "datetime" + + def get_declaration(self, value: datetime.datetime) -> str: + """Could be discussed, since technically I should return a datetime object, + but msrest will do fine. + """ + return f'"{value}"' + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_import("datetime", ImportType.STDLIB) + return file_import + + @property + def default_template_representation_declaration(self): + return self.get_declaration(datetime.datetime(2020, 2, 20)) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, datetime.datetime)" + + def imports_for_sample(self) -> FileImport: + file_import = super().imports_for_sample() + file_import.add_import("isodate", ImportType.STDLIB) + return file_import + + @staticmethod + def serialize_sample_value(value: Any) -> str: + return f"isodate.parse_datetime({repr(value)})" + + +class TimeType(PrimitiveType): + @property + def serialization_type(self) -> str: + return "time" + + def docstring_type(self, **kwargs: Any) -> str: + return "~" + self.type_annotation() + + def type_annotation(self, **kwargs: Any) -> str: + return "datetime.time" + + def docstring_text(self, **kwargs: Any) -> str: + return "time" + + def get_declaration(self, value: datetime.time) -> str: + """Could be discussed, since technically I should return a time object, + but msrest will do fine. + """ + return f'"{value}"' + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_import("datetime", ImportType.STDLIB) + return file_import + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration(datetime.time(12, 30, 0)) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, datetime.time)" + + def imports_for_sample(self) -> FileImport: + file_import = super().imports_for_sample() + file_import.add_import("isodate", ImportType.STDLIB) + return file_import + + @staticmethod + def serialize_sample_value(value: Any) -> str: + return f"isodate.parse_time({repr(value)})" + + +class UnixTimeType(PrimitiveType): + @property + def encode(self) -> str: + return "unix-timestamp" + + @property + def serialization_type(self) -> str: + return "unix-time" + + def docstring_type(self, **kwargs: Any) -> str: + return "~" + self.type_annotation() + + def type_annotation(self, **kwargs: Any) -> str: + return "datetime.datetime" + + def docstring_text(self, **kwargs: Any) -> str: + return "datetime" + + def get_declaration(self, value: datetime.datetime) -> str: + """Could be discussed, since technically I should return a datetime object, + but msrest will do fine. + """ + return f'"{value}"' + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_import("datetime", ImportType.STDLIB) + return file_import + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration(datetime.datetime(2020, 2, 20)) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, datetime.time)" + + def imports_for_sample(self) -> FileImport: + file_import = super().imports_for_sample() + file_import.add_import("datetime", ImportType.STDLIB) + return file_import + + @staticmethod + def serialize_sample_value(value: Any) -> str: + return f"datetime.datetime.fromtimestamp({repr(value)}, datetime.timezone.utc)" + + +class DateType(PrimitiveType): + @property + def serialization_type(self) -> str: + return "date" + + def docstring_type(self, **kwargs: Any) -> str: + return "~" + self.type_annotation() + + def type_annotation(self, **kwargs: Any) -> str: + return "datetime.date" + + def docstring_text(self, **kwargs: Any) -> str: + return "date" + + def get_declaration(self, value: datetime.date) -> str: + """Could be discussed, since technically I should return a datetime object, + but msrest will do fine. + """ + return f'"{value}"' + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_import("datetime", ImportType.STDLIB) + return file_import + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration(datetime.date(2020, 2, 20)) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, datetime.date)" + + def imports_for_sample(self) -> FileImport: + file_import = super().imports_for_sample() + file_import.add_import("isodate", ImportType.STDLIB) + return file_import + + @staticmethod + def serialize_sample_value(value: Any) -> str: + return f"isodate.parse_date({repr(value)})" + + +class DurationType(PrimitiveType): + @property + def serialization_type(self) -> str: + return "duration" + + def docstring_type(self, **kwargs: Any) -> str: + return "~" + self.type_annotation() + + def type_annotation(self, **kwargs: Any) -> str: + return "datetime.timedelta" + + def docstring_text(self, **kwargs: Any) -> str: + return "timedelta" + + def get_declaration(self, value: datetime.timedelta) -> str: + """Could be discussed, since technically I should return a datetime object, + but msrest will do fine. + """ + return f'"{value}"' + + def imports(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_import("datetime", ImportType.STDLIB) + return file_import + + @property + def default_template_representation_declaration(self) -> str: + return self.get_declaration(datetime.timedelta(1)) + + @property + def instance_check_template(self) -> str: + return "isinstance({}, datetime.timedelta)" + + def imports_for_sample(self) -> FileImport: + file_import = super().imports_for_sample() + file_import.add_import("isodate", ImportType.STDLIB) + return file_import + + @staticmethod + def serialize_sample_value(value: Any) -> str: + return f"isodate.parse_duration({repr(value)})" + + +class ByteArraySchema(PrimitiveType): + def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.encode = yaml_data.get("encode", "base64") + + @property + def serialization_type(self) -> str: + if self.encode == "base64url": + return "base64" + return "bytearray" + + def docstring_type(self, **kwargs: Any) -> str: + return "bytes" + + def get_declaration(self, value: str) -> str: + return f'bytes("{value}", encoding="utf-8")' + + @property + def instance_check_template(self) -> str: + return "isinstance({}, bytes)" + + +class SdkCoreType(PrimitiveType): + def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.name = yaml_data.get("name", "") + + def docstring_type(self, **kwargs: Any) -> str: + return f"~{self.code_model.core_library}.{self.type_annotation(**kwargs)}" + + def type_annotation(self, **kwargs: Any) -> str: + return self.name + + def imports(self, **kwargs: Any) -> FileImport: + file_import = super().imports(**kwargs) + file_import.add_submodule_import("", self.name, ImportType.SDKCORE) + return file_import + + @property + def instance_check_template(self) -> str: + return f"isinstance({{}}, {self.name})" + + @property + def serialization_type(self) -> str: + return self.name diff --git a/packages/autorest.python/generator/pygen/codegen/models/property.py b/packages/autorest.python/generator/pygen/codegen/models/property.py new file mode 100644 index 00000000000..bdda254c248 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/property.py @@ -0,0 +1,182 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict, Optional, TYPE_CHECKING, List, cast + +from .base import BaseModel +from .constant_type import ConstantType +from .enum_type import EnumType +from .base import BaseType +from .imports import FileImport, ImportType +from .utils import add_to_description, add_to_pylint_disable + +if TYPE_CHECKING: + from .code_model import CodeModel + from .model_type import ModelType + + +class Property(BaseModel): # pylint: disable=too-many-instance-attributes + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + type: BaseType, + ) -> None: + super().__init__(yaml_data, code_model) + self.wire_name: str = self.yaml_data["wireName"] + self.client_name: str = self.yaml_data["clientName"] + self.type = type + self.optional: bool = self.yaml_data["optional"] + self.readonly: bool = self.yaml_data.get("readonly", False) + self.visibility: List[str] = self.yaml_data.get("visibility", []) + self.is_polymorphic: bool = self.yaml_data.get("isPolymorphic", False) + self.is_discriminator: bool = yaml_data.get("isDiscriminator", False) + self.client_default_value = yaml_data.get("clientDefaultValue", None) + if self.client_default_value is None: + self.client_default_value = self.type.client_default_value + self.flattened_names: List[str] = yaml_data.get("flattenedNames", []) + self.is_multipart_file_input: bool = yaml_data.get("isMultipartFileInput", False) + self.flatten = self.yaml_data.get("flatten", False) and not getattr(self.type, "flattened_property", False) + + @property + def pylint_disable(self) -> str: + retval: str = "" + if self.yaml_data.get("pylintDisable"): + retval = add_to_pylint_disable(retval, self.yaml_data["pylintDisable"]) + return retval + + def description(self, *, is_operation_file: bool) -> str: + from .model_type import ModelType + + description = self.yaml_data.get("description", "") + if not (self.optional or self.client_default_value): + description = add_to_description(description, "Required.") + # don't want model type documentation as part of property doc + type_description = ( + "" if isinstance(self.type, ModelType) else self.type.description(is_operation_file=is_operation_file) + ) + return add_to_description(description, type_description) + + @property + def client_default_value_declaration(self) -> str: + if self.client_default_value is not None: + return self.get_declaration(self.client_default_value) + if self.type.client_default_value is not None: + return self.get_declaration(self.type.client_default_value) + return "None" + + @property + def constant(self) -> bool: + # this bool doesn't consider you to be constant if you are a discriminator + # you also have to be required to be considered a constant + return isinstance(self.type, ConstantType) and not self.optional and not self.is_discriminator + + @property + def is_input(self): + return not (self.constant or self.readonly or self.is_discriminator) + + @property + def serialization_type(self) -> str: + return self.type.serialization_type + + @property + def msrest_deserialization_key(self) -> str: + return self.type.msrest_deserialization_key + + @property + def is_enum_discriminator(self) -> bool: + return self.is_discriminator and self.type.type == "enum" + + @property + def is_base_discriminator(self) -> bool: + """If this discriminator is on the base model for polymorphic inheritance""" + if self.is_enum_discriminator: + return self.is_polymorphic and self.client_default_value is None + return self.is_discriminator and self.is_polymorphic and cast(ConstantType, self.type).value is None + + def type_annotation(self, *, is_operation_file: bool = False) -> str: + types_type_annotation = self.type.type_annotation(is_operation_file=is_operation_file) + if self.is_multipart_file_input: + # we only support FileType or list of FileType + types_type_annotation = types_type_annotation.replace("bytes", "FileType") + if self.is_base_discriminator: + return "str" + if self.optional and self.client_default_value is None: + return f"Optional[{types_type_annotation}]" + return types_type_annotation + + def get_declaration(self, value: Any = None) -> Any: + return self.type.get_declaration(value) + + def get_json_template_representation( + self, + *, + optional: bool = True, # pylint: disable=unused-argument + client_default_value_declaration: Optional[str] = None, + description: Optional[str] = None, + ) -> Any: + if self.is_multipart_file_input: + file_type_str = '"filetype"' if self.code_model.for_test else "filetype" + return f"[{file_type_str}]" if self.type.type == "list" else file_type_str + if self.client_default_value: + client_default_value_declaration = self.get_declaration(self.client_default_value) + if self.description(is_operation_file=True): + description = self.description(is_operation_file=True) + # make sure there is no \n otherwise the json template will be invalid + description = (description or "").replace("\n", " ") + return self.type.get_json_template_representation( + optional=self.optional, + client_default_value_declaration=client_default_value_declaration, + description=description, + ) + + def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: + from .model_type import ModelType + + if isinstance(self.type, ModelType): + self.type.get_polymorphic_subtypes(polymorphic_subtypes) + + @property + def validation(self) -> Optional[Dict[str, Any]]: + retval: Dict[str, Any] = {} + if not self.optional: + retval["required"] = True + if self.readonly: + retval["readonly"] = True + if self.constant: + retval["constant"] = True + retval.update(self.type.validation or {}) + return retval or None + + def imports(self, **kwargs) -> FileImport: + file_import = FileImport(self.code_model) + if self.is_discriminator and isinstance(self.type, EnumType): + return file_import + file_import.merge(self.type.imports(**kwargs, relative_path="..", model_typing=True)) + if self.optional and self.client_default_value is None: + file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) + if self.code_model.options["models_mode"] == "dpg": + file_import.add_submodule_import( + ".._model_base", + "rest_discriminator" if self.is_discriminator else "rest_field", + ImportType.LOCAL, + ) + if self.is_multipart_file_input: + file_import.add_submodule_import(".._vendor", "FileType", ImportType.LOCAL) + return file_import + + @classmethod + def from_yaml( + cls, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + ) -> "Property": + from . import build_type # pylint: disable=import-outside-toplevel + + return cls( + yaml_data=yaml_data, + code_model=code_model, + type=build_type(yaml_data["type"], code_model), + ) diff --git a/packages/autorest.python/generator/pygen/codegen/models/request_builder.py b/packages/autorest.python/generator/pygen/codegen/models/request_builder.py new file mode 100644 index 00000000000..b8786adba6f --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/request_builder.py @@ -0,0 +1,189 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import ( + Any, + Callable, + Dict, + List, + TypeVar, + TYPE_CHECKING, + Union, + Optional, +) +from abc import abstractmethod + +from .base_builder import BaseBuilder +from .utils import add_to_pylint_disable, NAME_LENGTH_LIMIT +from .parameter_list import ( + RequestBuilderParameterList, + OverloadedRequestBuilderParameterList, +) +from .imports import FileImport, ImportType, TypingSection, MsrestImportType + +if TYPE_CHECKING: + from .code_model import CodeModel + from .client import Client + +ParameterListType = TypeVar( + "ParameterListType", + bound=Union[RequestBuilderParameterList, OverloadedRequestBuilderParameterList], +) + + +class RequestBuilderBase(BaseBuilder[ParameterListType, List["RequestBuilder"]]): + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + client: "Client", + name: str, + parameters: ParameterListType, + *, + overloads: Optional[List["RequestBuilder"]] = None, + ) -> None: + super().__init__( + code_model=code_model, + client=client, + yaml_data=yaml_data, + name=name, + parameters=parameters, + overloads=overloads, + ) + self.overloads: List["RequestBuilder"] = overloads or [] + self.url: str = yaml_data["url"] + self.method: str = yaml_data["method"] + self.want_tracing = False + + @property + def has_form_data_body(self): + return self.parameters.has_form_data_body + + @property + def is_lro(self) -> bool: + return self.yaml_data.get("discriminator") in ("lro", "lropaging") + + @property + def pylint_disable(self) -> str: + if len(self.name) > NAME_LENGTH_LIMIT: + return add_to_pylint_disable("", "name-too-long") + return "" + + def response_type_annotation(self, **kwargs) -> str: + return "HttpRequest" + + def response_docstring_text(self, **kwargs) -> str: + return ( + f"Returns an :class:`~{self.response_docstring_type()}` that you will pass to the client's " + + "`send_request` method. See https://aka.ms/azsdk/dpcodegen/python/send_request for how to " + + "incorporate this response into your code flow." + ) + + def response_docstring_type(self, **kwargs) -> str: + return f"~{self.code_model.core_library}.rest.HttpRequest" + + def imports(self) -> FileImport: + file_import = FileImport(self.code_model) + relative_path = ".." + if not self.code_model.options["builders_visibility"] == "embedded" and self.group_name: + relative_path = "..." if self.group_name else ".." + if self.abstract: + return file_import + for parameter in self.parameters.method: + file_import.merge(parameter.imports(async_mode=False, relative_path=relative_path, operation=self)) + + file_import.add_submodule_import( + "rest", + "HttpRequest", + ImportType.SDKCORE, + ) + + if self.parameters.headers or self.parameters.query: + file_import.add_submodule_import( + "utils", + "case_insensitive_dict", + ImportType.SDKCORE, + ) + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, typing_section=TypingSection.CONDITIONAL) + file_import.add_msrest_import( + relative_path=( + "..." + if (not self.code_model.options["builders_visibility"] == "embedded" and self.group_name) + else ".." + ), + msrest_import_type=MsrestImportType.Serializer, + typing_section=TypingSection.REGULAR, + ) + if self.overloads and self.code_model.options["builders_visibility"] != "embedded": + file_import.add_submodule_import("typing", "overload", ImportType.STDLIB) + return file_import + + @staticmethod + @abstractmethod + def parameter_list_type() -> Callable[[Dict[str, Any], "CodeModel"], ParameterListType]: ... + + @classmethod + def get_name( + cls, + name: str, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + client: "Client", + ) -> str: + additional_mark = "" + if code_model.options["combine_operation_files"] and code_model.options["builders_visibility"] == "embedded": + additional_mark = yaml_data["groupName"] or client.yaml_data["builderPadName"] + names = [ + "build", + additional_mark, + name, + "request", + ] + return "_".join([n for n in names if n]) + + @classmethod + def from_yaml( + cls, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + client: "Client", + ): + # when combine embedded builders into one operation file, we need to avoid duplicated build function name. + # So add operation group name is effective method + + overloads = [ + RequestBuilder.from_yaml(rb_yaml_data, code_model, client) + for rb_yaml_data in yaml_data.get("overloads", []) + ] + parameter_list = cls.parameter_list_type()(yaml_data, code_model) + + return cls( + yaml_data=yaml_data, + code_model=code_model, + client=client, + name=cls.get_name(yaml_data["name"], yaml_data, code_model, client), + parameters=parameter_list, + overloads=overloads, + ) + + +class RequestBuilder(RequestBuilderBase[RequestBuilderParameterList]): + @staticmethod + def parameter_list_type() -> Callable[[Dict[str, Any], "CodeModel"], RequestBuilderParameterList]: + return RequestBuilderParameterList.from_yaml + + +class OverloadedRequestBuilder(RequestBuilderBase[OverloadedRequestBuilderParameterList]): + @staticmethod + def parameter_list_type() -> Callable[[Dict[str, Any], "CodeModel"], OverloadedRequestBuilderParameterList]: + return OverloadedRequestBuilderParameterList.from_yaml + + +def get_request_builder( + yaml_data: Dict[str, Any], code_model: "CodeModel", client: "Client" +) -> Union[RequestBuilder, OverloadedRequestBuilder]: + if yaml_data.get("overloads"): + return OverloadedRequestBuilder.from_yaml(yaml_data, code_model, client) + return RequestBuilder.from_yaml(yaml_data, code_model, client) diff --git a/packages/autorest.python/generator/pygen/codegen/models/request_builder_parameter.py b/packages/autorest.python/generator/pygen/codegen/models/request_builder_parameter.py new file mode 100644 index 00000000000..9ed4e9e590b --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/request_builder_parameter.py @@ -0,0 +1,115 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import TYPE_CHECKING, Any, Dict +from .parameter import ( + ParameterLocation, + ParameterMethodLocation, + Parameter, + BodyParameter, +) +from .base import BaseType +from .primitive_types import BinaryType, StringType +from .combined_type import CombinedType + +if TYPE_CHECKING: + from .code_model import CodeModel + + +class RequestBuilderBodyParameter(BodyParameter): + """BOdy parmaeter for RequestBuilders""" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + if ( + isinstance(self.type, (BinaryType, StringType)) + or any("xml" in ct for ct in self.content_types) + or self.code_model.options["models_mode"] == "dpg" + ): + self.client_name = "content" + else: + self.client_name = "json" + + def type_annotation(self, **kwargs: Any) -> str: + if self.type.is_xml: + return "Any" # xml type technically not in type signature for HttpRequest content param + return super().type_annotation(**kwargs) + + @property + def in_method_signature(self) -> bool: + return ( + super().in_method_signature and not self.is_partial_body and self.code_model.options["models_mode"] != "dpg" + ) + + @property + def method_location(self) -> ParameterMethodLocation: + return ( + ParameterMethodLocation.KWARG + if (self.constant or isinstance(self.type, CombinedType)) + else ParameterMethodLocation.KEYWORD_ONLY + ) + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "RequestBuilderBodyParameter": + return super().from_yaml(yaml_data, code_model) # type: ignore + + @property + def name_in_high_level_operation(self) -> str: + if self.client_name == "json": + return "_json" + return "_content" + + +class RequestBuilderParameter(Parameter): + """Basic RequestBuilder Parameter.""" + + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + type: BaseType, + ) -> None: + super().__init__(yaml_data, code_model, type) + # we don't want any default content type behavior in request builder + if self.is_content_type: + self.client_default_value = None + if self.grouped_by and self.client_name[0] == "_": + # we don't want hidden parameters for grouped by in request builders + self.client_name = self.client_name[1:] + + @property + def hide_in_operation_signature(self) -> bool: + return False + + @property + def in_method_signature(self) -> bool: + if self.grouped_by and not self.in_flattened_body: + return True + return super().in_method_signature and not ( + self.location == ParameterLocation.ENDPOINT_PATH or self.in_flattened_body or self.grouper + ) + + @property + def full_client_name(self) -> str: + return self.client_name + + @property + def method_location(self) -> ParameterMethodLocation: + super_method_location = super().method_location + if super_method_location == ParameterMethodLocation.KWARG: + return super_method_location + if self.in_overriden and super_method_location == ParameterMethodLocation.KEYWORD_ONLY: + return ParameterMethodLocation.KWARG + if self.location != ParameterLocation.PATH: + return ParameterMethodLocation.KEYWORD_ONLY + return super_method_location + + @property + def name_in_high_level_operation(self) -> str: + if self.grouped_by: + return f"_{self.client_name}" + if self.implementation == "Client": + return f"self._config.{self.client_name}" + return self.client_name diff --git a/packages/autorest.python/generator/pygen/codegen/models/response.py b/packages/autorest.python/generator/pygen/codegen/models/response.py new file mode 100644 index 00000000000..bd78c440f12 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/response.py @@ -0,0 +1,348 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Dict, Optional, List, Any, TYPE_CHECKING, Union + +from .base import BaseModel +from .base import BaseType +from .imports import FileImport, ImportType, TypingSection +from .primitive_types import BinaryType, BinaryIteratorType, ByteArraySchema +from .dictionary_type import DictionaryType +from .list_type import ListType +from .model_type import ModelType +from .combined_type import CombinedType + +if TYPE_CHECKING: + from .code_model import CodeModel + + +class ResponseHeader(BaseModel): + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + type: BaseType, + ) -> None: + super().__init__(yaml_data, code_model) + self.wire_name: str = yaml_data["wireName"] + self.type = type + + @property + def serialization_type(self) -> str: + return self.type.serialization_type + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "ResponseHeader": + from . import build_type + + return cls( + yaml_data=yaml_data, + code_model=code_model, + type=build_type(yaml_data["type"], code_model), + ) + + +class Response(BaseModel): + def __init__( + self, + yaml_data: Dict[str, Any], + code_model: "CodeModel", + *, + headers: Optional[List[ResponseHeader]] = None, + type: Optional[BaseType] = None, + ) -> None: + super().__init__(yaml_data=yaml_data, code_model=code_model) + self.status_codes: List[Union[int, str]] = yaml_data["statusCodes"] + self.headers = headers or [] + self.type = type + self.nullable = yaml_data.get("nullable") + self.default_content_type = yaml_data.get("defaultContentType") + + @property + def result_property(self) -> str: + field = self.yaml_data.get("resultProperty") + if field: + return f'.get("{field}")' + return "" + + def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: + if self.type: + self.type.get_polymorphic_subtypes(polymorphic_subtypes) + + def get_json_template_representation(self) -> Any: + if not self.type: + return None + if not isinstance(self.type, (DictionaryType, ListType, ModelType)): + return None + return self.type.get_json_template_representation() + + @property + def is_stream_response(self) -> bool: + """Is the response expected to be streamable, like a download.""" + retval = isinstance(self.type, BinaryIteratorType) or ( + isinstance(self.type, ByteArraySchema) + and bool(self.default_content_type) + and self.default_content_type != "application/json" + ) + return retval + + @property + def serialization_type(self) -> str: + if self.type: + return self.type.serialization_type + return "None" + + def type_annotation(self, **kwargs: Any) -> str: + if self.type: + kwargs["is_operation_file"] = True + type_annot = self.type.type_annotation(**kwargs) + if self.nullable: + return f"Optional[{type_annot}]" + return type_annot + return "None" + + def docstring_text(self, **kwargs: Any) -> str: + if self.nullable and self.type: + return f"{self.type.docstring_text(**kwargs)} or None" + return self.type.docstring_text(**kwargs) if self.type else "None" + + def docstring_type(self, **kwargs: Any) -> str: + if self.nullable and self.type: + return f"{self.type.docstring_type(**kwargs)} or None" + return self.type.docstring_type(**kwargs) if self.type else "None" + + def _imports_shared(self, **kwargs: Any) -> FileImport: + file_import = FileImport(self.code_model) + if self.type: + file_import.merge(self.type.imports(**kwargs)) + if self.nullable: + file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) + if isinstance(self.type, CombinedType) and self.type.name: + async_mode = kwargs.get("async_mode", False) + file_import.add_submodule_import( + "..." if async_mode else "..", + "_types", + ImportType.LOCAL, + TypingSection.TYPING, + ) + return file_import + + def imports(self, **kwargs: Any) -> FileImport: + return self._imports_shared(**kwargs) + + def imports_for_multiapi(self, **kwargs: Any) -> FileImport: + return self._imports_shared(**kwargs) + + def _get_import_type(self, input_path: str) -> ImportType: + # helper function to return imports for responses based off + # of whether we're importing from the core library, or users + # are customizing responses + return ImportType.SDKCORE if self.code_model.core_library.split(".")[0] in input_path else ImportType.THIRDPARTY + + @classmethod + def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "Response": + type = code_model.lookup_type(id(yaml_data["type"])) if yaml_data.get("type") else None + # use ByteIteratorType if we are returning a binary type + default_content_type = yaml_data.get("defaultContentType", "application/json") + if isinstance(type, BinaryType) or ( + isinstance(type, ByteArraySchema) and default_content_type != "application/json" + ): + type = BinaryIteratorType(type.yaml_data, type.code_model) + return cls( + yaml_data=yaml_data, + code_model=code_model, + headers=[ResponseHeader.from_yaml(header, code_model) for header in yaml_data["headers"]], + type=type, + ) + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} {self.status_codes}>" + + +class PagingResponse(Response): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.item_type = self.code_model.lookup_type(id(self.yaml_data["itemType"])) + self.pager_sync: str = self.yaml_data.get("pagerSync") or f"{self.code_model.core_library}.paging.ItemPaged" + default_paging_submodule = f"{'async_' if self.code_model.is_azure_flavor else ''}paging" + self.pager_async: str = ( + self.yaml_data.get("pagerAsync") + or f"{self.code_model.core_library}.{default_paging_submodule}.AsyncItemPaged" + ) + + def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: + return self.item_type.get_polymorphic_subtypes(polymorphic_subtypes) + + def get_json_template_representation(self) -> Any: + return self.item_type.get_json_template_representation() + + def get_pager_import_path(self, async_mode: bool) -> str: + return ".".join(self.get_pager_path(async_mode).split(".")[:-1]) + + def get_pager_path(self, async_mode: bool) -> str: + return self.pager_async if async_mode else self.pager_sync + + def get_pager(self, async_mode: bool) -> str: + return self.get_pager_path(async_mode).split(".")[-1] + + def type_annotation(self, **kwargs: Any) -> str: + iterable = "AsyncIterable" if kwargs["async_mode"] else "Iterable" + return f"{iterable}[{self.item_type.type_annotation(**kwargs)}]" + + def docstring_text(self, **kwargs: Any) -> str: + base_description = "An iterator like instance of " + if not self.code_model.options["version_tolerant"]: + base_description += "either " + return base_description + self.item_type.docstring_text(**kwargs) + + def docstring_type(self, **kwargs: Any) -> str: + return f"~{self.get_pager_path(kwargs['async_mode'])}[{self.item_type.docstring_type(**kwargs)}]" + + def _imports_shared(self, **kwargs: Any) -> FileImport: + file_import = super()._imports_shared(**kwargs) + async_mode = kwargs.get("async_mode", False) + pager = self.get_pager(async_mode) + pager_path = self.get_pager_import_path(async_mode) + + file_import.add_submodule_import(pager_path, pager, self._get_import_type(pager_path)) + return file_import + + def imports(self, **kwargs: Any) -> FileImport: + file_import = self._imports_shared(**kwargs) + async_mode = kwargs.get("async_mode") + if async_mode: + file_import.add_submodule_import( + f"{'async_' if self.code_model.is_azure_flavor else ''}paging", + "AsyncList", + ImportType.SDKCORE, + ) + + return file_import + + def imports_for_multiapi(self, **kwargs: Any) -> FileImport: + return self._imports_shared(**kwargs) + + +class LROResponse(Response): + def get_poller_path(self, async_mode: bool) -> str: + return self.yaml_data["pollerAsync"] if async_mode else self.yaml_data["pollerSync"] + + def get_poller(self, async_mode: bool) -> str: + """Get the name of the poller. Default is LROPoller / AsyncLROPoller""" + return self.get_poller_path(async_mode).split(".")[-1] + + def get_polling_method_path(self, async_mode: bool) -> str: + """Get the full name of the poller path. Default are the azure core pollers""" + return self.yaml_data["pollingMethodAsync"] if async_mode else self.yaml_data["pollingMethodSync"] + + def get_polling_method(self, async_mode: bool) -> str: + """Get the default pollint method""" + return self.get_polling_method_path(async_mode).split(".")[-1] + + @staticmethod + def get_no_polling_method_path(async_mode: bool) -> str: + """Get the path of the default of no polling method""" + return f"azure.core.polling.{'Async' if async_mode else ''}NoPolling" + + def get_no_polling_method(self, async_mode: bool) -> str: + """Get the default no polling method""" + return self.get_no_polling_method_path(async_mode).split(".")[-1] + + @staticmethod + def get_base_polling_method_path(async_mode: bool) -> str: + """Get the base polling method path. Used in docstrings and type annotations.""" + return f"azure.core.polling.{'Async' if async_mode else ''}PollingMethod" + + def get_base_polling_method(self, async_mode: bool) -> str: + """Get the base polling method.""" + return self.get_base_polling_method_path(async_mode).split(".")[-1] + + def type_annotation(self, **kwargs: Any) -> str: + return f"{self.get_poller(kwargs.get('async_mode', False))}[{super().type_annotation(**kwargs)}]" + + def docstring_type(self, **kwargs: Any) -> str: + return f"~{self.get_poller_path(kwargs.get('async_mode', False))}[{super().docstring_type(**kwargs)}]" + + def docstring_text(self, **kwargs) -> str: + super_text = super().docstring_text(**kwargs) + base_description = f"An instance of {self.get_poller(kwargs.get('async_mode', False))} that returns " + if not self.code_model.options["version_tolerant"]: + base_description += "either " + return base_description + super_text + + def _imports_shared(self, **kwargs: Any) -> FileImport: + file_import = super()._imports_shared(**kwargs) + async_mode = kwargs["async_mode"] + poller_import_path = ".".join(self.get_poller_path(async_mode).split(".")[:-1]) + poller = self.get_poller(async_mode) + file_import.add_submodule_import(poller_import_path, poller, self._get_import_type(poller_import_path)) + return file_import + + def imports(self, **kwargs: Any) -> FileImport: + file_import = self._imports_shared(**kwargs) + async_mode = kwargs["async_mode"] + + default_polling_method_import_path = ".".join(self.get_polling_method_path(async_mode).split(".")[:-1]) + default_polling_method = self.get_polling_method(async_mode) + file_import.add_submodule_import( + default_polling_method_import_path, + default_polling_method, + self._get_import_type(default_polling_method_import_path), + ) + default_no_polling_method_import_path = ".".join(self.get_no_polling_method_path(async_mode).split(".")[:-1]) + default_no_polling_method = self.get_no_polling_method(async_mode) + file_import.add_submodule_import( + default_no_polling_method_import_path, + default_no_polling_method, + self._get_import_type(default_no_polling_method_import_path), + ) + + base_polling_method_import_path = ".".join(self.get_base_polling_method_path(async_mode).split(".")[:-1]) + base_polling_method = self.get_base_polling_method(async_mode) + file_import.add_submodule_import( + base_polling_method_import_path, + base_polling_method, + self._get_import_type(base_polling_method_import_path), + ) + return file_import + + def imports_for_multiapi(self, **kwargs: Any) -> FileImport: + return self._imports_shared(**kwargs) + + +class LROPagingResponse(LROResponse, PagingResponse): + def type_annotation(self, **kwargs: Any) -> str: + paging_type_annotation = PagingResponse.type_annotation(self, **kwargs) + return f"{self.get_poller(kwargs.get('async_mode', False))}[{paging_type_annotation}]" + + def docstring_type(self, **kwargs: Any) -> str: + paging_docstring_type = PagingResponse.docstring_type(self, **kwargs) + return f"~{self.get_poller_path(kwargs.get('async_mode', False))}[{paging_docstring_type}]" + + def docstring_text(self, **kwargs) -> str: + base_description = "An instance of LROPoller that returns an iterator like instance of " + if not self.code_model.options["version_tolerant"]: + base_description += "either " + return base_description + Response.docstring_text(self) + + def imports_for_multiapi(self, **kwargs: Any) -> FileImport: + file_import = LROResponse.imports_for_multiapi(self, **kwargs) + file_import.merge(PagingResponse.imports_for_multiapi(self, **kwargs)) + return file_import + + def imports(self, **kwargs: Any) -> FileImport: + file_import = LROResponse.imports(self, **kwargs) + file_import.merge(PagingResponse.imports(self, **kwargs)) + return file_import + + +def get_response(yaml_data: Dict[str, Any], code_model: "CodeModel") -> Response: + if yaml_data["discriminator"] == "lropaging": + return LROPagingResponse.from_yaml(yaml_data, code_model) + if yaml_data["discriminator"] == "lro": + return LROResponse.from_yaml(yaml_data, code_model) + if yaml_data["discriminator"] == "paging": + return PagingResponse.from_yaml(yaml_data, code_model) + return Response.from_yaml(yaml_data, code_model) diff --git a/packages/autorest.python/generator/pygen/codegen/models/utils.py b/packages/autorest.python/generator/pygen/codegen/models/utils.py new file mode 100644 index 00000000000..374d659135a --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/models/utils.py @@ -0,0 +1,23 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import TypeVar, Dict + +T = TypeVar("T") +OrderedSet = Dict[T, None] + +NAME_LENGTH_LIMIT = 40 + + +def add_to_description(description: str, entry: str) -> str: + if description: + return f"{description} {entry}" + return entry + + +def add_to_pylint_disable(curr_str: str, entry: str) -> str: + if curr_str: + return f"{curr_str},{entry}" + return f" # pylint: disable={entry}" diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/__init__.py b/packages/autorest.python/generator/pygen/codegen/serializers/__init__.py new file mode 100644 index 00000000000..ee3d27ff8ad --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/__init__.py @@ -0,0 +1,570 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from typing import List, Optional, Any, Union +from pathlib import Path +from jinja2 import PackageLoader, Environment, FileSystemLoader, StrictUndefined + +from ... import ReaderAndWriter +from ..models import ( + OperationGroup, + RequestBuilder, + OverloadedRequestBuilder, + CodeModel, + Client, +) +from .enum_serializer import EnumSerializer +from .general_serializer import GeneralSerializer +from .model_init_serializer import ModelInitSerializer +from .model_serializer import DpgModelSerializer, MsrestModelSerializer +from .operations_init_serializer import OperationsInitSerializer +from .operation_groups_serializer import OperationGroupsSerializer +from .metadata_serializer import MetadataSerializer +from .request_builders_serializer import RequestBuildersSerializer +from .patch_serializer import PatchSerializer +from .sample_serializer import SampleSerializer +from .test_serializer import TestSerializer, TestGeneralSerializer +from .types_serializer import TypesSerializer +from ...utils import to_snake_case +from .._utils import VALID_PACKAGE_MODE +from .utils import ( + extract_sample_name, + get_namespace_from_package_name, + get_namespace_config, + get_all_operation_groups_recursively, +) + +_LOGGER = logging.getLogger(__name__) + +__all__ = [ + "JinjaSerializer", +] + +_PACKAGE_FILES = [ + "CHANGELOG.md.jinja2", + "dev_requirements.txt.jinja2", + "LICENSE.jinja2", + "MANIFEST.in.jinja2", + "README.md.jinja2", + "setup.py.jinja2", +] + +_REGENERATE_FILES = {"setup.py", "MANIFEST.in"} + + +# extract sub folders. For example, source_file_path is like: +# "xxx/resource-manager/Microsoft.XX/stable/2023-04-01/examples/Compute/createOrUpdate/AKSCompute.json", +# and we want to extract the sub folders after "examples/", which is "compute/create_or_update" +def _sample_output_path(source_file_path: str) -> Path: + posix_path = Path(source_file_path).as_posix() + if "examples/" in posix_path: + after_examples = Path(posix_path.split("examples/", maxsplit=1)[-1]).parent + return Path("/".join([to_snake_case(i) for i in after_examples.parts])) + return Path("") + + +class JinjaSerializer(ReaderAndWriter): # pylint: disable=abstract-method + def __init__( + self, + code_model: CodeModel, + *, + output_folder: Union[str, Path], + **kwargs: Any, + ) -> None: + super().__init__(output_folder=output_folder, **kwargs) + self.code_model = code_model + + @property + def has_aio_folder(self) -> bool: + return not self.code_model.options["no_async"] and bool(self.code_model.has_operations) + + @property + def has_operations_folder(self) -> bool: + return self.code_model.options["show_operations"] and bool(self.code_model.has_operations) + + def _serialize_namespace_level(self, env: Environment, namespace_path: Path, clients: List[Client]) -> None: + # if there was a patch file before, we keep it + self._keep_patch_file(namespace_path / Path("_patch.py"), env) + if self.has_aio_folder: + self._keep_patch_file(namespace_path / Path("aio") / Path("_patch.py"), env) + + if self.has_operations_folder: + self._keep_patch_file( + namespace_path / Path(self.code_model.operations_folder_name) / Path("_patch.py"), + env, + ) + if self.has_aio_folder: + self._keep_patch_file( + namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path("_patch.py"), + env, + ) + self._serialize_and_write_top_level_folder(env=env, namespace_path=namespace_path, clients=clients) + + if any(c for c in self.code_model.clients if c.operation_groups): + if self.code_model.options["builders_visibility"] != "embedded": + self._serialize_and_write_rest_layer(env=env, namespace_path=namespace_path) + if self.has_aio_folder: + self._serialize_and_write_aio_top_level_folder( + env=env, + namespace_path=namespace_path, + clients=clients, + ) + + if self.has_operations_folder: + self._serialize_and_write_operations_folder(clients, env=env, namespace_path=namespace_path) + if self.code_model.options["multiapi"]: + self._serialize_and_write_metadata(env=env, namespace_path=namespace_path) + if self.code_model.options["package_mode"]: + self._serialize_and_write_package_files(namespace_path=namespace_path) + + if ( + self.code_model.options["show_operations"] + and self.code_model.has_operations + and self.code_model.options["generate_sample"] + ): + self._serialize_and_write_sample(env, namespace_path) + + if ( + self.code_model.options["show_operations"] + and self.code_model.has_operations + and self.code_model.options["generate_test"] + and not self.code_model.options["azure_arm"] + ): + self._serialize_and_write_test(env, namespace_path) + + def serialize(self) -> None: + env = Environment( + loader=PackageLoader("pygen.codegen", "templates"), + keep_trailing_newline=True, + line_statement_prefix="##", + line_comment_prefix="###", + trim_blocks=True, + lstrip_blocks=True, + ) + + namespace_path = ( + Path(".") if self.code_model.options["no_namespace_folders"] else Path(*self._name_space().split(".")) + ) + + p = namespace_path.parent + general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) + while p != Path("."): + # write pkgutil init file + self.write_file( + p / Path("__init__.py"), + general_serializer.serialize_pkgutil_init_file(), + ) + p = p.parent + + # serialize main module + self._serialize_namespace_level( + env, + namespace_path, + [c for c in self.code_model.clients if c.has_operations], + ) + # serialize sub modules + for ( + subnamespace, + clients, + ) in self.code_model.subnamespace_to_clients.items(): + subnamespace_path = namespace_path / Path(subnamespace) + self._serialize_namespace_level(env, subnamespace_path, [c for c in clients if c.has_operations]) + + if self.code_model.options["models_mode"] and (self.code_model.model_types or self.code_model.enums): + self._keep_patch_file(namespace_path / Path("models") / Path("_patch.py"), env) + + if self.code_model.options["models_mode"] and (self.code_model.model_types or self.code_model.enums): + self._serialize_and_write_models_folder(env=env, namespace_path=namespace_path) + if not self.code_model.options["models_mode"]: + # keep models file if users ended up just writing a models file + if self.read_file(namespace_path / Path("models.py")): + self.write_file( + namespace_path / Path("models.py"), + self.read_file(namespace_path / Path("models.py")), + ) + if self.code_model.named_unions: + self.write_file( + namespace_path / Path("_types.py"), + TypesSerializer(code_model=self.code_model, env=env).serialize(), + ) + + def _serialize_and_write_package_files(self, namespace_path: Path) -> None: + root_of_sdk = self._package_root_folder(namespace_path) + if self.code_model.options["package_mode"] in VALID_PACKAGE_MODE: + env = Environment( + loader=PackageLoader("pygen.codegen", "templates/packaging_templates"), + undefined=StrictUndefined, + ) + + package_files = _PACKAGE_FILES + elif Path(self.code_model.options["package_mode"]).exists(): + env = Environment( + loader=FileSystemLoader(str(Path(self.code_model.options["package_mode"]))), + keep_trailing_newline=True, + undefined=StrictUndefined, + ) + package_files = env.list_templates() + else: + return + serializer = GeneralSerializer(self.code_model, env, async_mode=False) + params = self.code_model.options["packaging_files_config"] or {} + for template_name in package_files: + if not self.code_model.is_azure_flavor and template_name == "dev_requirements.txt.jinja2": + continue + file = template_name.replace(".jinja2", "") + output_name = root_of_sdk / file + if not self.read_file(output_name) or file in _REGENERATE_FILES: + self.write_file( + output_name, + serializer.serialize_package_file(template_name, **params), + ) + + def _keep_patch_file(self, path_file: Path, env: Environment): + if self.read_file(path_file): + self.write_file(path_file, self.read_file(path_file)) + else: + self.write_file( + path_file, + PatchSerializer(env=env, code_model=self.code_model).serialize(), + ) + + def _serialize_and_write_models_folder(self, env: Environment, namespace_path: Path) -> None: + # Write the models folder + models_path = namespace_path / Path("models") + serializer = DpgModelSerializer if self.code_model.options["models_mode"] == "dpg" else MsrestModelSerializer + if self.code_model.model_types: + self.write_file( + models_path / Path(f"{self.code_model.models_filename}.py"), + serializer(code_model=self.code_model, env=env).serialize(), + ) + if self.code_model.enums: + self.write_file( + models_path / Path(f"{self.code_model.enums_filename}.py"), + EnumSerializer(code_model=self.code_model, env=env).serialize(), + ) + self.write_file( + models_path / Path("__init__.py"), + ModelInitSerializer(code_model=self.code_model, env=env).serialize(), + ) + + def _serialize_and_write_rest_layer(self, env: Environment, namespace_path: Path) -> None: + rest_path = namespace_path / Path(self.code_model.rest_layer_name) + group_names = {rb.group_name for c in self.code_model.clients for rb in c.request_builders} + + for group_name in group_names: + request_builders = [ + r for c in self.code_model.clients for r in c.request_builders if r.group_name == group_name + ] + self._serialize_and_write_single_rest_layer(env, rest_path, request_builders) + if not "" in group_names: + self.write_file( + rest_path / Path("__init__.py"), + self.code_model.options["license_header"], + ) + + def _serialize_and_write_single_rest_layer( + self, + env: Environment, + rest_path: Path, + request_builders: List[Union[RequestBuilder, OverloadedRequestBuilder]], + ) -> None: + group_name = request_builders[0].group_name + output_path = rest_path / Path(group_name) if group_name else rest_path + # write generic request builders file + self.write_file( + output_path / Path("_request_builders.py"), + RequestBuildersSerializer( + code_model=self.code_model, + env=env, + request_builders=request_builders, + ).serialize_request_builders(), + ) + + # write rest init file + self.write_file( + output_path / Path("__init__.py"), + RequestBuildersSerializer( + code_model=self.code_model, + env=env, + request_builders=request_builders, + ).serialize_init(), + ) + + def _serialize_and_write_operations_file( + self, + env: Environment, + clients: List[Client], + namespace_path: Path, + operation_group: Optional[OperationGroup] = None, + ) -> None: + filename = operation_group.filename if operation_group else "_operations" + # write first sync file + operation_group_serializer = OperationGroupsSerializer( + code_model=self.code_model, + clients=clients, + env=env, + async_mode=False, + operation_group=operation_group, + ) + self.write_file( + namespace_path / Path(self.code_model.operations_folder_name) / Path(f"{filename}.py"), + operation_group_serializer.serialize(), + ) + + if self.has_aio_folder: + # write async operation group and operation files + operation_group_async_serializer = OperationGroupsSerializer( + code_model=self.code_model, + clients=clients, + env=env, + async_mode=True, + operation_group=operation_group, + ) + self.write_file( + (namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path(f"{filename}.py")), + operation_group_async_serializer.serialize(), + ) + + def _serialize_and_write_operations_folder( + self, clients: List[Client], env: Environment, namespace_path: Path + ) -> None: + # write sync operations init file + operations_init_serializer = OperationsInitSerializer( + code_model=self.code_model, clients=clients, env=env, async_mode=False + ) + self.write_file( + namespace_path / Path(self.code_model.operations_folder_name) / Path("__init__.py"), + operations_init_serializer.serialize(), + ) + + # write async operations init file + if self.has_aio_folder: + operations_async_init_serializer = OperationsInitSerializer( + code_model=self.code_model, clients=clients, env=env, async_mode=True + ) + self.write_file( + namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path("__init__.py"), + operations_async_init_serializer.serialize(), + ) + + if self.code_model.options["combine_operation_files"]: + self._serialize_and_write_operations_file( + env=env, + namespace_path=namespace_path, + clients=clients, + ) + else: + for operation_group in get_all_operation_groups_recursively(self.code_model.clients): + self._serialize_and_write_operations_file( + env=env, + namespace_path=namespace_path, + operation_group=operation_group, + clients=clients, + ) + + def _serialize_and_write_version_file( + self, + namespace_path: Path, + general_serializer: GeneralSerializer, + ): + def _read_version_file(original_version_file_name: str) -> str: + return self.read_file(namespace_path / original_version_file_name) + + def _write_version_file(original_version_file_name: str) -> None: + self.write_file( + namespace_path / Path("_version.py"), + _read_version_file(original_version_file_name), + ) + + keep_version_file = self.code_model.options["keep_version_file"] + if keep_version_file and _read_version_file("_version.py"): + _write_version_file(original_version_file_name="_version.py") + elif keep_version_file and _read_version_file("version.py"): + _write_version_file(original_version_file_name="version.py") + elif self.code_model.options["package_version"]: + self.write_file( + namespace_path / Path("_version.py"), + general_serializer.serialize_version_file(), + ) + + def _serialize_client_and_config_files( + self, + namespace_path: Path, + general_serializer: GeneralSerializer, + async_mode: bool, + clients: List[Client], + ) -> None: + if self.code_model.has_operations: + namespace_path = namespace_path / Path("aio") if async_mode else namespace_path + self.write_file( + namespace_path / Path(f"{self.code_model.client_filename}.py"), + general_serializer.serialize_service_client_file(clients), + ) + self.write_file( + namespace_path / Path("_configuration.py"), + general_serializer.serialize_config_file(clients), + ) + + def _serialize_and_write_top_level_folder( + self, env: Environment, namespace_path: Path, clients: List[Client] + ) -> None: + general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) + + self.write_file( + namespace_path / Path("__init__.py"), + general_serializer.serialize_init_file(clients), + ) + + # Write the service client + self._serialize_client_and_config_files(namespace_path, general_serializer, async_mode=False, clients=clients) + if self.code_model.need_vendored_code(async_mode=False): + self.write_file( + namespace_path / Path("_vendor.py"), + general_serializer.serialize_vendor_file(clients), + ) + + self._serialize_and_write_version_file(namespace_path, general_serializer) + + # write the empty py.typed file + self.write_file(namespace_path / Path("py.typed"), "# Marker file for PEP 561.") + + if not self.code_model.options["client_side_validation"] and not self.code_model.options["multiapi"]: + self.write_file( + namespace_path / Path("_serialization.py"), + general_serializer.serialize_serialization_file(), + ) + if self.code_model.options["models_mode"] == "dpg": + self.write_file( + namespace_path / Path("_model_base.py"), + general_serializer.serialize_model_base_file(), + ) + + if any(og for client in self.code_model.clients for og in client.operation_groups if og.need_validation): + self.write_file( + namespace_path / Path("_validation.py"), + general_serializer.serialize_validation_file(), + ) + if self.code_model.options.get("emit_cross_language_definition_file"): + self.write_file( + Path("./apiview_mapping_python.json"), + general_serializer.serialize_cross_language_definition_file(), + ) + + # Write the setup file + if self.code_model.options["basic_setup_py"]: + self.write_file(Path("setup.py"), general_serializer.serialize_setup_file()) + + def _serialize_and_write_aio_top_level_folder( + self, env: Environment, namespace_path: Path, clients: List[Client] + ) -> None: + aio_general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=True) + + aio_path = namespace_path / Path("aio") + + # Write the __init__ file + self.write_file( + aio_path / Path("__init__.py"), + aio_general_serializer.serialize_init_file(clients), + ) + + # Write the service client + self._serialize_client_and_config_files( + namespace_path, aio_general_serializer, async_mode=True, clients=clients + ) + if self.code_model.need_vendored_code(async_mode=True): + self.write_file( + aio_path / Path("_vendor.py"), + aio_general_serializer.serialize_vendor_file(clients), + ) + + def _serialize_and_write_metadata(self, env: Environment, namespace_path: Path) -> None: + metadata_serializer = MetadataSerializer(self.code_model, env) + self.write_file(namespace_path / Path("_metadata.json"), metadata_serializer.serialize()) + + @property + def _namespace_from_package_name(self) -> str: + return get_namespace_from_package_name(self.code_model.options["package_name"]) + + def _name_space(self) -> str: + if self.code_model.namespace.count(".") >= self._namespace_from_package_name.count("."): + return self.code_model.namespace + + return self._namespace_from_package_name + + # find root folder where "setup.py" is + def _package_root_folder(self, namespace_path: Path) -> Path: + return namespace_path / Path("../" * (self._name_space().count(".") + 1)) + + @property + def _additional_folder(self) -> Path: + namespace_config = get_namespace_config(self.code_model.namespace, self.code_model.options["multiapi"]) + num_of_namespace = namespace_config.count(".") + 1 + num_of_package_namespace = self._namespace_from_package_name.count(".") + 1 + if num_of_namespace > num_of_package_namespace: + return Path("/".join(namespace_config.split(".")[num_of_package_namespace:])) + return Path("") + + def _serialize_and_write_sample(self, env: Environment, namespace_path: Path): + out_path = self._package_root_folder(namespace_path) / Path("generated_samples") + for client in self.code_model.clients: + for op_group in client.operation_groups: + for operation in op_group.operations: + if ( + self.code_model.options["multiapi"] + and operation.api_versions[0] != self.code_model.options["default_api_version"] + ): + continue + samples = operation.yaml_data["samples"] + if not samples or operation.name.startswith("_"): + continue + for value in samples.values(): + file = value.get("x-ms-original-file", "sample.json") + file_name = to_snake_case(extract_sample_name(file)) + ".py" + try: + self.write_file( + out_path / self._additional_folder / _sample_output_path(file) / file_name, + SampleSerializer( + code_model=self.code_model, + env=env, + operation_group=op_group, + operation=operation, + sample=value, + file_name=file_name, + ).serialize(), + ) + except Exception as e: # pylint: disable=broad-except + # sample generation shall not block code generation, so just log error + log_error = f"error happens in sample {file}: {e}" + _LOGGER.error(log_error) + + def _serialize_and_write_test(self, env: Environment, namespace_path: Path): + self.code_model.for_test = True + out_path = self._package_root_folder(namespace_path) / Path("generated_tests") + general_serializer = TestGeneralSerializer(code_model=self.code_model, env=env) + self.write_file(out_path / "conftest.py", general_serializer.serialize_conftest()) + for is_async in (True, False): + async_suffix = "_async" if is_async else "" + general_serializer.is_async = is_async + self.write_file( + out_path / f"testpreparer{async_suffix}.py", + general_serializer.serialize_testpreparer(), + ) + + for client in self.code_model.clients: + for og in client.operation_groups: + test_serializer = TestSerializer(self.code_model, env, client=client, operation_group=og) + for is_async in (True, False): + try: + test_serializer.is_async = is_async + self.write_file( + out_path / f"{to_snake_case(test_serializer.test_class_name)}.py", + test_serializer.serialize_test(), + ) + except Exception as e: # pylint: disable=broad-except + # test generation shall not block code generation, so just log error + log_error = f"error happens in test generation for operation group {og.class_name}: {e}" + _LOGGER.error(log_error) + self.code_model.for_test = False diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/base_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/base_serializer.py new file mode 100644 index 00000000000..0ac623166a1 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/base_serializer.py @@ -0,0 +1,21 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from jinja2 import Environment +from ..models import ( + FileImport, + CodeModel, +) + + +class BaseSerializer: + """Base serializer for SDK root level files""" + + def __init__(self, code_model: CodeModel, env: Environment): + self.code_model = code_model + self.env = env + + def init_file_import(self) -> FileImport: + return FileImport(self.code_model) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/builder_serializer.py new file mode 100644 index 00000000000..007661f9116 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/builder_serializer.py @@ -0,0 +1,1453 @@ +# pylint: disable=too-many-lines,multiple-statements +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from abc import abstractmethod +from collections import defaultdict +from typing import Generic, List, Type, TypeVar, Dict, Union, Optional, cast + +from ..models import ( + Operation, + PagingOperation, + CodeModel, + LROOperation, + LROPagingOperation, + ModelType, + DictionaryType, + ListType, + RequestBuilder, + ParameterLocation, + Response, + BinaryType, + BodyParameter, + ParameterMethodLocation, + RequestBuilderBodyParameter, + OverloadedRequestBuilder, + Property, + RequestBuilderType, + CombinedType, + JSONModelType, + DPGModelType, + ParameterListType, + ByteArraySchema, +) +from .parameter_serializer import ParameterSerializer, PopKwargType +from ..models.parameter_list import ParameterType +from . import utils +from ...utils import JSON_REGEXP + +T = TypeVar("T") +OrderedSet = Dict[T, None] + +BuilderType = TypeVar( + "BuilderType", + bound=Union[ + RequestBuilder, + Operation, + PagingOperation, + LROOperation, + LROPagingOperation, + OverloadedRequestBuilder, + ], +) +OperationType = TypeVar( + "OperationType", + bound=Union[Operation, PagingOperation, LROOperation, LROPagingOperation], +) + + +def _json_serializable(content_type: str) -> bool: + return bool(JSON_REGEXP.match(content_type.split(";")[0].strip().lower())) + + +def _need_type_ignore(builder: OperationType) -> bool: + for excep in builder.non_default_errors: + for status_code in excep.status_codes: + if status_code in (401, 404, 409, 304): + return True + return False + + +def _xml_config(send_xml: bool, content_types: List[str]) -> str: + if not (send_xml and "xml" in str(content_types)): + return "" + if len(content_types) == 1: + return ", is_xml=True" + return ", is_xml='xml' in str(content_type)" + + +def _escape_str(input_str: str) -> str: + replace = input_str.replace("'", "\\'") + return f'"{replace}"' + + +def _get_polymorphic_subtype_template(polymorphic_subtype: ModelType) -> List[str]: + retval: List[str] = [] + retval.append("") + retval.append(f'# JSON input template for discriminator value "{polymorphic_subtype.discriminator_value}":') + subtype_template = utils.json_dumps_template( + polymorphic_subtype.get_json_template_representation(), + ) + + def _get_polymorphic_parent( + polymorphic_subtype: Optional[ModelType], + ) -> Optional[ModelType]: + if not polymorphic_subtype: + return None + try: + return next(p for p in polymorphic_subtype.parents if p.discriminated_subtypes) + except StopIteration: + return None + + polymorphic_parent = _get_polymorphic_parent(polymorphic_subtype) + while _get_polymorphic_parent(polymorphic_parent): + polymorphic_parent = _get_polymorphic_parent(polymorphic_parent) + retval.extend(f"{cast(ModelType, polymorphic_parent).snake_case_name} = {subtype_template}".splitlines()) + return retval + + +def _serialize_grouped_body(builder: BuilderType) -> List[str]: + retval: List[str] = [] + for grouped_parameter in builder.parameters.grouped: + retval.append(f"{grouped_parameter.client_name} = None") + groupers = [p for p in builder.parameters if p.grouper] + for grouper in groupers: + retval.append(f"if {grouper.client_name} is not None:") + retval.extend( + [ + f" {parameter} = {grouper.client_name}.{property}" + for property, parameter in grouper.property_to_parameter_name.items() + ] + ) + return retval + + +def _serialize_flattened_body(body_parameter: BodyParameter) -> List[str]: + retval: List[str] = [] + if not body_parameter.property_to_parameter_name: + raise ValueError("This method can't be called if the operation doesn't need parameter flattening") + + parameter_string = ", ".join( + f"{property_name}={parameter_name}" + for property_name, parameter_name in body_parameter.property_to_parameter_name.items() + ) + model_type = cast(ModelType, body_parameter.type) + retval.append(f"{body_parameter.client_name} = _models.{model_type.name}({parameter_string})") + return retval + + +def _serialize_json_model_body(body_parameter: BodyParameter, parameters: List[ParameterType]) -> List[str]: + retval: List[str] = [] + if not body_parameter.property_to_parameter_name: + raise ValueError("This method can't be called if the operation doesn't need parameter flattening") + + retval.append(f"if {body_parameter.client_name} is _Unset:") + for p in parameters: + if p.client_default_value is None and not p.optional and p.default_to_unset_sentinel: + retval.append(f" if {p.client_name} is _Unset:") + retval.append(f" raise TypeError('missing required argument: {p.client_name}')") + parameter_string = ", \n".join( + f'"{property_name}": {parameter_name}' + for property_name, parameter_name in body_parameter.property_to_parameter_name.items() + ) + model_type = cast(ModelType, body_parameter.type) + if isinstance(model_type, CombinedType) and model_type.target_model_subtype((JSONModelType,)): + model_type = model_type.target_model_subtype((JSONModelType,)) + retval.append(f" {body_parameter.client_name} = {{{parameter_string}}}") + retval.append(f" {body_parameter.client_name} = {{") + retval.append(f" k: v for k, v in {body_parameter.client_name}.items() if v is not None") + retval.append(" }") + return retval + + +def _serialize_multipart_body(builder: BuilderType) -> List[str]: + retval: List[str] = [] + body_param = builder.parameters.body_parameter + # we have to construct our form data before passing to the request as well + retval.append("# Construct form data") + retval.append(f"_{body_param.client_name} = {{") + for param in body_param.entries: + retval.append(f' "{param.wire_name}": {param.client_name},') + retval.append("}") + return retval + + +def _get_json_response_template_to_status_codes( + builder: OperationType, +) -> Dict[str, List[str]]: + retval = defaultdict(list) + for response in builder.responses: + json_template = response.get_json_template_representation() + if not json_template: + continue + status_codes = [str(status_code) for status_code in response.status_codes] + response_json = utils.json_dumps_template(json_template) + retval[response_json].extend(status_codes) + return retval + + +def _api_version_validation(builder: OperationType) -> str: + retval: List[str] = [] + if builder.added_on: + retval.append(f' method_added_on="{builder.added_on}",') + params_added_on = defaultdict(list) + for parameter in builder.parameters: + if parameter.added_on: + params_added_on[parameter.added_on].append(parameter.client_name) + if params_added_on: + retval.append(f" params_added_on={dict(params_added_on)},") + if retval: + retval_str = "\n".join(retval) + return f"@api_version_validation(\n{retval_str}\n){builder.pylint_disable}" + return "" + + +def is_json_model_type(parameters: ParameterListType) -> bool: + return ( + parameters.has_body + and parameters.body_parameter.has_json_model_type + and any(p.in_flattened_body for p in parameters.parameters) + ) + + +class _BuilderBaseSerializer(Generic[BuilderType]): # pylint: disable=abstract-method + def __init__(self, code_model: CodeModel, async_mode: bool) -> None: + self.code_model = code_model + self.async_mode = async_mode + self.parameter_serializer = ParameterSerializer() + + @property + @abstractmethod + def _need_self_param(self) -> bool: ... + + @property + @abstractmethod + def _function_def(self) -> str: + """The def keyword for the builder we're serializing, i.e. 'def' or 'async def'""" + + @property + @abstractmethod + def _call_method(self) -> str: + """How to call network calls. Await if we have to await network calls""" + + @property + @abstractmethod + def serializer_name(self) -> str: + """Name of serializer""" + + @abstractmethod + def response_docstring(self, builder: BuilderType) -> List[str]: + """Response portion of the docstring""" + + def decorators(self, builder: BuilderType) -> List[str]: + """Decorators for the method""" + retval: List[str] = [] + if builder.is_overload: + return ["@overload"] + if self.code_model.options["tracing"] and builder.want_tracing: + retval.append(f"@distributed_trace{'_async' if self.async_mode else ''}") + return retval + + def _method_signature(self, builder: BuilderType) -> str: + return self.parameter_serializer.serialize_method( + function_def=self._function_def, + method_name=builder.name, + need_self_param=self._need_self_param, + method_param_signatures=builder.method_signature(self.async_mode), + pylint_disable=builder.pylint_disable, + ) + + def method_signature_and_response_type_annotation( + self, builder: BuilderType, *, want_decorators: Optional[bool] = True + ) -> str: + response_type_annotation = builder.response_type_annotation(async_mode=self.async_mode) + method_signature = self._method_signature(builder) + decorators = self.decorators(builder) + decorators_str = "" + if decorators and want_decorators: + decorators_str = "\n".join(decorators) + "\n" + return decorators_str + utils.method_signature_and_response_type_annotation_template( + method_signature=method_signature, + response_type_annotation=response_type_annotation, + ) + + def description_and_summary(self, builder: BuilderType) -> List[str]: + description_list: List[str] = [] + description_list.append(f"{builder.summary.strip() if builder.summary else builder.description.strip()}") + if builder.summary and builder.description: + description_list.append("") + description_list.append(builder.description.strip()) + description_list.append("") + return description_list + + @staticmethod + def line_too_long(docs: List[str]) -> bool: + return any(len(line) > 120 for line in docs) + + def example_template(self, builder: BuilderType) -> List[str]: + template = [] + if builder.abstract: + return [] + if self._json_input_example_template(builder): + template.append("") + template += self._json_input_example_template(builder) + return template + + def param_description(self, builder: BuilderType) -> List[str]: + description_list: List[str] = [] + for param in builder.parameters.method: + if ( + not param.in_docstring + or param.hide_in_operation_signature + or param.method_location == ParameterMethodLocation.KWARG + ): + continue + description_list.extend( + f":{param.description_keyword} {param.client_name}: {param.description}".replace("\n", "\n ").split( + "\n" + ) + ) + docstring_type = param.docstring_type( + async_mode=self.async_mode, + ) + description_list.append(f":{param.docstring_type_keyword} {param.client_name}: {docstring_type}") + return description_list + + def param_description_and_response_docstring(self, builder: BuilderType) -> List[str]: + if builder.abstract: + return [] + return self.param_description(builder) + self.response_docstring(builder) + + @property + @abstractmethod + def _json_response_template_name(self) -> str: ... + + def _json_input_example_template(self, builder: BuilderType) -> List[str]: + template: List[str] = [] + if not builder.parameters.has_body or builder.parameters.body_parameter.flattened: + # No input template if now body parameter + return template + + body_param = builder.parameters.body_parameter + if not isinstance(body_param.type, (ListType, DictionaryType, ModelType, CombinedType)): + return template + + if ( + isinstance(body_param.type, (ListType, DictionaryType)) + and self.code_model.options["models_mode"] == "msrest" + ): + return template + + if isinstance(body_param.type, ModelType) and body_param.type.base == "msrest": + return template + + json_type = body_param.type + if isinstance(body_param.type, CombinedType): + target_model_type = body_param.type.target_model_subtype((JSONModelType, DPGModelType)) + if target_model_type is None: + return template + json_type = target_model_type + + polymorphic_subtypes: List[ModelType] = [] + json_type.get_polymorphic_subtypes(polymorphic_subtypes) + if polymorphic_subtypes: + # we just assume one kind of polymorphic body for input + discriminator_name = cast(Property, polymorphic_subtypes[0].discriminator).wire_name + template.append( + "# The input is polymorphic. The following are possible polymorphic " + f'inputs based off discriminator "{discriminator_name}":' + ) + for idx in range( + min( + self.code_model.options["polymorphic_examples"], + len(polymorphic_subtypes), + ) + ): + template.extend(_get_polymorphic_subtype_template(polymorphic_subtypes[idx])) + template.append("") + template.append("# JSON input template you can fill out and use as your body input.") + json_template = utils.json_dumps_template( + json_type.get_json_template_representation(), + ) + template.extend(f"{builder.parameters.body_parameter.client_name} = {json_template}".splitlines()) + return template + + def serialize_path(self, builder: BuilderType) -> List[str]: + return self.parameter_serializer.serialize_path(builder.parameters.path, self.serializer_name) + + @property + def pipeline_name(self) -> str: + return f"{'_' if self.code_model.is_azure_flavor else ''}pipeline" + + +############################## REQUEST BUILDERS ############################## + + +class RequestBuilderSerializer(_BuilderBaseSerializer[RequestBuilderType]): # pylint: disable=abstract-method + def description_and_summary(self, builder: RequestBuilderType) -> List[str]: + retval = super().description_and_summary(builder) + retval += [ + "See https://aka.ms/azsdk/dpcodegen/python/send_request for how to incorporate this " + "request builder into your code flow.", + "", + ] + return retval + + @property + def _call_method(self) -> str: + return "" + + @property + def serializer_name(self) -> str: + return "_SERIALIZER" + + @property + def _json_response_template_name(self) -> str: + return "response.json()" + + @staticmethod + def declare_non_inputtable_constants(builder: RequestBuilderType) -> List[str]: + def _get_value(param): + if param.location in [ParameterLocation.HEADER, ParameterLocation.QUERY]: + kwarg_dict = "headers" if param.location == ParameterLocation.HEADER else "params" + return f"_{kwarg_dict}.pop('{param.wire_name}', {param.get_declaration()})" + return f"{param.get_declaration()}" + + return [f"{p.client_name} = {_get_value(p)}" for p in builder.parameters.constant if not p.in_method_signature] + + @property + def _function_def(self) -> str: + return "def" + + @property + def _need_self_param(self) -> bool: + return False + + def response_docstring(self, builder: RequestBuilderType) -> List[str]: + request_full_path = f"{self.code_model.core_library}.rest.HttpRequest" + response_str = ( + f":return: Returns an :class:`~{request_full_path}` that you will pass to the client's " + + "`send_request` method. See https://aka.ms/azsdk/dpcodegen/python/send_request for how to " + + "incorporate this response into your code flow." + ) + rtype_str = f":rtype: ~{request_full_path}" + return [response_str, rtype_str] + + def pop_kwargs_from_signature(self, builder: RequestBuilderType) -> List[str]: + return self.parameter_serializer.pop_kwargs_from_signature( + builder.parameters.kwargs_to_pop, + check_kwarg_dict=True, + pop_headers_kwarg=(PopKwargType.CASE_INSENSITIVE if bool(builder.parameters.headers) else PopKwargType.NO), + pop_params_kwarg=(PopKwargType.CASE_INSENSITIVE if bool(builder.parameters.query) else PopKwargType.NO), + ) + + @staticmethod + def create_http_request(builder: RequestBuilderType) -> List[str]: + retval = ["return HttpRequest("] + retval.append(f' method="{builder.method}",') + retval.append(" url=_url,") + if builder.parameters.query: + retval.append(" params=_params,") + if builder.parameters.headers: + retval.append(" headers=_headers,") + if builder.parameters.has_body and builder.parameters.body_parameter.in_method_signature: + body_param = builder.parameters.body_parameter + if body_param.constant or body_param.method_location != ParameterMethodLocation.KWARG: + # we only need to pass it through if it's not a kwarg or it's a popped kwarg + retval.append( + f" {builder.parameters.body_parameter.client_name}=" + f"{builder.parameters.body_parameter.client_name}," + ) + retval.append(" **kwargs") + retval.append(")") + return retval + + def serialize_headers(self, builder: RequestBuilderType) -> List[str]: + headers = [ + h + for h in builder.parameters.headers + if not builder.has_form_data_body or h.wire_name.lower() != "content-type" + ] + retval = ["# Construct headers"] if headers else [] + for header in headers: + retval.extend( + self.parameter_serializer.serialize_query_header( + header, + "headers", + self.serializer_name, + self.code_model.is_legacy, + ) + ) + return retval + + def serialize_query(self, builder: RequestBuilderType) -> List[str]: + retval = ["# Construct parameters"] + for parameter in builder.parameters.query: + retval.extend( + self.parameter_serializer.serialize_query_header( + parameter, + "params", + self.serializer_name, + self.code_model.is_legacy, + ) + ) + return retval + + def construct_url(self, builder: RequestBuilderType) -> str: + if any(o for o in ["low_level_client", "version_tolerant"] if self.code_model.options.get(o)): + url_value = _escape_str(builder.url) + else: + url_value = f'kwargs.pop("template_url", {_escape_str(builder.url)})' + return f"_url = {url_value}{' # pylint: disable=line-too-long' if len(url_value) > 114 else ''}" + + +############################## NORMAL OPERATIONS ############################## + + +class _OperationSerializer(_BuilderBaseSerializer[OperationType]): # pylint: disable=abstract-method + def description_and_summary(self, builder: OperationType) -> List[str]: + retval = super().description_and_summary(builder) + if builder.deprecated: + retval.append(".. warning::") + retval.append(" This method is deprecated") + retval.append("") + if builder.external_docs and builder.external_docs.get("url"): + retval.append(".. seealso::") + retval.append(f" - {builder.external_docs['url']}") + retval.append("") + return retval + + @property + def _json_response_template_name(self) -> str: + return "response" + + def example_template(self, builder: OperationType) -> List[str]: + retval = super().example_template(builder) + if self.code_model.options["models_mode"] == "msrest": + return retval + for response in builder.responses: + polymorphic_subtypes: List[ModelType] = [] + if not response.type: + continue + response.get_polymorphic_subtypes(polymorphic_subtypes) + if polymorphic_subtypes: + # we just assume one kind of polymorphic body for input + discriminator_name = cast(Property, polymorphic_subtypes[0].discriminator).wire_name + retval.append("") + retval.append( + "# The response is polymorphic. The following are possible polymorphic " + f'responses based off discriminator "{discriminator_name}":' + ) + for idx in range( + min( + self.code_model.options["polymorphic_examples"], + len(polymorphic_subtypes), + ) + ): + retval.extend(_get_polymorphic_subtype_template(polymorphic_subtypes[idx])) + + if _get_json_response_template_to_status_codes(builder): + retval.append("") + for ( + response_body, + status_codes, + ) in _get_json_response_template_to_status_codes(builder).items(): + retval.append("# response body for status code(s): {}".format(", ".join(status_codes))) + retval.extend(f"{self._json_response_template_name} == {response_body}".splitlines()) + return retval + + def make_pipeline_call(self, builder: OperationType) -> List[str]: + type_ignore = self.async_mode and builder.group_name == "" # is in a mixin + stream_value = ( + f'kwargs.pop("stream", {builder.has_stream_response})' + if builder.expose_stream_keyword and builder.has_response_body + else builder.has_stream_response + ) + return [ + f"_stream = {stream_value}", + f"pipeline_response: PipelineResponse = {self._call_method}self._client.{self.pipeline_name}.run( " + + f"{'# type: ignore' if type_ignore else ''} # pylint: disable=protected-access", + " _request,", + " stream=_stream,", + " **kwargs", + ")", + ] + + @property + def _function_def(self) -> str: + return "async def" if self.async_mode else "def" + + @property + def _need_self_param(self) -> bool: + return True + + @property + def serializer_name(self) -> str: + return "self._serialize" + + def decorators(self, builder: OperationType) -> List[str]: + """Decorators for the method""" + retval = super().decorators(builder) + if _api_version_validation(builder): + retval.append(_api_version_validation(builder)) + return retval + + def pop_kwargs_from_signature(self, builder: OperationType) -> List[str]: + kwargs_to_pop = builder.parameters.kwargs_to_pop + kwargs = self.parameter_serializer.pop_kwargs_from_signature( + kwargs_to_pop, + check_kwarg_dict=True, + pop_headers_kwarg=( + PopKwargType.CASE_INSENSITIVE + if builder.has_kwargs_to_pop_with_default(kwargs_to_pop, ParameterLocation.HEADER) # type: ignore + else PopKwargType.SIMPLE + ), + pop_params_kwarg=( + PopKwargType.CASE_INSENSITIVE + if builder.has_kwargs_to_pop_with_default(kwargs_to_pop, ParameterLocation.QUERY) # type: ignore + else PopKwargType.SIMPLE + ), + check_client_input=not self.code_model.options["multiapi"], + operation_name=f"('{builder.name}')" if builder.group_name == "" else "", + ) + for p in builder.parameters.parameters: + if p.hide_in_operation_signature: + kwargs.append(f'{p.client_name} = kwargs.pop("{p.client_name}", None)') + cls_annotation = builder.cls_type_annotation(async_mode=self.async_mode) + pylint_disable = "" + if any(x.startswith("_") for x in cls_annotation.split(".")): + pylint_disable = " # pylint: disable=protected-access" + kwargs.append(f"cls: {cls_annotation} = kwargs.pop({pylint_disable}\n 'cls', None\n)") + return kwargs + + def response_docstring(self, builder: OperationType) -> List[str]: + response_str = f":return: {builder.response_docstring_text(async_mode=self.async_mode)}" + rtype_str = f":rtype: {builder.response_docstring_type(async_mode=self.async_mode)}" + return [ + response_str, + rtype_str, + f":raises ~{self.code_model.core_library}.exceptions.HttpResponseError:", + ] + + def _serialize_body_parameter(self, builder: OperationType) -> List[str]: + """We need to serialize params if they're not meant to be streamed in. + + This function serializes the body params that need to be serialized. + """ + retval: List[str] = [] + body_param = builder.parameters.body_parameter + if body_param.is_form_data: + model_type = cast( + ModelType, + ( + body_param.type.target_model_subtype((JSONModelType, DPGModelType)) + if isinstance(body_param.type, CombinedType) + else body_param.type + ), + ) + file_fields = [p.wire_name for p in model_type.properties if p.is_multipart_file_input] + data_fields = [p.wire_name for p in model_type.properties if not p.is_multipart_file_input] + retval.extend( + [ + "_body = (", + f" {body_param.client_name}.as_dict()", + f" if isinstance({body_param.client_name}, _model_base.Model) else", + f" {body_param.client_name}", + ")", + f"_file_fields: List[str] = {file_fields}", + f"_data_fields: List[str] = {data_fields}", + "_files, _data = prepare_multipart_form_data(_body, _file_fields, _data_fields)", + ] + ) + return retval + + body_kwarg_name = builder.request_builder.parameters.body_parameter.client_name + send_xml = builder.parameters.body_parameter.type.is_xml + xml_serialization_ctxt = body_param.type.xml_serialization_ctxt if send_xml else None + ser_ctxt_name = "serialization_ctxt" + if xml_serialization_ctxt and self.code_model.options["models_mode"]: + retval.append(f'{ser_ctxt_name} = {{"xml": {{{xml_serialization_ctxt}}}}}') + if self.code_model.options["models_mode"] == "msrest": + is_xml_cmd = _xml_config(send_xml, builder.parameters.body_parameter.content_types) + serialization_ctxt_cmd = f", {ser_ctxt_name}={ser_ctxt_name}" if xml_serialization_ctxt else "" + create_body_call = ( + f"_{body_kwarg_name} = self._serialize.body({body_param.client_name}, " + f"'{body_param.type.serialization_type}'{is_xml_cmd}{serialization_ctxt_cmd})" + ) + elif self.code_model.options["models_mode"] == "dpg": + if _json_serializable(body_param.default_content_type): + if hasattr(body_param.type, "encode") and body_param.type.encode: # type: ignore + create_body_call = ( + f"_{body_kwarg_name} = json.dumps({body_param.client_name}, " + "cls=SdkJSONEncoder, exclude_readonly=True, " + f"format='{body_param.type.encode}') # type: ignore" # type: ignore + ) + else: + create_body_call = ( + f"_{body_kwarg_name} = json.dumps({body_param.client_name}, " + "cls=SdkJSONEncoder, exclude_readonly=True) # type: ignore" + ) + else: + create_body_call = f"_{body_kwarg_name} = {body_param.client_name}" + else: + create_body_call = f"_{body_kwarg_name} = {body_param.client_name}" + if body_param.optional: + retval.append(f"if {body_param.client_name} is not None:") + retval.append(" " + create_body_call) + retval.append("else:") + retval.append(f" _{body_kwarg_name} = None") + else: + retval.append(create_body_call) + return retval + + def _create_body_parameter( + self, + builder: OperationType, + ) -> List[str]: + """Create the body parameter before we pass it as either json or content to the request builder""" + retval = [] + body_param = builder.parameters.body_parameter + if body_param.entries: + return _serialize_multipart_body(builder) + body_kwarg_name = builder.request_builder.parameters.body_parameter.client_name + body_param_type = body_param.type + if isinstance(body_param_type, BinaryType) or ( + isinstance(body_param.type, ByteArraySchema) and body_param.default_content_type != "application/json" + ): + retval.append(f"_{body_kwarg_name} = {body_param.client_name}") + if ( + not body_param.default_content_type + and not next(p for p in builder.parameters if p.wire_name.lower() == "content-type").optional + ): + content_types = "'" + "', '".join(body_param.content_types) + "'" + retval.extend( + [ + "if not content_type:", + f' raise TypeError("Missing required keyword-only argument: content_type. ' + f'Known values are:" + "{content_types}")', + ] + ) + else: + retval.extend(self._serialize_body_parameter(builder)) + return retval + + def _initialize_overloads(self, builder: OperationType, is_paging: bool = False) -> List[str]: + retval: List[str] = [] + # For paging, we put body parameter in local place outside `prepare_request` + if is_paging: + return retval + same_content_type = len(set(o.parameters.body_parameter.default_content_type for o in builder.overloads)) == 1 + if same_content_type: + default_content_type = builder.overloads[0].parameters.body_parameter.default_content_type + retval.append(f'content_type = content_type or "{default_content_type}"') + client_names = [ + overload.request_builder.parameters.body_parameter.client_name for overload in builder.overloads + ] + for v in sorted(set(client_names), key=client_names.index): + retval.append(f"_{v} = None") + try: + # if there is a binary overload, we do a binary check first. + binary_overload = cast( + OperationType, + next((o for o in builder.overloads if isinstance(o.parameters.body_parameter.type, BinaryType))), + ) + binary_body_param = binary_overload.parameters.body_parameter + retval.append(f"if {binary_body_param.type.instance_check_template.format(binary_body_param.client_name)}:") + if binary_body_param.default_content_type and not same_content_type: + retval.append(f' content_type = content_type or "{binary_body_param.default_content_type}"') + retval.extend(f" {l}" for l in self._create_body_parameter(binary_overload)) + retval.append("else:") + other_overload = cast( + OperationType, + next((o for o in builder.overloads if not isinstance(o.parameters.body_parameter.type, BinaryType))), + ) + retval.extend(f" {l}" for l in self._create_body_parameter(other_overload)) + if other_overload.parameters.body_parameter.default_content_type and not same_content_type: + retval.append( + " content_type = content_type or " + f'"{other_overload.parameters.body_parameter.default_content_type}"' + ) + except StopIteration: + for idx, overload in enumerate(builder.overloads): + if_statement = "if" if idx == 0 else "elif" + body_param = overload.parameters.body_parameter + retval.append( + f"{if_statement} {body_param.type.instance_check_template.format(body_param.client_name)}:" + ) + if body_param.default_content_type and not same_content_type: + retval.append(f' content_type = content_type or "{body_param.default_content_type}"') + retval.extend(f" {l}" for l in self._create_body_parameter(cast(OperationType, overload))) + return retval + + def _create_request_builder_call( + self, + builder: OperationType, + request_builder: RequestBuilderType, + is_next_request: bool = False, + ) -> List[str]: + retval: List[str] = [] + if self.code_model.options["builders_visibility"] == "embedded": + request_path_name = request_builder.name + else: + group_name = request_builder.group_name + request_path_name = "rest{}.{}".format( + ("_" + group_name) if group_name else "", + request_builder.name, + ) + retval.append(f"_request = {request_path_name}(") + for parameter in request_builder.parameters.method: + if parameter.location == ParameterLocation.BODY: + # going to pass in body later based off of overloads + continue + if ( + is_next_request + and builder.operation_type == "paging" + and not bool(builder.next_request_builder) # type: ignore + and parameter.location == ParameterLocation.QUERY + ): + # if we don't want to reformat query parameters for next link calls + # in paging operations with a single swagger operation defintion, + # we skip passing query params when building the next request + continue + type_ignore = ( + parameter.grouped_by + and parameter.client_default_value is not None + and next(p for p in builder.parameters if p.grouper and p.client_name == parameter.grouped_by).optional + ) + retval.append( + f" {parameter.client_name}={parameter.name_in_high_level_operation}," + f"{' # type: ignore' if type_ignore else ''}" + ) + if builder.parameters.has_body and builder.parameters.body_parameter.entries: + # this is for legacy + client_name = builder.parameters.body_parameter.client_name + retval.append(f" {client_name}=_{client_name},") + elif request_builder.has_form_data_body: + retval.append(" files=_files,") + retval.append(" data=_data,") + elif request_builder.overloads: + seen_body_params = set() + for overload in request_builder.overloads: + body_param = cast(RequestBuilderBodyParameter, overload.parameters.body_parameter) + if body_param.client_name in seen_body_params: + continue + seen_body_params.add(body_param.client_name) + + retval.append(f" {body_param.client_name}={body_param.name_in_high_level_operation},") + elif request_builder.parameters.has_body: + body_param = cast(RequestBuilderBodyParameter, request_builder.parameters.body_parameter) + retval.append(f" {body_param.client_name}={body_param.name_in_high_level_operation},") + retval.append(" headers=_headers,") + retval.append(" params=_params,") + retval.append(")") + return retval + + def _postprocess_http_request(self, builder: OperationType, template_url: Optional[str] = None) -> List[str]: + retval: List[str] = [] + if builder.parameters.path: + retval.extend(self.serialize_path(builder)) + url_to_format = "_request.url" + if self.code_model.options["version_tolerant"] and template_url: + url_to_format = template_url + retval.append( + "_request.url = self._client.format_url({}{})".format( + url_to_format, + ", **path_format_arguments" if builder.parameters.path else "", + ) + ) + return retval + + def _call_request_builder_helper( # pylint: disable=too-many-statements + self, + builder: OperationType, + request_builder: RequestBuilderType, + template_url: Optional[str] = None, + is_next_request: bool = False, + is_paging: bool = False, + ) -> List[str]: + retval = [] + if builder.parameters.grouped: + # request builders don't allow grouped parameters, so we group them before making the call + retval.extend(_serialize_grouped_body(builder)) + if builder.parameters.has_body and builder.parameters.body_parameter.flattened: + # unflatten before passing to request builder as well + retval.extend(_serialize_flattened_body(builder.parameters.body_parameter)) + if is_json_model_type(builder.parameters): + retval.extend(_serialize_json_model_body(builder.parameters.body_parameter, builder.parameters.parameters)) + if builder.has_form_data_body: + retval.extend(self._create_body_parameter(builder)) + elif builder.overloads: + # we are only dealing with two overloads. If there are three, we generate an abstract operation + retval.extend(self._initialize_overloads(builder, is_paging=is_paging)) + elif builder.parameters.has_body: + # non-overloaded body + retval.extend(self._create_body_parameter(builder)) + retval.append("") + retval.extend(self._create_request_builder_call(builder, request_builder, is_next_request)) + retval.extend(self._postprocess_http_request(builder, template_url)) + return retval + + def call_request_builder(self, builder: OperationType, is_paging: bool = False) -> List[str]: + return self._call_request_builder_helper(builder, builder.request_builder, is_paging=is_paging) + + def response_headers_and_deserialization( + self, + builder: OperationType, + response: Response, + ) -> List[str]: + retval: List[str] = [ + ( + f"response_headers['{response_header.wire_name}']=self._deserialize(" + f"'{response_header.serialization_type}', response.headers.get('{response_header.wire_name}'))" + ) + for response_header in response.headers + ] + if response.headers: + retval.append("") + deserialize_code: List[str] = [] + stream_logic = True + if builder.has_stream_response: + if isinstance(response.type, ByteArraySchema): + deserialized = f"{'await ' if self.async_mode else ''}response.read()" + else: + stream_logic = False + if self.code_model.options["version_tolerant"]: + deserialized = "response.iter_bytes()" + else: + deserialized = f"response.stream_download(self._client.{self.pipeline_name})" + deserialize_code.append(f"deserialized = {deserialized}") + elif response.type: + pylint_disable = "" + if isinstance(response.type, ModelType) and response.type.internal: + pylint_disable = " # pylint: disable=protected-access" + if self.code_model.options["models_mode"] == "msrest": + deserialize_code.append("deserialized = self._deserialize(") + deserialize_code.append(f" '{response.serialization_type}',{pylint_disable}") + deserialize_code.append(" pipeline_response") + deserialize_code.append(")") + elif self.code_model.options["models_mode"] == "dpg": + if builder.has_stream_response: + deserialize_code.append("deserialized = response.content") + else: + format_filed = ( + f', format="{response.type.encode}"' + if isinstance(response.type, ByteArraySchema) + and response.default_content_type == "application/json" + else "" + ) + response_attr = "json" if _json_serializable(str(response.default_content_type)) else "text" + deserialize_code.append("deserialized = _deserialize(") + deserialize_code.append( + f" {response.type.type_annotation(is_operation_file=True)},{pylint_disable}" + ) + deserialize_code.append(f" response.{response_attr}(){response.result_property}{format_filed}") + deserialize_code.append(")") + + else: + deserialized_value = "ET.fromstring(response.text())" if response.type.is_xml else "response.json()" + deserialize_code.append("if response.content:") + deserialize_code.append(f" deserialized = {deserialized_value}") + deserialize_code.append("else:") + deserialize_code.append(" deserialized = None") + if len(deserialize_code) > 0: + if builder.expose_stream_keyword and stream_logic: + retval.append("if _stream:") + retval.append(" deserialized = response.iter_bytes()") + retval.append("else:") + retval.extend([f" {dc}" for dc in deserialize_code]) + else: + retval.extend(deserialize_code) + return retval + + def handle_error_response(self, builder: OperationType) -> List[str]: + async_await = "await " if self.async_mode else "" + retval = [f"if response.status_code not in {str(builder.success_status_codes)}:"] + retval.extend( + [ + " if _stream:", + f" {async_await} response.read() # Load the body in memory and close the socket", + ] + ) + type_ignore = " # type: ignore" if _need_type_ignore(builder) else "" + retval.append( + f" map_error(status_code=response.status_code, response=response, error_map=error_map){type_ignore}" + ) + error_model = "" + if builder.default_error_deserialization and self.code_model.options["models_mode"]: + if self.code_model.options["models_mode"] == "dpg": + retval.append(f" error = _deserialize({builder.default_error_deserialization}, response.json())") + else: + retval.append( + f" error = self._deserialize.failsafe_deserialize({builder.default_error_deserialization}, " + "pipeline_response)" + ) + error_model = ", model=error" + retval.append( + " raise HttpResponseError(response=response{}{})".format( + error_model, + (", error_format=ARMErrorFormat" if self.code_model.options["azure_arm"] else ""), + ) + ) + return retval + + def handle_response(self, builder: OperationType) -> List[str]: + retval: List[str] = ["response = pipeline_response.http_response"] + retval.append("") + retval.extend(self.handle_error_response(builder)) + retval.append("") + if builder.has_optional_return_type: + retval.append("deserialized = None") + if builder.any_response_has_headers: + retval.append("response_headers = {}") + if builder.has_response_body or builder.any_response_has_headers: + if len(builder.responses) > 1: + for status_code in builder.success_status_codes: + response = builder.get_response_from_status(status_code) + if response.headers or response.type: + retval.append(f"if response.status_code == {status_code}:") + retval.extend( + [f" {line}" for line in self.response_headers_and_deserialization(builder, response)] + ) + retval.append("") + else: + retval.extend(self.response_headers_and_deserialization(builder, builder.responses[0])) + retval.append("") + if builder.has_optional_return_type or self.code_model.options["models_mode"]: + deserialized = "deserialized" + else: + deserialized = f"cast({builder.response_type_annotation(async_mode=self.async_mode)}, deserialized)" + retval.append("if cls:") + retval.append( + " return cls(pipeline_response, {}, {}){}".format( + deserialized if builder.has_response_body else "None", + "response_headers" if builder.any_response_has_headers else "{}", + " # type: ignore", + ) + ) + if builder.has_response_body and any( + response.is_stream_response or response.type for response in builder.responses + ): + retval.append("") + retval.append(f"return {deserialized} # type: ignore") + if builder.request_builder.method == "HEAD" and self.code_model.options["head_as_boolean"]: + retval.append("return 200 <= response.status_code <= 299") + return retval + + def error_map(self, builder: OperationType) -> List[str]: + retval = ["error_map: MutableMapping[int, Type[HttpResponseError]] = {"] + if builder.non_default_errors: + if not 401 in builder.non_default_error_status_codes: + retval.append(" 401: ClientAuthenticationError,") + if not 404 in builder.non_default_error_status_codes: + retval.append(" 404: ResourceNotFoundError,") + if not 409 in builder.non_default_error_status_codes: + retval.append(" 409: ResourceExistsError,") + if not 304 in builder.non_default_error_status_codes: + retval.append(" 304: ResourceNotModifiedError,") + for excep in builder.non_default_errors: + error_model_str = "" + if isinstance(excep.type, ModelType): + if self.code_model.options["models_mode"] == "msrest": + error_model_str = ( + f", model=self._deserialize(" f"_models.{excep.type.serialization_type}, response)" + ) + elif self.code_model.options["models_mode"] == "dpg": + error_model_str = f", model=_deserialize(_models.{excep.type.name}, response.json())" + error_format_str = ", error_format=ARMErrorFormat" if self.code_model.options["azure_arm"] else "" + for status_code in excep.status_codes: + if status_code == 401: + retval.append( + " 401: cast(Type[HttpResponseError], " + "lambda response: ClientAuthenticationError(response=response" + f"{error_model_str}{error_format_str}))," + ) + elif status_code == 404: + retval.append( + " 404: cast(Type[HttpResponseError], " + "lambda response: ResourceNotFoundError(response=response" + f"{error_model_str}{error_format_str}))," + ) + elif status_code == 409: + retval.append( + " 409: cast(Type[HttpResponseError], " + "lambda response: ResourceExistsError(response=response" + f"{error_model_str}{error_format_str}))," + ) + elif status_code == 304: + retval.append( + " 304: cast(Type[HttpResponseError], " + "lambda response: ResourceNotModifiedError(response=response" + f"{error_model_str}{error_format_str}))," + ) + elif not error_model_str and not error_format_str: + retval.append(f" {status_code}: HttpResponseError,") + else: + retval.append( + f" {status_code}: cast(Type[HttpResponseError], " + "lambda response: HttpResponseError(response=response" + f"{error_model_str}{error_format_str}))," + ) + else: + retval.append( + " 401: ClientAuthenticationError, 404: ResourceNotFoundError, 409: ResourceExistsError, " + "304: ResourceNotModifiedError" + ) + retval.append("}") + if builder.has_etag: + retval.extend( + [ + "if match_condition == MatchConditions.IfNotModified:", + " error_map[412] = ResourceModifiedError", + "elif match_condition == MatchConditions.IfPresent:", + " error_map[412] = ResourceNotFoundError", + "elif match_condition == MatchConditions.IfMissing:", + " error_map[412] = ResourceExistsError", + ] + ) + retval.append("error_map.update(kwargs.pop('error_map', {}) or {})") + return retval + + @property + def _call_method(self) -> str: + return "await " if self.async_mode else "" + + +class OperationSerializer(_OperationSerializer[Operation]): ... + + +############################## PAGING OPERATIONS ############################## + +PagingOperationType = TypeVar("PagingOperationType", bound=Union[PagingOperation, LROPagingOperation]) + + +class _PagingOperationSerializer(_OperationSerializer[PagingOperationType]): # pylint: disable=abstract-method + def __init__(self, code_model: CodeModel, async_mode: bool) -> None: + # for pylint reasons need to redefine init + # probably because inheritance is going too deep + super().__init__(code_model, async_mode) + self.code_model = code_model + self.async_mode = async_mode + self.parameter_serializer = ParameterSerializer() + + def serialize_path(self, builder: PagingOperationType) -> List[str]: + return self.parameter_serializer.serialize_path(builder.parameters.path, self.serializer_name) + + def decorators(self, builder: PagingOperationType) -> List[str]: + """Decorators for the method""" + retval: List[str] = [] + if builder.is_overload: + return ["@overload"] + if self.code_model.options["tracing"] and builder.want_tracing: + retval.append("@distributed_trace") + if _api_version_validation(builder): + retval.append(_api_version_validation(builder)) + return retval + + def call_next_link_request_builder(self, builder: PagingOperationType) -> List[str]: + if builder.next_request_builder: + request_builder = builder.next_request_builder + template_url = None + else: + request_builder = builder.request_builder + template_url = "next_link" + + request_builder = builder.next_request_builder or builder.request_builder + if builder.next_request_builder: + return self._call_request_builder_helper( + builder, + request_builder, + template_url=template_url, + is_next_request=True, + ) + retval: List[str] = [] + query_str = "" + next_link_str = "next_link" + try: + api_version_param = next( + p for p in builder.client.parameters if p.is_api_version and p.location == ParameterLocation.QUERY + ) + retval.append("# make call to next link with the client's api-version") + retval.append("_parsed_next_link = urllib.parse.urlparse(next_link)") + retval.extend( + [ + "_next_request_params = case_insensitive_dict({", + " key: [urllib.parse.quote(v) for v in value]" + " for key, value in urllib.parse.parse_qs(_parsed_next_link.query).items()" + "})", + ] + ) + api_version = ( + "self._api_version" + if self.code_model.options["multiapi"] and builder.group_name + else api_version_param.full_client_name + ) + retval.append(f'_next_request_params["api-version"] = {api_version}') + query_str = ", params=_next_request_params" + next_link_str = "urllib.parse.urljoin(next_link, _parsed_next_link.path)" + except StopIteration: + pass + + retval.append(f'_request = HttpRequest("GET", {next_link_str}{query_str})') + retval.extend(self._postprocess_http_request(builder, "_request.url")) + + return retval + + def _prepare_request_callback(self, builder: PagingOperationType) -> List[str]: + retval = self._initialize_overloads(builder) + retval.append("def prepare_request(next_link=None):") + retval.append(" if not next_link:") + retval.extend([f" {line}" for line in self.call_request_builder(builder, is_paging=True)]) + retval.append("") + retval.append(" else:") + retval.extend([f" {line}" for line in self.call_next_link_request_builder(builder)]) + if not builder.next_request_builder and self.code_model.is_legacy: + retval.append(' _request.method = "GET"') + else: + retval.append("") + retval.append(" return _request") + return retval + + @property + def _function_def(self) -> str: + return "def" + + def _extract_data_callback(self, builder: PagingOperationType) -> List[str]: + retval = [f"{'async ' if self.async_mode else ''}def extract_data(pipeline_response):"] + response = builder.responses[0] + deserialized = "pipeline_response.http_response.json()" + if self.code_model.options["models_mode"] == "msrest": + deserialize_type = response.serialization_type + pylint_disable = " # pylint: disable=protected-access" + if isinstance(response.type, ModelType) and not response.type.internal: + deserialize_type = f'"{response.serialization_type}"' + pylint_disable = "" + deserialized = f"self._deserialize(\n {deserialize_type},{pylint_disable}\n pipeline_response\n)" + retval.append(f" deserialized = {deserialized}") + elif self.code_model.options["models_mode"] == "dpg": + # we don't want to generate paging models for DPG + retval.append(f" deserialized = {deserialized}") + else: + retval.append(f" deserialized = {deserialized}") + item_name = builder.item_name + access = f".{item_name}" if self.code_model.options["models_mode"] == "msrest" else f'["{item_name}"]' + list_of_elem_deserialized = "" + if self.code_model.options["models_mode"] == "dpg": + item_type = builder.item_type.type_annotation(is_operation_file=True) + list_of_elem_deserialized = f"_deserialize({item_type}, deserialized{access})" + else: + list_of_elem_deserialized = f"deserialized{access}" + retval.append(f" list_of_elem = {list_of_elem_deserialized}") + retval.append(" if cls:") + retval.append(" list_of_elem = cls(list_of_elem) # type: ignore") + + continuation_token_name = builder.continuation_token_name + if not continuation_token_name: + cont_token_property = "None" + elif self.code_model.options["models_mode"] == "msrest": + cont_token_property = f"deserialized.{continuation_token_name} or None" + else: + cont_token_property = f'deserialized.get("{continuation_token_name}") or None' + list_type = "AsyncList" if self.async_mode else "iter" + retval.append(f" return {cont_token_property}, {list_type}(list_of_elem)") + return retval + + def _get_next_callback(self, builder: PagingOperationType) -> List[str]: + retval = [f"{'async ' if self.async_mode else ''}def get_next(next_link=None):"] + retval.append(" _request = prepare_request(next_link)") + retval.append("") + retval.extend([f" {l}" for l in self.make_pipeline_call(builder)]) + retval.append(" response = pipeline_response.http_response") + retval.append("") + retval.extend([f" {line}" for line in self.handle_error_response(builder)]) + retval.append("") + retval.append(" return pipeline_response") + return retval + + def set_up_params_for_pager(self, builder: PagingOperationType) -> List[str]: + retval = [] + retval.extend(self.error_map(builder)) + retval.extend(self._prepare_request_callback(builder)) + retval.append("") + retval.extend(self._extract_data_callback(builder)) + retval.append("") + retval.extend(self._get_next_callback(builder)) + return retval + + +class PagingOperationSerializer(_PagingOperationSerializer[PagingOperation]): ... + + +############################## LRO OPERATIONS ############################## + +LROOperationType = TypeVar("LROOperationType", bound=Union[LROOperation, LROPagingOperation]) + + +class _LROOperationSerializer(_OperationSerializer[LROOperationType]): + def __init__(self, code_model: CodeModel, async_mode: bool) -> None: + # for pylint reasons need to redefine init + # probably because inheritance is going too deep + super().__init__(code_model, async_mode) + self.code_model = code_model + self.async_mode = async_mode + self.parameter_serializer = ParameterSerializer() + + def serialize_path(self, builder: LROOperationType) -> List[str]: + return self.parameter_serializer.serialize_path(builder.parameters.path, self.serializer_name) + + def initial_call(self, builder: LROOperationType) -> List[str]: + retval = [ + f"polling: Union[bool, {builder.get_base_polling_method(self.async_mode)}] = kwargs.pop('polling', True)", + ] + retval.append("lro_delay = kwargs.pop(") + retval.append(" 'polling_interval',") + retval.append(" self._config.polling_interval") + retval.append(")") + retval.append("cont_token: Optional[str] = kwargs.pop('continuation_token', None)") + retval.append("if cont_token is None:") + retval.append( + f" raw_result = {self._call_method}self.{builder.initial_operation.name}(" + f"{'' if any(rsp.type for rsp in builder.initial_operation.responses) else ' # type: ignore'}" + ) + retval.extend( + [f" {parameter.client_name}={parameter.client_name}," for parameter in builder.parameters.method] + ) + retval.append(" cls=lambda x,y,z: x,") + retval.append(" headers=_headers,") + retval.append(" params=_params,") + retval.append(" **kwargs") + retval.append(" )") + retval.append("kwargs.pop('error_map', None)") + return retval + + def return_lro_poller(self, builder: LROOperationType) -> List[str]: + retval = [] + lro_options_str = ( + "lro_options={'final-state-via': '" + builder.lro_options["final-state-via"] + "'}," + if builder.lro_options + else "" + ) + path_format_arguments_str = "" + if builder.parameters.path: + path_format_arguments_str = "path_format_arguments=path_format_arguments," + retval.extend(self.serialize_path(builder)) + retval.append("") + retval.extend( + [ + "if polling is True:", + f" polling_method: {builder.get_base_polling_method(self.async_mode)} " + + f"= cast({builder.get_base_polling_method(self.async_mode)}, " + f"{builder.get_polling_method(self.async_mode)}(", + " lro_delay,", + f" {lro_options_str}", + f" {path_format_arguments_str}", + " **kwargs", + "))", + ] + ) + retval.append( + f"elif polling is False: polling_method = cast({builder.get_base_polling_method(self.async_mode)}, " + f"{builder.get_no_polling_method(self.async_mode)}())" + ) + retval.append("else: polling_method = polling") + retval.append("if cont_token:") + retval.append(f" return {builder.get_poller_with_response_type(self.async_mode)}.from_continuation_token(") + retval.append(" polling_method=polling_method,") + retval.append(" continuation_token=cont_token,") + retval.append(" client=self._client,") + retval.append(" deserialization_callback=get_long_running_output") + retval.append(" )") + retval.append(f"return {builder.get_poller_with_response_type(self.async_mode)}(") + retval.append(" self._client, raw_result, get_long_running_output, polling_method # type: ignore") + retval.append(" )") + return retval + + def get_long_running_output(self, builder: LROOperationType) -> List[str]: + pylint_disable = "" + if not builder.lro_response: + pylint_disable = " # pylint: disable=inconsistent-return-statements" + retval = [f"def get_long_running_output(pipeline_response):{pylint_disable}"] + if builder.lro_response: + if builder.lro_response.headers: + retval.append(" response_headers = {}") + if ( + not self.code_model.options["models_mode"] + or self.code_model.options["models_mode"] == "dpg" + or builder.lro_response.headers + ): + retval.append(" response = pipeline_response.http_response") + retval.extend( + [f" {line}" for line in self.response_headers_and_deserialization(builder, builder.lro_response)] + ) + retval.append(" if cls:") + retval.append( + " return cls(pipeline_response, {}, {}){}".format( + ("deserialized" if builder.lro_response and builder.lro_response.type else "None"), + ("response_headers" if builder.lro_response and builder.lro_response.headers else "{}"), + " # type: ignore", + ) + ) + if builder.lro_response and builder.lro_response.type: + retval.append(" return deserialized") + return retval + + +class LROOperationSerializer(_LROOperationSerializer[LROOperation]): ... + + +############################## LRO PAGING OPERATIONS ############################## + + +class LROPagingOperationSerializer( + _LROOperationSerializer[LROPagingOperation], + _PagingOperationSerializer[LROPagingOperation], +): # pylint: disable=abstract-method + @property + def _call_method(self) -> str: + return "await " if self.async_mode else "" + + @property + def _function_def(self) -> str: + return "async def" if self.async_mode else "def" + + def get_long_running_output(self, builder: LROPagingOperation) -> List[str]: + retval = ["def get_long_running_output(pipeline_response):"] + retval.append(f" {self._function_def} internal_get_next(next_link=None):") + retval.append(" if next_link is None:") + retval.append(" return pipeline_response") + retval.append(f" return {self._call_method}get_next(next_link)") + retval.append("") + retval.append(f" return {builder.get_pager(self.async_mode)}(") + retval.append(" internal_get_next, extract_data") + retval.append(" )") + return retval + + def decorators(self, builder: LROPagingOperation) -> List[str]: + """Decorators for the method""" + return _LROOperationSerializer.decorators(self, builder) # type: ignore + + +def get_operation_serializer( + builder: Operation, + code_model, + async_mode: bool, +) -> Union[ + OperationSerializer, + PagingOperationSerializer, + LROOperationSerializer, + LROPagingOperationSerializer, +]: + retcls: Union[ + Type[OperationSerializer], + Type[PagingOperationSerializer], + Type[LROOperationSerializer], + Type[LROPagingOperationSerializer], + ] = OperationSerializer + if builder.operation_type == "lropaging": + retcls = LROPagingOperationSerializer + elif builder.operation_type == "lro": + retcls = LROOperationSerializer + elif builder.operation_type == "paging": + retcls = PagingOperationSerializer + return retcls(code_model, async_mode) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/client_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/client_serializer.py new file mode 100644 index 00000000000..28041370efb --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/client_serializer.py @@ -0,0 +1,295 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import List + +from . import utils +from ..models import Client, ParameterMethodLocation +from .parameter_serializer import ParameterSerializer, PopKwargType +from ...utils import build_policies + + +class ClientSerializer: + def __init__(self, client: Client) -> None: + self.client = client + self.parameter_serializer = ParameterSerializer() + + def _init_signature(self, async_mode: bool) -> str: + pylint_disable = "" + if not self.client.parameters.credential: + pylint_disable = " # pylint: disable=missing-client-constructor-parameter-credential" + return self.parameter_serializer.serialize_method( + function_def="def", + method_name="__init__", + need_self_param=True, + method_param_signatures=self.client.parameters.method_signature(async_mode), + pylint_disable=pylint_disable, + ) + + def init_signature_and_response_type_annotation(self, async_mode: bool) -> str: + init_signature = self._init_signature(async_mode) + return utils.method_signature_and_response_type_annotation_template( + method_signature=init_signature, + response_type_annotation="None", + ) + + def pop_kwargs_from_signature(self) -> List[str]: + return self.parameter_serializer.pop_kwargs_from_signature( + self.client.parameters.kwargs_to_pop, + check_kwarg_dict=False, + pop_headers_kwarg=PopKwargType.NO, + pop_params_kwarg=PopKwargType.NO, + ) + + @property + def class_definition(self) -> str: + class_name = self.client.name + base_class = "" + if self.client.has_mixin: + base_class = f"{class_name}OperationsMixin" + pylint_disable = self.client.pylint_disable + if base_class: + return f"class {class_name}({base_class}):{pylint_disable}" + return f"class {class_name}:{pylint_disable}" + + def property_descriptions(self, async_mode: bool) -> List[str]: + retval: List[str] = [] + operations_folder = ".aio.operations." if async_mode else ".operations." + for og in [og for og in self.client.operation_groups if not og.is_mixin]: + retval.append(f":ivar {og.property_name}: {og.class_name} operations") + property_type = f"{self.client.code_model.namespace}{operations_folder}{og.class_name}" + retval.append(f":vartype {og.property_name}: {property_type}") + for param in self.client.parameters.method: + retval.append(f":{param.description_keyword} {param.client_name}: {param.description}") + retval.append( + f":{param.docstring_type_keyword} {param.client_name}: {param.docstring_type(async_mode=async_mode)}" + ) + if self.client.has_public_lro_operations: + retval.append( + ":keyword int polling_interval: Default waiting time between two polls for LRO operations " + "if no Retry-After header is present." + ) + retval = [s.replace("\\", "\\\\") for s in retval] + retval.append('"""') + return retval + + def initialize_config(self) -> str: + config_name = f"{self.client.name}Configuration" + config_call = ", ".join( + [ + f"{p.client_name}={p.client_name}" + for p in self.client.config.parameters.method + if p.method_location != ParameterMethodLocation.KWARG + ] + + ["**kwargs"] + ) + return f"self._config = {config_name}({config_call})" + + @property + def host_variable_name(self) -> str: + try: + return next(p for p in self.client.parameters if p.is_host).client_name + except StopIteration: + return "_endpoint" + + @property + def should_init_super(self) -> bool: + return any(og for og in self.client.operation_groups if og.is_mixin and og.has_abstract_operations) + + def initialize_pipeline_client(self, async_mode: bool) -> List[str]: + result = [] + pipeline_client_name = self.client.pipeline_class(async_mode) + endpoint_name = "base_url" if self.client.code_model.is_azure_flavor else "endpoint" + params = { + endpoint_name: self.host_variable_name, + "policies": "_policies", + } + if not self.client.code_model.is_legacy and self.client.request_id_header_name: + result.append(f'kwargs["request_id_header_name"] = "{self.client.request_id_header_name}"') + policies = build_policies( + self.client.code_model.options["azure_arm"], + async_mode, + is_azure_flavor=self.client.code_model.is_azure_flavor, + tracing=self.client.code_model.options["tracing"], + ) + result.extend( + [ + "_policies = kwargs.pop('policies', None)", + "if _policies is None:", + f' _policies = [{",".join(policies)}]', + f"self._client: {pipeline_client_name} = {pipeline_client_name}(" + f"{', '.join(f'{k}={v}' for k, v in params.items())}, **kwargs)", + ] + ) + return result + + def serializers_and_operation_groups_properties(self) -> List[str]: + retval = [] + + def _get_client_models_value(models_dict_name: str) -> str: + if self.client.code_model.model_types: + return f"{{k: v for k, v in {models_dict_name}.__dict__.items() if isinstance(v, type)}}" + return "{}" + + is_msrest_model = self.client.code_model.options["models_mode"] == "msrest" + if is_msrest_model: + add_private_models = len(self.client.code_model.model_types) != len( + self.client.code_model.public_model_types + ) + model_dict_name = f"_models.{self.client.code_model.models_filename}" if add_private_models else "_models" + retval.append( + f"client_models{': Dict[str, Any]' if not self.client.code_model.model_types else ''}" + f" = {_get_client_models_value(model_dict_name)}" + ) + if add_private_models and self.client.code_model.model_types: + update_dict = "{k: v for k, v in _models.__dict__.items() if isinstance(v, type)}" + retval.append(f"client_models.update({update_dict})") + client_models_str = "client_models" if is_msrest_model else "" + retval.append(f"self._serialize = Serializer({client_models_str})") + retval.append(f"self._deserialize = Deserializer({client_models_str})") + if not self.client.code_model.options["client_side_validation"]: + retval.append("self._serialize.client_side_validation = False") + operation_groups = [og for og in self.client.operation_groups if not og.is_mixin] + for og in operation_groups: + if og.code_model.options["multiapi"]: + api_version = f", '{og.api_versions[0]}'" if og.api_versions else ", None" + else: + api_version = "" + retval.extend( + [ + f"self.{og.property_name} = {og.class_name}(", + f" self._client, self._config, self._serialize, self._deserialize{api_version}", + ")", + ] + ) + return retval + + def _send_request_signature(self) -> str: + send_request_signature = [ + "request: HttpRequest, *, stream: bool = False," + ] + self.client.parameters.method_signature_kwargs + return self.parameter_serializer.serialize_method( + function_def="def", + method_name=self.client.send_request_name, + need_self_param=True, + method_param_signatures=send_request_signature, + ) + + def send_request_signature_and_response_type_annotation(self, async_mode: bool) -> str: + send_request_signature = self._send_request_signature() + return utils.method_signature_and_response_type_annotation_template( + method_signature=send_request_signature, + response_type_annotation=("Awaitable[AsyncHttpResponse]" if async_mode else "HttpResponse"), + ) + + def _example_make_call(self, async_mode: bool) -> List[str]: + http_response = "AsyncHttpResponse" if async_mode else "HttpResponse" + retval = [f">>> response = {'await ' if async_mode else ''}client.{self.client.send_request_name}(request)"] + retval.append(f"<{http_response}: 200 OK>") + return retval + + def _request_builder_example(self, async_mode: bool) -> List[str]: + retval = [ + "We have helper methods to create requests specific to this service in " + + f"`{self.client.code_model.namespace}.{self.client.code_model.rest_layer_name}`." + ] + retval.append("Use these helper methods to create the request you pass to this method.") + retval.append("") + + request_builder = self.client.request_builders[0] + request_builder_signature = ", ".join(request_builder.parameters.call) + if request_builder.group_name: + rest_imported = request_builder.group_name + request_builder_name = f"{request_builder.group_name}.{request_builder.name}" + else: + rest_imported = request_builder.name + request_builder_name = request_builder.name + full_path = f"{self.client.code_model.namespace}.{self.client.code_model.rest_layer_name}" + retval.append(f">>> from {full_path} import {rest_imported}") + retval.append(f">>> request = {request_builder_name}({request_builder_signature})") + retval.append(f"") + retval.extend(self._example_make_call(async_mode)) + return retval + + def _rest_request_example(self, async_mode: bool) -> List[str]: + retval = [f">>> from {self.client.code_model.core_library}.rest import HttpRequest"] + retval.append('>>> request = HttpRequest("GET", "https://www.example.org/")') + retval.append("") + retval.extend(self._example_make_call(async_mode)) + return retval + + def send_request_description(self, async_mode: bool) -> List[str]: + rest_library = f"{self.client.code_model.core_library}.rest" + retval = ['"""Runs the network request through the client\'s chained policies.'] + retval.append("") + if self.client.code_model.options["builders_visibility"] != "embedded": + retval.extend(self._request_builder_example(async_mode)) + else: + retval.extend(self._rest_request_example(async_mode)) + retval.append("") + retval.append("For more information on this code flow, see https://aka.ms/azsdk/dpcodegen/python/send_request") + retval.append("") + retval.append(":param request: The network request you want to make. Required.") + retval.append(f":type request: ~{rest_library}.HttpRequest") + retval.append(":keyword bool stream: Whether the response payload will be streamed. Defaults to False.") + retval.append(":return: The response of your network call. Does not do error handling on your response.") + http_response = "AsyncHttpResponse" if async_mode else "HttpResponse" + retval.append(f":rtype: ~{rest_library}.{http_response}") + retval.append('"""') + return retval + + def serialize_path(self) -> List[str]: + return self.parameter_serializer.serialize_path(self.client.parameters.path, "self._serialize") + + +class ConfigSerializer: + def __init__(self, client: Client) -> None: + self.client = client + self.parameter_serializer = ParameterSerializer() + + def _init_signature(self, async_mode: bool) -> str: + return self.parameter_serializer.serialize_method( + function_def="def", + method_name="__init__", + need_self_param=True, + method_param_signatures=self.client.config.parameters.method_signature(async_mode), + ) + + def init_signature_and_response_type_annotation(self, async_mode: bool) -> str: + init_signature = self._init_signature(async_mode) + return utils.method_signature_and_response_type_annotation_template( + method_signature=init_signature, + response_type_annotation="None", + ) + + def pop_kwargs_from_signature(self) -> List[str]: + return self.parameter_serializer.pop_kwargs_from_signature( + self.client.config.parameters.kwargs_to_pop, + check_kwarg_dict=False, + pop_headers_kwarg=PopKwargType.NO, + pop_params_kwarg=PopKwargType.NO, + ) + + def set_constants(self) -> List[str]: + return [ + f"self.{p.client_name} = {p.client_default_value_declaration}" + for p in self.client.config.parameters.constant + if p not in self.client.config.parameters.method + ] + + def check_required_parameters(self) -> List[str]: + return [ + f"if {p.client_name} is None:\n" f" raise ValueError(\"Parameter '{p.client_name}' must not be None.\")" + for p in self.client.config.parameters.method + if not (p.optional or p.constant) + ] + + def property_descriptions(self, async_mode: bool) -> List[str]: + retval: List[str] = [] + for p in self.client.config.parameters.method: + retval.append(f":{p.description_keyword} {p.client_name}: {p.description}") + retval.append(f":{p.docstring_type_keyword} {p.client_name}: {p.docstring_type(async_mode=async_mode)}") + retval.append('"""') + return retval diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/enum_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/enum_serializer.py new file mode 100644 index 00000000000..4b9ce87e3fc --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/enum_serializer.py @@ -0,0 +1,15 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +from .base_serializer import BaseSerializer +from ..models import FileImport + + +class EnumSerializer(BaseSerializer): + def serialize(self) -> str: + # Generate the enum file + template = self.env.get_template("enum_container.py.jinja2") + return template.render(code_model=self.code_model, file_import=FileImport(self.code_model)) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/general_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/general_serializer.py new file mode 100644 index 00000000000..b3b0877da3c --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/general_serializer.py @@ -0,0 +1,212 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import json +from typing import Any, List +from jinja2 import Environment +from .import_serializer import FileImportSerializer, TypingSection +from ..models.imports import MsrestImportType, FileImport +from ..models import ( + ImportType, + CodeModel, + TokenCredentialType, + Client, +) +from .client_serializer import ClientSerializer, ConfigSerializer +from .base_serializer import BaseSerializer + + +class GeneralSerializer(BaseSerializer): + """General serializer for SDK root level files""" + + def __init__(self, code_model: CodeModel, env: Environment, async_mode: bool): + super().__init__(code_model, env) + self.async_mode = async_mode + + def serialize_setup_file(self) -> str: + template = self.env.get_template("packaging_templates/setup.py.jinja2") + params = {} + params.update(self.code_model.options) + return template.render(code_model=self.code_model, **params) + + def serialize_package_file(self, template_name: str, **kwargs: Any) -> str: + template = self.env.get_template(template_name) + package_parts = (self.code_model.options["package_name"] or "").split("-")[:-1] + token_credential = any( + c for c in self.code_model.clients if isinstance(getattr(c.credential, "type", None), TokenCredentialType) + ) + version = self.code_model.options["package_version"] + if any(x in version for x in ["a", "b", "rc"]) or version[0] == "0": + dev_status = "4 - Beta" + else: + dev_status = "5 - Production/Stable" + params = { + "code_model": self.code_model, + "dev_status": dev_status, + "token_credential": token_credential, + "pkgutil_names": [".".join(package_parts[: i + 1]) for i in range(len(package_parts))], + "init_names": ["/".join(package_parts[: i + 1]) + "/__init__.py" for i in range(len(package_parts))], + "client_name": self.code_model.clients[0].name, + "namespace": self.code_model.namespace, + } + params.update(self.code_model.options) + params.update(kwargs) + return template.render(file_import=FileImport(self.code_model), **params) + + def serialize_pkgutil_init_file(self) -> str: + template = self.env.get_template("pkgutil_init.py.jinja2") + return template.render() + + def serialize_init_file(self, clients: List[Client]) -> str: + template = self.env.get_template("init.py.jinja2") + return template.render( + code_model=self.code_model, + clients=clients, + async_mode=self.async_mode, + ) + + def serialize_service_client_file(self, clients: List[Client]) -> str: + template = self.env.get_template("client_container.py.jinja2") + + imports = FileImport(self.code_model) + for client in clients: + imports.merge(client.imports(self.async_mode)) + + return template.render( + code_model=self.code_model, + clients=clients, + async_mode=self.async_mode, + get_serializer=ClientSerializer, + imports=FileImportSerializer(imports), + ) + + def serialize_vendor_file(self, clients: List[Client]) -> str: + template = self.env.get_template("vendor.py.jinja2") + + # configure imports + file_import = FileImport(self.code_model) + if self.code_model.need_mixin_abc: + file_import.add_submodule_import( + "abc", + "ABC", + ImportType.STDLIB, + ) + file_import.add_submodule_import( + "" if self.code_model.is_azure_flavor else "runtime", + f"{'Async' if self.async_mode else ''}PipelineClient", + ImportType.SDKCORE, + TypingSection.TYPING, + ) + file_import.add_msrest_import( + relative_path=".." if self.async_mode else ".", + msrest_import_type=MsrestImportType.SerializerDeserializer, + typing_section=TypingSection.TYPING, + ) + for client in clients: + file_import.add_submodule_import( + "._configuration", + f"{client.name}Configuration", + ImportType.LOCAL, + ) + if self.code_model.has_etag: + file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) + file_import.add_submodule_import( + "", + "MatchConditions", + ImportType.SDKCORE, + ) + if self.code_model.has_form_data and self.code_model.options["models_mode"] == "dpg" and not self.async_mode: + file_import.add_submodule_import("typing", "IO", ImportType.STDLIB) + file_import.add_submodule_import("typing", "Tuple", ImportType.STDLIB) + file_import.add_submodule_import("typing", "Union", ImportType.STDLIB) + file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) + file_import.add_submodule_import("typing", "Mapping", ImportType.STDLIB) + file_import.add_submodule_import("typing", "Sequence", ImportType.STDLIB) + file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) + file_import.add_submodule_import("typing", "List", ImportType.STDLIB) + file_import.add_submodule_import( + "._model_base", + "SdkJSONEncoder", + ImportType.LOCAL, + ) + file_import.add_submodule_import( + "._model_base", + "Model", + ImportType.LOCAL, + ) + file_import.add_import("json", ImportType.STDLIB) + + return template.render( + code_model=self.code_model, + imports=FileImportSerializer( + file_import, + ), + async_mode=self.async_mode, + clients=clients, + ) + + def serialize_config_file(self, clients: List[Client]) -> str: + template = self.env.get_template("config_container.py.jinja2") + imports = FileImport(self.code_model) + for client in self.code_model.clients: + imports.merge(client.config.imports(self.async_mode)) + return template.render( + code_model=self.code_model, + async_mode=self.async_mode, + imports=FileImportSerializer(imports), + get_serializer=ConfigSerializer, + clients=clients, + ) + + def serialize_version_file(self) -> str: + template = self.env.get_template("version.py.jinja2") + return template.render(code_model=self.code_model) + + def serialize_serialization_file(self) -> str: + template = self.env.get_template("serialization.py.jinja2") + return template.render( + code_model=self.code_model, + ) + + def serialize_model_base_file(self) -> str: + template = self.env.get_template("model_base.py.jinja2") + return template.render(code_model=self.code_model, file_import=FileImport(self.code_model)) + + def serialize_validation_file(self) -> str: + template = self.env.get_template("validation.py.jinja2") + return template.render(code_model=self.code_model) + + def serialize_cross_language_definition_file(self) -> str: + cross_langauge_def_dict = { + f"{self.code_model.namespace}.models.{model.name}": model.cross_language_definition_id + for model in self.code_model.model_types + } + cross_langauge_def_dict.update( + { + f"{self.code_model.namespace}.models.{enum.name}": enum.cross_language_definition_id + for enum in self.code_model.enums + } + ) + cross_langauge_def_dict.update( + { + ( + f"{self.code_model.namespace}.{client.name}." + + ("" if operation_group.is_mixin else f"{operation_group.property_name}.") + + f"{operation.name}" + ): operation.cross_language_definition_id + for client in self.code_model.clients + for operation_group in client.operation_groups + for operation in operation_group.operations + if not operation.name.startswith("_") + } + ) + return json.dumps( + { + "CrossLanguagePackageId": self.code_model.cross_language_package_id, + "CrossLanguageDefinitionId": cross_langauge_def_dict, + }, + indent=4, + ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/import_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/import_serializer.py new file mode 100644 index 00000000000..2824a1e6750 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/import_serializer.py @@ -0,0 +1,127 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from copy import deepcopy +from typing import List +from ..models.imports import ( + ImportType, + FileImport, + ImportModel, + TypingSection, + TypeDefinition, +) + + +def _serialize_package(imports: List[ImportModel], delimiter: str) -> str: + buffer = [] + if any(i for i in imports if i.submodule_name is None): + buffer.append(f"import {imports[0].module_name}{f' as {imports[0].alias}' if imports[0].alias else ''}") + else: + import_str = ", ".join( + sorted( + set( + f"{i.submodule_name} as {i.alias}" if i.alias else i.submodule_name for i in imports # type: ignore + ) + ) + ) + buffer.append(f"from {imports[0].module_name} import {import_str}") + return delimiter.join(buffer) + + +def _serialize_versioned_package(i: ImportModel, delimiter: str) -> str: + if not i.version_modules: + return "" + buffer = [] + for n, (version, module_name, comment) in enumerate(i.version_modules): + buffer.append("{} sys.version_info >= {}:".format("if" if n == 0 else "elif", version)) + buffer.append( + f" from {module_name} import {i.submodule_name}{f' as {i.alias}' if i.alias else ''}" + f"{f' # {comment}' if comment else ''}" + ) + buffer.append("else:") + buffer.append( + f" from {i.module_name} import {i.submodule_name}{f' as {i.alias}' if i.alias else ''}" + " # type: ignore # pylint: disable=ungrouped-imports" + ) + return delimiter.join(buffer) + + +def _serialize_import_type(imports: List[ImportModel], delimiter: str) -> str: + """Serialize a given import type.""" + import_list = [] + for module_name in sorted(set(i.module_name for i in imports)): + normal_imports = [i for i in imports if i.module_name == module_name and not i.version_modules] + versioned_imports = [i for i in imports if i.module_name == module_name and i.version_modules] + if normal_imports: + import_list.append(_serialize_package(normal_imports, delimiter)) + for i in versioned_imports: + import_list.append(_serialize_versioned_package(i, delimiter)) + return delimiter.join(import_list) + + +def _get_import_clauses(imports: List[ImportModel], delimiter: str) -> List[str]: + import_clause = [] + for import_type in ImportType: + imports_with_import_type = [i for i in imports if i.import_type == import_type] + if imports_with_import_type: + import_clause.append(_serialize_import_type(imports_with_import_type, delimiter)) + return import_clause + + +class FileImportSerializer: + def __init__(self, file_import: FileImport, async_mode: bool = False) -> None: + self.file_import = file_import + self.async_mode = async_mode + + def _get_imports_list(self, baseline_typing_section: TypingSection, add_conditional_typing: bool): + # If this is a python 3 file, our regular imports include the CONDITIONAL category + # If this is not a python 3 file, our typing imports include the CONDITIONAL category + file_import_copy = deepcopy(self.file_import) + if add_conditional_typing and any(self.file_import.get_imports_from_section(TypingSection.CONDITIONAL)): + # we switch the TypingSection key for the CONDITIONAL typing imports so we can merge + # the imports together + for i in file_import_copy.imports: + if i.typing_section == TypingSection.CONDITIONAL: + i.typing_section = baseline_typing_section + return file_import_copy.get_imports_from_section(baseline_typing_section) + + def _add_type_checking_import(self): + if any(self.file_import.get_imports_from_section(TypingSection.TYPING)): + self.file_import.add_submodule_import("typing", "TYPE_CHECKING", ImportType.STDLIB) + + def get_typing_definitions(self) -> str: + def declare_definition(type_name: str, type_definition: TypeDefinition) -> List[str]: + ret: List[str] = [] + definition_value = type_definition.async_definition if self.async_mode else type_definition.sync_definition + ret.append("{} = {}".format(type_name, definition_value)) + return ret + + if not self.file_import.type_definitions: + return "" + declarations: List[str] = [""] + for type_name, value in self.file_import.type_definitions.items(): + declarations.extend(declare_definition(type_name, value)) + return "\n".join(declarations) + + def __str__(self) -> str: + self._add_type_checking_import() + regular_imports = "" + regular_imports_list = self._get_imports_list( + baseline_typing_section=TypingSection.REGULAR, + add_conditional_typing=True, + ) + + if regular_imports_list: + regular_imports = "\n\n".join(_get_import_clauses(regular_imports_list, "\n")) + + typing_imports = "" + typing_imports_list = self._get_imports_list( + baseline_typing_section=TypingSection.TYPING, + add_conditional_typing=False, + ) + if typing_imports_list: + typing_imports += "\n\nif TYPE_CHECKING:\n # pylint: disable=unused-import,ungrouped-imports\n " + typing_imports += "\n\n ".join(_get_import_clauses(typing_imports_list, "\n ")) + return regular_imports + typing_imports + self.get_typing_definitions() diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/metadata_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/metadata_serializer.py new file mode 100644 index 00000000000..35d374983fc --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/metadata_serializer.py @@ -0,0 +1,198 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import functools +import json +from typing import List, Optional, Set, Tuple, Dict, Union, Any +from jinja2 import Environment +from ..models import ( + OperationGroup, + LROOperation, + PagingOperation, + TypingSection, + ImportType, + CodeModel, +) +from .builder_serializer import get_operation_serializer +from .import_serializer import FileImportSerializer + + +def _to_string(data: Union[Tuple[Any], List[Any], str]) -> str: + if isinstance(data, (list, tuple)): + return "".join([_to_string(item) for item in data]) + return str(data) + + +def _json_serialize_imports( + imports: Dict[ + TypingSection, + Dict[ + ImportType, + Dict[ + str, + Set[ + Optional[ + Union[ + str, + Tuple[str, str], + Tuple[ + str, + Optional[str], + Tuple[Tuple[Tuple[int, int], str, Optional[str]]], + ], + ] + ] + ], + ], + ], + ] +) -> str: + if not imports: + return "" + + json_serialize_imports = {} + # need to make name_import set -> list to make the dictionary json serializable + # not using an OrderedDict since we're iterating through a set and the order there varies + # going to sort the list instead + + for typing_section_key, typing_section_value in imports.items(): + json_import_type_dictionary = {} + for import_type_key, import_type_value in typing_section_value.items(): + json_package_name_dictionary = {} + for package_name, name_imports in import_type_value.items(): + name_import_ordered_list = [] + if name_imports: + name_import_ordered_list = list(name_imports) + name_import_ordered_list.sort( + key=lambda e: ( + _to_string(e) # type: ignore + if isinstance(e, (list, tuple)) + else e if isinstance(e, str) else "" + ) + ) + json_package_name_dictionary[package_name] = name_import_ordered_list + json_import_type_dictionary[import_type_key] = json_package_name_dictionary + json_serialize_imports[typing_section_key] = json_import_type_dictionary + return json.dumps(json_serialize_imports) + + +def _mixin_imports( + mixin_operation_group: Optional[OperationGroup], +) -> Tuple[Optional[str], Optional[str]]: + if not mixin_operation_group: + return None, None + + sync_mixin_imports = mixin_operation_group.imports_for_multiapi(async_mode=False) + async_mixin_imports = mixin_operation_group.imports_for_multiapi(async_mode=True) + + return _json_serialize_imports(sync_mixin_imports.to_dict()), _json_serialize_imports(async_mixin_imports.to_dict()) + + +def _mixin_typing_definitions( + mixin_operation_group: Optional[OperationGroup], +) -> Tuple[Optional[str], Optional[str]]: + if not mixin_operation_group: + return None, None + + sync_mixin_imports = mixin_operation_group.imports_for_multiapi(async_mode=False) + async_mixin_imports = mixin_operation_group.imports_for_multiapi(async_mode=True) + sync_mixin_typing_definitions = FileImportSerializer(sync_mixin_imports, False).get_typing_definitions() + async_mixin_typing_definitions = FileImportSerializer(async_mixin_imports, True).get_typing_definitions() + + return sync_mixin_typing_definitions, async_mixin_typing_definitions + + +class MetadataSerializer: + def __init__(self, code_model: CodeModel, env: Environment) -> None: + self.code_model = code_model + self.client = self.code_model.clients[0] # we only do one client for multiapi + self.env = env + + def _choose_api_version(self) -> Tuple[str, List[str]]: + chosen_version = "" + total_api_version_set: Set[str] = set() + for client in self.code_model.clients: + for operation_group in client.operation_groups: + total_api_version_set.update(operation_group.api_versions) + + total_api_version_list = list(total_api_version_set) + total_api_version_list.sort() + + # switching ' to " so json can decode the dict we end up writing to file + total_api_version_list = [str(api_version).replace("'", '"') for api_version in total_api_version_list] + if len(total_api_version_list) == 1: + chosen_version = total_api_version_list[0] + elif len(total_api_version_list) > 1: + module_version = self.code_model.namespace.split(".")[-1] + for api_version in total_api_version_list: + if "v{}".format(api_version.replace("-", "_")) == module_version: + chosen_version = api_version + + return chosen_version, total_api_version_list + + def serialize(self) -> str: + def _is_lro(operation): + return isinstance(operation, LROOperation) + + def _is_paging(operation): + return isinstance(operation, PagingOperation) + + mixin_operation_group: Optional[OperationGroup] = next( + ( + operation_group + for client in self.code_model.clients + for operation_group in client.operation_groups + if operation_group.is_mixin + ), + None, + ) + mixin_operations = mixin_operation_group.operations if mixin_operation_group else [] + sync_mixin_imports, async_mixin_imports = _mixin_imports(mixin_operation_group) + ( + sync_mixin_typing_definitions, + async_mixin_typing_definitions, + ) = _mixin_typing_definitions(mixin_operation_group) + + chosen_version, total_api_version_list = self._choose_api_version() + + # setting to true, because for multiapi we always generate with a version file with version 0.1.0 + self.code_model.options["package_version"] = "0.1.0" + template = self.env.get_template("metadata.json.jinja2") + + return template.render( + code_model=self.code_model, + chosen_version=chosen_version, + total_api_version_list=total_api_version_list, + client=self.client, + global_parameters=self.client.parameters, + mixin_operations=mixin_operations, + any=any, + is_lro=_is_lro, + is_paging=_is_paging, + str=str, + sync_mixin_imports=sync_mixin_imports, + async_mixin_imports=async_mixin_imports, + sync_mixin_typing_definitions=sync_mixin_typing_definitions, + async_mixin_typing_definitions=async_mixin_typing_definitions, + sync_client_imports=_json_serialize_imports(self.client.imports_for_multiapi(async_mode=False).to_dict()), + async_client_imports=_json_serialize_imports(self.client.imports_for_multiapi(async_mode=True).to_dict()), + sync_config_imports=_json_serialize_imports( + self.client.config.imports_for_multiapi(async_mode=False).to_dict() + ), + async_config_imports=_json_serialize_imports( + self.client.config.imports_for_multiapi(async_mode=True).to_dict() + ), + get_async_operation_serializer=functools.partial( + get_operation_serializer, + code_model=self.client.code_model, + async_mode=True, + ), + get_sync_operation_serializer=functools.partial( + get_operation_serializer, + code_model=self.client.code_model, + async_mode=False, + ), + has_credential=bool(self.client.credential), + ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/model_init_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/model_init_serializer.py new file mode 100644 index 00000000000..5df688adbb8 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/model_init_serializer.py @@ -0,0 +1,33 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from jinja2 import Environment +from ..models import CodeModel + + +class ModelInitSerializer: + def __init__(self, code_model: CodeModel, env: Environment) -> None: + self.code_model = code_model + self.env = env + + def serialize(self) -> str: + schemas = [s.name for s in self.code_model.public_model_types] + schemas.sort() + enums = [e.name for e in self.code_model.enums if not e.internal] if self.code_model.enums else None + + if enums: + enums.sort() + + # check to see if we have any duplicate names between enum and object schemas + model_enum_name_intersection = set(schemas).intersection(set(enums)) + if model_enum_name_intersection: + raise ValueError( + "We have models and enums sharing the following names: {}".format( + ", ".join(model_enum_name_intersection) + ) + ) + + template = self.env.get_template("model_init.py.jinja2") + return template.render(code_model=self.code_model, schemas=schemas, enums=enums) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/model_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/model_serializer.py new file mode 100644 index 00000000000..b42d35e9709 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/model_serializer.py @@ -0,0 +1,287 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import List +from abc import ABC, abstractmethod + +from ..models import ModelType, Property, ConstantType, EnumValue +from ..models.imports import FileImport, TypingSection, MsrestImportType, ImportType +from .import_serializer import FileImportSerializer +from .base_serializer import BaseSerializer + + +def _documentation_string(prop: Property, description_keyword: str, docstring_type_keyword: str) -> List[str]: + retval: List[str] = [] + sphinx_prefix = f":{description_keyword} {prop.client_name}:" + description = prop.description(is_operation_file=False).replace("\\", "\\\\") + retval.append(f"{sphinx_prefix} {description}" if description else sphinx_prefix) + retval.append(f":{docstring_type_keyword} {prop.client_name}: {prop.type.docstring_type()}") + return retval + + +class _ModelSerializer(BaseSerializer, ABC): + @abstractmethod + def imports(self) -> FileImport: ... + + def serialize(self) -> str: + # Generate the models + template = self.env.get_template("model_container.py.jinja2") + return template.render( + code_model=self.code_model, + imports=FileImportSerializer(self.imports()), + str=str, + serializer=self, + ) + + @abstractmethod + def declare_model(self, model: ModelType) -> str: ... + + @staticmethod + def escape_dot(s: str): + return s.replace(".", "\\\\.") + + @staticmethod + def input_documentation_string(prop: Property) -> List[str]: + # building the param line of the property doc + return _documentation_string(prop, "keyword", "paramtype") + + @staticmethod + def variable_documentation_string(prop: Property) -> List[str]: + return _documentation_string(prop, "ivar", "vartype") + + def super_call(self, model: ModelType) -> List[str]: + return [f"super().__init__({self.properties_to_pass_to_super(model)})"] + + @staticmethod + def initialize_discriminator_property(model: ModelType, prop: Property) -> str: + discriminator_value = f"'{model.discriminator_value}'" if model.discriminator_value else None + if not discriminator_value: + typing = "Optional[str]" + else: + typing = "str" + return f"self.{prop.client_name}: {typing} = {discriminator_value}" + + @staticmethod + def initialize_standard_property(prop: Property): + if not (prop.optional or prop.client_default_value is not None): + return f"{prop.client_name}: {prop.type_annotation()},{prop.pylint_disable}" + return ( + f"{prop.client_name}: {prop.type_annotation()} = " + f"{prop.client_default_value_declaration},{prop.pylint_disable}" + ) + + @staticmethod + def discriminator_docstring(model: ModelType) -> str: + return ( + "You probably want to use the sub-classes and not this class directly. " + f"Known sub-classes are: {', '.join(v.name for v in model.discriminated_subtypes.values())}" + ) + + @staticmethod + def _init_line_parameters(model: ModelType): + return [p for p in model.properties if not p.readonly and not p.is_discriminator and not p.constant] + + def init_line(self, model: ModelType) -> List[str]: + init_properties_declaration = [] + init_line_parameters = self._init_line_parameters(model) + init_line_parameters.sort(key=lambda x: x.optional) + if init_line_parameters: + init_properties_declaration.append("*,") + for param in init_line_parameters: + init_properties_declaration.append(self.initialize_standard_property(param)) + + return init_properties_declaration + + @staticmethod + def properties_to_pass_to_super(model: ModelType) -> str: + properties_to_pass_to_super = [] + for parent in model.parents: + for prop in model.properties: + if prop in parent.properties and not prop.is_discriminator and not prop.constant and not prop.readonly: + properties_to_pass_to_super.append(f"{prop.client_name}={prop.client_name}") + properties_to_pass_to_super.append("**kwargs") + return ", ".join(properties_to_pass_to_super) + + +class MsrestModelSerializer(_ModelSerializer): + def imports(self) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_msrest_import( + relative_path="..", + msrest_import_type=MsrestImportType.Module, + typing_section=TypingSection.REGULAR, + ) + for model in self.code_model.model_types: + file_import.merge(model.imports(is_operation_file=False)) + for param in self._init_line_parameters(model): + file_import.merge(param.imports()) + + return file_import + + def declare_model(self, model: ModelType) -> str: + basename = ( + "msrest.serialization.Model" + if self.code_model.options["client_side_validation"] + else "_serialization.Model" + ) + if model.parents: + basename = ", ".join([m.name for m in model.parents]) + return f"class {model.name}({basename}):{model.pylint_disable}" + + @staticmethod + def get_properties_to_initialize(model: ModelType) -> List[Property]: + if model.parents: + properties_to_initialize = list( + { + p.client_name: p + for bm in model.parents + for p in model.properties + if p not in bm.properties or p.is_discriminator or p.constant + }.values() + ) + else: + properties_to_initialize = model.properties + return properties_to_initialize + + def initialize_properties(self, model: ModelType) -> List[str]: + init_args = [] + for prop in self.get_properties_to_initialize(model): + if prop.is_discriminator: + init_args.append(self.initialize_discriminator_property(model, prop)) + elif prop.readonly: + init_args.append(f"self.{prop.client_name} = None") + elif not prop.constant: + init_args.append(f"self.{prop.client_name} = {prop.client_name}") + return init_args + + @staticmethod + def declare_property(prop: Property) -> str: + if prop.flattened_names: + attribute_key = ".".join(_ModelSerializer.escape_dot(n) for n in prop.flattened_names) + else: + attribute_key = _ModelSerializer.escape_dot(prop.wire_name) + if prop.type.xml_serialization_ctxt: + xml_metadata = f", 'xml': {{{prop.type.xml_serialization_ctxt}}}" + else: + xml_metadata = "" + return ( + f'"{prop.client_name}": {{"key": "{attribute_key}",' + f' "type": "{prop.msrest_deserialization_key}"{xml_metadata}}},' + ) + + +class DpgModelSerializer(_ModelSerializer): + def super_call(self, model: ModelType) -> List[str]: + super_call = f"super().__init__({self.properties_to_pass_to_super(model)})" + if model.flattened_property: + return [ + "_flattened_input = {k: kwargs.pop(k) for k in kwargs.keys() & self.__flattened_items}", + super_call, + "for k, v in _flattened_input.items():", + " setattr(self, k, v)", + ] + return [super_call] + + def imports(self) -> FileImport: + file_import = FileImport(self.code_model) + file_import.add_submodule_import( + "..", + "_model_base", + ImportType.LOCAL, + TypingSection.REGULAR, + ) + + for model in self.code_model.model_types: + file_import.merge(model.imports(is_operation_file=False)) + for prop in model.properties: + file_import.merge(prop.imports()) + if model.is_polymorphic: + file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) + if not model.internal and self.init_line(model): + file_import.add_submodule_import("typing", "overload", ImportType.STDLIB) + file_import.add_submodule_import("typing", "Mapping", ImportType.STDLIB) + file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) + return file_import + + def declare_model(self, model: ModelType) -> str: + basename = "_model_base.Model" + if model.parents: + basename = ", ".join([m.name for m in model.parents]) + if model.discriminator_value: + basename += f", discriminator='{model.discriminator_value}'" + return f"class {model.name}({basename}):{model.pylint_disable}" + + @staticmethod + def get_properties_to_declare(model: ModelType) -> List[Property]: + if model.parents: + parent_properties = [p for bm in model.parents for p in bm.properties] + properties_to_declare = [ + p + for p in model.properties + if not any( + p.client_name == pp.client_name + and p.type_annotation() == pp.type_annotation() + and not p.is_base_discriminator + for pp in parent_properties + ) + ] + else: + properties_to_declare = model.properties + if any(p for p in properties_to_declare if p.client_name == "_"): + raise ValueError("We do not generate anonymous properties") + return properties_to_declare + + @staticmethod + def declare_property(prop: Property) -> str: + args = [] + if prop.client_name != prop.wire_name or prop.is_discriminator: + args.append(f'name="{prop.wire_name}"') + if prop.visibility: + v_list = ", ".join(f'"{x}"' for x in prop.visibility) + args.append(f"visibility=[{v_list}]") + if prop.client_default_value is not None: + args.append(f"default={prop.client_default_value_declaration}") + + if prop.is_multipart_file_input: + args.append("is_multipart_file_input=True") + elif hasattr(prop.type, "encode") and prop.type.encode: # type: ignore + args.append(f'format="{prop.type.encode}"') # type: ignore + + field = "rest_discriminator" if prop.is_discriminator else "rest_field" + type_ignore = prop.is_discriminator and isinstance(prop.type, (ConstantType, EnumValue)) and prop.type.value + return ( + f"{prop.client_name}: {prop.type_annotation()} =" + f' {field}({", ".join(args)}){" # type: ignore" if type_ignore else ""}' + ) + + def initialize_properties(self, model: ModelType) -> List[str]: + init_args = [] + for prop in self.get_properties_to_declare(model): + if prop.constant and not prop.is_base_discriminator: + init_args.append(f"self.{prop.client_name}: {prop.type_annotation()} = " f"{prop.get_declaration()}") + return init_args + + @staticmethod + def _init_line_parameters(model: ModelType): + return [ + p + for p in model.properties + if p.is_base_discriminator or not p.is_discriminator and not p.constant and p.visibility != ["read"] + ] + + @staticmethod + def properties_to_pass_to_super(model: ModelType) -> str: + properties_to_pass_to_super = ["*args"] + for parent in model.parents: + for prop in model.properties: + if ( + prop.client_name in [prop.client_name for prop in parent.properties if prop.is_base_discriminator] + and prop.is_discriminator + and not prop.constant + and not prop.readonly + ): + properties_to_pass_to_super.append(f"{prop.client_name}={prop.get_declaration()}") + properties_to_pass_to_super.append("**kwargs") + return ", ".join(properties_to_pass_to_super) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/operation_groups_serializer.py new file mode 100644 index 00000000000..4c04efbcd2e --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/operation_groups_serializer.py @@ -0,0 +1,89 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Optional, List, Union +import functools +from jinja2 import Environment + +from .utils import get_all_operation_groups_recursively +from ..models import ( + CodeModel, + OperationGroup, + RequestBuilder, + OverloadedRequestBuilder, + Client, + FileImport, +) +from .import_serializer import FileImportSerializer +from .builder_serializer import ( + get_operation_serializer, + RequestBuilderSerializer, +) +from .base_serializer import BaseSerializer + + +class OperationGroupsSerializer(BaseSerializer): + def __init__( + self, + code_model: CodeModel, + clients: List[Client], + env: Environment, + async_mode: bool, + operation_group: Optional[OperationGroup] = None, + ): + super().__init__(code_model, env) + self.clients = clients + self.async_mode = async_mode + self.operation_group = operation_group + + def _get_request_builders( + self, operation_group: OperationGroup + ) -> List[Union[OverloadedRequestBuilder, RequestBuilder]]: + return [ + r + for client in self.clients + for r in client.request_builders + if r.client.name == operation_group.client.name + and r.group_name == operation_group.identify_name + and not r.is_overload + and not r.abstract + and not r.is_lro # lro has already initial builder + ] + + def serialize(self) -> str: + if self.operation_group: + operation_groups = [self.operation_group] + else: + operation_groups = get_all_operation_groups_recursively(self.clients) + + imports = FileImport(self.code_model) + for operation_group in operation_groups: + imports.merge( + operation_group.imports( + async_mode=self.async_mode, + ) + ) + + template = self.env.get_or_select_template("operation_groups_container.py.jinja2") + + return template.render( + code_model=self.code_model, + operation_groups=operation_groups, + imports=FileImportSerializer( + imports, + async_mode=self.async_mode, + ), + async_mode=self.async_mode, + get_operation_serializer=functools.partial( + get_operation_serializer, + code_model=self.code_model, + async_mode=self.async_mode, + ), + request_builder_serializer=RequestBuilderSerializer( + self.code_model, + async_mode=False, + ), + get_request_builders=self._get_request_builders, + ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/operations_init_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/operations_init_serializer.py new file mode 100644 index 00000000000..02232c527c5 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/operations_init_serializer.py @@ -0,0 +1,44 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import List +from jinja2 import Environment + +from ..models.operation_group import OperationGroup +from ..models import CodeModel, Client + + +class OperationsInitSerializer: + def __init__( + self, + code_model: CodeModel, + clients: List[Client], + env: Environment, + async_mode: bool, + ) -> None: + self.code_model = code_model + self.clients = clients + self.env = env + self.async_mode = async_mode + + def operation_group_imports(self) -> List[str]: + def _get_filename(operation_group: OperationGroup) -> str: + return "_operations" if self.code_model.options["combine_operation_files"] else operation_group.filename + + return [ + f"from .{_get_filename(og)} import {og.class_name}" + for client in self.clients + for og in client.operation_groups + ] + + def serialize(self) -> str: + operation_group_init_template = self.env.get_template("operations_folder_init.py.jinja2") + + return operation_group_init_template.render( + code_model=self.code_model, + async_mode=self.async_mode, + operation_group_imports=self.operation_group_imports, + clients=self.clients, + ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/parameter_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/parameter_serializer.py new file mode 100644 index 00000000000..8a41c850a7d --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/parameter_serializer.py @@ -0,0 +1,221 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import List, Sequence, Union, Optional, Dict +from enum import Enum, auto + +from ..models import ( + Parameter, + ParameterLocation, + ListType, + ParameterDelimeter, + RequestBuilderParameter, + ClientParameter, + ConfigParameter, + ParameterType, +) +from ..models.parameter import _ParameterBase + + +class PopKwargType(Enum): + NO = auto() + SIMPLE = auto() + CASE_INSENSITIVE = auto() + + +SPECIAL_HEADER_SERIALIZATION: Dict[str, List[str]] = { + "repeatability-request-id": [ + """if "Repeatability-Request-ID" not in _headers:""", + """ _headers["Repeatability-Request-ID"] = str(uuid.uuid4())""", + ], + "repeatability-first-sent": [ + """if "Repeatability-First-Sent" not in _headers:""", + """ _headers["Repeatability-First-Sent"] = _SERIALIZER.serialize_data(""", + """ datetime.datetime.now(datetime.timezone.utc), "rfc-1123")""", + ], + "client-request-id": [], + "x-ms-client-request-id": [], + "return-client-request-id": [], + "etag": [ + """if_match = prep_if_match(etag, match_condition)""", + """if if_match is not None:""", + """ _headers["If-Match"] = _SERIALIZER.header("if_match", if_match, "str")""", + ], + "match-condition": [ + """if_none_match = prep_if_none_match(etag, match_condition)""", + """if if_none_match is not None:""", + """ _headers["If-None-Match"] = _SERIALIZER.header("if_none_match", if_none_match, "str")""", + ], +} + + +class ParameterSerializer: + @staticmethod + def serialize_parameter(parameter: ParameterType, serializer_name: str) -> str: + optional_parameters = [] + + if parameter.skip_url_encoding: + optional_parameters.append("skip_quote=True") + + if parameter.delimiter and not parameter.explode: + if parameter.delimiter == ParameterDelimeter.COMMA: + div_char = "," + elif parameter.delimiter == ParameterDelimeter.SPACE: + div_char = " " + elif parameter.delimiter == ParameterDelimeter.PIPE: + div_char = "|" + elif parameter.delimiter == ParameterDelimeter.TAB: + div_char = "\t" + else: + raise ValueError(f"We do not support {parameter.delimiter} yet") + optional_parameters.append(f"div='{div_char}'") + + if parameter.explode: + if not isinstance(parameter.type, ListType): + raise ValueError("Got a explode boolean on a non-array schema") + type = parameter.type.element_type + else: + type = parameter.type + + serialization_constraints = type.serialization_constraints + if serialization_constraints: + optional_parameters += serialization_constraints + + origin_name = parameter.full_client_name + + parameters = [ + f'"{origin_name.lstrip("_")}"', + "q" if parameter.explode else origin_name, + f"'{type.serialization_type}'", + *optional_parameters, + ] + parameters_line = ", ".join(parameters) + + msrest_function_name = { + ParameterLocation.PATH: "url", + ParameterLocation.ENDPOINT_PATH: "url", + ParameterLocation.HEADER: "header", + ParameterLocation.QUERY: "query", + }[parameter.location] + + serialize_line = f"{serializer_name}.{msrest_function_name}({parameters_line})" + + if parameter.explode: + return f"[{serialize_line} if q is not None else '' for q in {origin_name}]" + return serialize_line + + @staticmethod + def serialize_path( + parameters: Union[ + List[Parameter], + List[RequestBuilderParameter], + List[ClientParameter], + List[ConfigParameter], + ], + serializer_name: str, + ) -> List[str]: + retval = ["path_format_arguments = {"] + retval.extend( + [ + ' "{}": {},'.format( + path_parameter.wire_name, + ParameterSerializer.serialize_parameter(path_parameter, serializer_name), + ) + for path_parameter in parameters + ] + ) + retval.append("}") + return retval + + @staticmethod + def serialize_query_header( + param: Parameter, + kwarg_name: str, + serializer_name: str, + is_legacy: bool, + ) -> List[str]: + if ( + not is_legacy + and param.location == ParameterLocation.HEADER + and param.wire_name.lower() in SPECIAL_HEADER_SERIALIZATION + ): + return SPECIAL_HEADER_SERIALIZATION[param.wire_name.lower()] + + set_parameter = "_{}['{}'] = {}".format( + kwarg_name, + param.wire_name, + ParameterSerializer.serialize_parameter(param, serializer_name), + ) + if not param.optional: + retval = [set_parameter] + else: + retval = [ + f"if {param.full_client_name} is not None:", + f" {set_parameter}", + ] + return retval + + @staticmethod + def pop_kwargs_from_signature( + parameters: Sequence[_ParameterBase], + check_kwarg_dict: bool, + pop_headers_kwarg: PopKwargType, + pop_params_kwarg: PopKwargType, + check_client_input: bool = False, + operation_name: Optional[str] = None, + ) -> List[str]: + retval = [] + + def append_pop_kwarg(key: str, pop_type: PopKwargType) -> None: + if PopKwargType.CASE_INSENSITIVE == pop_type: + retval.append(f'_{key} = case_insensitive_dict(kwargs.pop("{key}", {{}}) or {{}})') + elif PopKwargType.SIMPLE == pop_type: + retval.append(f'_{key} = kwargs.pop("{key}", {{}}) or {{}}') + + append_pop_kwarg("headers", pop_headers_kwarg) + append_pop_kwarg("params", pop_params_kwarg) + if pop_headers_kwarg != PopKwargType.NO or pop_params_kwarg != PopKwargType.NO: + retval.append("") + for kwarg in parameters: + type_annot = kwarg.type_annotation() + if kwarg.client_default_value is not None or kwarg.optional: + if check_client_input and kwarg.check_client_input: + default_value = f"self._config.{kwarg.client_name}" + else: + default_value = kwarg.client_default_value_declaration + if check_kwarg_dict and (kwarg.location in [ParameterLocation.HEADER, ParameterLocation.QUERY]): + kwarg_dict = "headers" if kwarg.location == ParameterLocation.HEADER else "params" + if ( + kwarg.client_name == "api_version" + and kwarg.code_model.options["multiapi"] + and operation_name is not None + ): + default_value = f"self._api_version{operation_name} or {default_value}" + default_value = f"_{kwarg_dict}.pop('{kwarg.wire_name}', {default_value})" + + retval.append( + f"{kwarg.client_name}: {type_annot} = kwargs.pop('{kwarg.client_name}', " + f"{default_value})" + ) + else: + retval.append(f"{kwarg.client_name}: {type_annot} = kwargs.pop('{kwarg.client_name}')") + return retval + + @staticmethod + def serialize_method( + *, + function_def: str, + method_name: str, + need_self_param: bool, + method_param_signatures: List[str], + pylint_disable: str = "", + ): + lines: List[str] = [] + first_line = f"{function_def} {method_name}({pylint_disable}" + lines.append(first_line) + if need_self_param: + lines.append(" self,") + lines.extend([(" " + line) for line in method_param_signatures]) + lines.append(")") + return "\n".join(lines) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/patch_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/patch_serializer.py new file mode 100644 index 00000000000..8e9c8ef8e7f --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/patch_serializer.py @@ -0,0 +1,19 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from .import_serializer import FileImportSerializer +from ..models import ImportType, FileImport +from .base_serializer import BaseSerializer + + +class PatchSerializer(BaseSerializer): + def serialize(self) -> str: + template = self.env.get_template("patch.py.jinja2") + imports = FileImport(self.code_model) + imports.add_submodule_import("typing", "List", ImportType.STDLIB) + return template.render( + code_model=self.code_model, + imports=FileImportSerializer(imports), + ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/request_builders_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/request_builders_serializer.py new file mode 100644 index 00000000000..f9829843646 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/request_builders_serializer.py @@ -0,0 +1,52 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import List +from jinja2 import Environment + +from ..models import FileImport +from .import_serializer import FileImportSerializer +from ..models import CodeModel, RequestBuilderType +from .builder_serializer import RequestBuilderSerializer +from .base_serializer import BaseSerializer + + +class RequestBuildersSerializer(BaseSerializer): + def __init__( + self, + code_model: CodeModel, + env: Environment, + request_builders: List[RequestBuilderType], + ) -> None: + super().__init__(code_model, env) + self.request_builders = request_builders + self.group_name = request_builders[0].group_name + + @property + def imports(self) -> FileImport: + file_import = FileImport(self.code_model) + for request_builder in self.request_builders: + if request_builder.group_name == self.group_name: + file_import.merge(request_builder.imports()) + return file_import + + def serialize_init(self) -> str: + template = self.env.get_template("rest_init.py.jinja2") + return template.render( + code_model=self.code_model, + request_builders=[r for r in self.request_builders if not r.is_overload], + ) + + def serialize_request_builders(self) -> str: + template = self.env.get_template("request_builders.py.jinja2") + + return template.render( + code_model=self.code_model, + request_builders=[rb for rb in self.request_builders if not rb.abstract], + imports=FileImportSerializer( + self.imports, + ), + request_builder_serializer=RequestBuilderSerializer(self.code_model, async_mode=False), + ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/sample_serializer.py new file mode 100644 index 00000000000..5694804c687 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/sample_serializer.py @@ -0,0 +1,163 @@ +# pylint: disable=too-many-lines +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from typing import Dict, Any, Union, Tuple +from jinja2 import Environment + +from ..models.operation import OperationBase +from .import_serializer import FileImportSerializer +from .base_serializer import BaseSerializer +from ..models import ( + CodeModel, + KeyCredentialType, + TokenCredentialType, + ImportType, + OperationGroup, + Parameter, + BodyParameter, + FileImport, +) +from .utils import get_namespace_config, get_namespace_from_package_name +from ...utils import to_snake_case + +_LOGGER = logging.getLogger(__name__) + + +class SampleSerializer(BaseSerializer): + def __init__( + self, + code_model: CodeModel, + env: Environment, + operation_group: OperationGroup, + operation: OperationBase[Any], + sample: Dict[str, Any], + file_name: str, + ) -> None: + super().__init__(code_model, env) + self.operation_group = operation_group + self.operation = operation + self.sample = sample + self.file_name = file_name + self.sample_params = {to_snake_case(k): v for k, v in sample.get("parameters", {}).items()} + + def _imports(self) -> FileImportSerializer: + imports = FileImport(self.code_model) + namespace_from_package_name = get_namespace_from_package_name(self.code_model.options["package_name"]) + namespace_config = get_namespace_config(self.code_model.namespace, self.code_model.options["multiapi"]) + namespace = namespace_from_package_name or namespace_config + # mainly for "azure-mgmt-rdbms" + if not self.code_model.options["multiapi"] and namespace_config.count(".") > namespace_from_package_name.count( + "." + ): + namespace = namespace_config + client = self.code_model.clients[0] + imports.add_submodule_import(namespace, client.name, ImportType.LOCAL) + credential_type = getattr(client.credential, "type", None) + if isinstance(credential_type, TokenCredentialType): + imports.add_submodule_import("azure.identity", "DefaultAzureCredential", ImportType.SDKCORE) + elif isinstance(credential_type, KeyCredentialType): + imports.add_import("os", ImportType.STDLIB) + imports.add_submodule_import( + "credentials", + "AzureKeyCredential", + ImportType.SDKCORE, + ) + for param in self.operation.parameters.positional: + if not param.client_default_value and not param.optional and param.client_name in self.sample_params: + imports.merge(param.type.imports_for_sample()) + return FileImportSerializer(imports, True) + + def _client_params(self) -> Dict[str, Any]: + # client params + special_param = {} + credential_type = getattr(self.code_model.clients[0].credential, "type", None) + if isinstance(credential_type, TokenCredentialType): + special_param.update({"credential": "DefaultAzureCredential()"}) + elif isinstance(credential_type, KeyCredentialType): + special_param.update({"credential": 'AzureKeyCredential(key=os.getenv("AZURE_KEY"))'}) + + params_positional = [ + p for p in self.code_model.clients[0].parameters.positional if not (p.optional or p.client_default_value) + ] + client_params = { + p.client_name: special_param.get( + p.client_name, + f'"{self.sample_params.get(p.client_name) or p.client_name.upper()}"', + ) + for p in params_positional + } + + return client_params + + @staticmethod + def handle_param(param: Union[Parameter, BodyParameter], param_value: Any) -> str: + if isinstance(param_value, str): + if any(i in param_value for i in '\r\n"'): + return f'"""{param_value}"""' + + return param.type.serialize_sample_value(param_value) + + # prepare operation parameters + def _operation_params(self) -> Dict[str, Any]: + params_positional = [p for p in self.operation.parameters.positional if not p.client_default_value] + failure_info = "fail to find required param named {}" + operation_params = {} + for param in params_positional: + name = param.client_name + param_value = self.sample_params.get(name) + if not param.optional: + if not param_value: + raise Exception(failure_info.format(name)) # pylint: disable=broad-exception-raised + operation_params[param.client_name] = self.handle_param(param, param_value) + return operation_params + + def _operation_group_name(self) -> str: + if self.operation_group.is_mixin: + return "" + return f".{self.operation_group.property_name}" + + def _operation_result(self) -> Tuple[str, str]: + is_response_none = "None" in self.operation.response_type_annotation(async_mode=False) + lro = ".result()" + if is_response_none: + paging, normal_print, return_var = "", "", "" + else: + paging = "\n for item in response:\n print(item)" + normal_print = "\n print(response)" + return_var = "response = " + + if self.operation.operation_type == "paging": + return paging, return_var + if self.operation.operation_type == "lro": + return lro + normal_print, return_var + if self.operation.operation_type == "lropaging": + return lro + paging, return_var + return normal_print, return_var + + def _operation_name(self) -> str: + return f".{self.operation.name}" + + def _origin_file(self) -> str: + name = self.sample.get("x-ms-original-file", "") + if "specification" in name: + return "specification" + name.split("specification")[-1] + return "" + + def serialize(self) -> str: + operation_result, return_var = self._operation_result() + return self.env.get_template("sample.py.jinja2").render( + code_model=self.code_model, + file_name=self.file_name, + operation_result=operation_result, + operation_params=self._operation_params(), + operation_group_name=self._operation_group_name(), + operation_name=self._operation_name(), + imports=self._imports(), + client_params=self._client_params(), + origin_file=self._origin_file(), + return_var=return_var, + ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/test_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/test_serializer.py new file mode 100644 index 00000000000..5314ff2c768 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/test_serializer.py @@ -0,0 +1,263 @@ +# pylint: disable=too-many-lines +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Dict, Any, List, Optional +from jinja2 import Environment + +from .import_serializer import FileImportSerializer +from .base_serializer import BaseSerializer +from ..models import ( + CodeModel, + ImportType, + OperationGroup, + Client, + OperationType, + ModelType, + BaseType, + CombinedType, +) +from .utils import get_namespace_from_package_name, json_dumps_template + + +class TestName: + def __init__(self, client_name: str, *, is_async: bool = False) -> None: + self.client_name = client_name + self.is_async = is_async + + @property + def async_suffix_capt(self) -> str: + return "Async" if self.is_async else "" + + @property + def create_client_name(self) -> str: + return "create_async_client" if self.is_async else "create_client" + + @property + def prefix(self) -> str: + return self.client_name.replace("Client", "") + + @property + def preparer_name(self) -> str: + return self.prefix + "Preparer" + + @property + def base_test_class_name(self) -> str: + return f"{self.client_name}TestBase{self.async_suffix_capt}" + + +class TestCase: + def __init__( + self, + operation_groups: List[OperationGroup], + params: Dict[str, Any], + operation: OperationType, + *, + is_async: bool = False, + ) -> None: + self.operation_groups = operation_groups + self.params = params + self.operation = operation + self.is_async = is_async + + @property + def operation_group_prefix(self) -> str: + if self.operation_groups[-1].is_mixin: + return "" + return "." + ".".join([og.property_name for og in self.operation_groups]) + + @property + def response(self) -> str: + if self.is_async: + if self.operation.operation_type == "lropaging": + return "response = await (await " + return "response = await " + return "response = " + + @property + def lro_comment(self) -> str: + return " # poll until service return final result" + + @property + def operation_suffix(self) -> str: + if self.operation.operation_type == "lropaging": + extra = ")" if self.is_async else "" + return f"{extra}.result(){self.lro_comment}" + return "" + + @property + def extra_operation(self) -> str: + if self.is_async: + if self.operation.operation_type == "lro": + return f"result = await response.result(){self.lro_comment}" + if self.operation.operation_type == ("lropaging", "paging"): + return "result = [r async for r in response]" + else: + if self.operation.operation_type == "lro": + return f"result = response.result(){self.lro_comment}" + if self.operation.operation_type in ("lropaging", "paging"): + return "result = [r for r in response]" + return "" + + +class Test(TestName): + def __init__( + self, + client_name: str, + operation_group: OperationGroup, + testcases: List[TestCase], + test_class_name: str, + *, + is_async: bool = False, + ) -> None: + super().__init__(client_name, is_async=is_async) + self.operation_group = operation_group + self.testcases = testcases + self.test_class_name = test_class_name + + +class TestGeneralSerializer(BaseSerializer): + def __init__(self, code_model: CodeModel, env: Environment, *, is_async: bool = False) -> None: + super().__init__(code_model, env) + self.is_async = is_async + + @property + def aio_str(self) -> str: + return ".aio" if self.is_async else "" + + @property + def test_names(self) -> List[TestName]: + return [TestName(c.name, is_async=self.is_async) for c in self.code_model.clients] + + @property + def import_clients(self) -> FileImportSerializer: + imports = self.init_file_import() + namespace = get_namespace_from_package_name(self.code_model.options["package_name"]) + + imports.add_submodule_import("devtools_testutils", "AzureRecordedTestCase", ImportType.STDLIB) + if not self.is_async: + imports.add_import("functools", ImportType.STDLIB) + imports.add_submodule_import("devtools_testutils", "PowerShellPreparer", ImportType.STDLIB) + for client in self.code_model.clients: + imports.add_submodule_import(namespace + self.aio_str, client.name, ImportType.STDLIB) + return FileImportSerializer(imports, self.is_async) + + def serialize_conftest(self) -> str: + return self.env.get_template("conftest.py.jinja2").render( + test_names=self.test_names, + code_model=self.code_model, + ) + + def serialize_testpreparer(self) -> str: + return self.env.get_template("testpreparer.py.jinja2").render( + test_names=self.test_names, + imports=self.import_clients, + code_model=self.code_model, + ) + + +class TestSerializer(TestGeneralSerializer): + def __init__( + self, + code_model: CodeModel, + env: Environment, + *, + client: Client, + operation_group: OperationGroup, + is_async: bool = False, + ) -> None: + super().__init__(code_model, env, is_async=is_async) + self.client = client + self.operation_group = operation_group + + @property + def import_test(self) -> FileImportSerializer: + imports = self.init_file_import() + test_name = TestName(self.client.name, is_async=self.is_async) + async_suffix = "_async" if self.is_async else "" + imports.add_submodule_import( + "testpreparer" + async_suffix, + test_name.base_test_class_name, + ImportType.LOCAL, + ) + imports.add_submodule_import("testpreparer", test_name.preparer_name, ImportType.LOCAL) + imports.add_submodule_import( + "devtools_testutils" + self.aio_str, + "recorded_by_proxy" + async_suffix, + ImportType.LOCAL, + ) + return FileImportSerializer(imports, self.is_async) + + @property + def breadth_search_operation_group(self) -> List[List[OperationGroup]]: + result = [] + queue = [[self.operation_group]] + while queue: + current = queue.pop(0) + if current[-1].operations: + result.append(current) + if current[-1].operation_groups: + queue.extend([current + [og] for og in current[-1].operation_groups]) + return result + + def get_sub_type(self, param_type: ModelType) -> ModelType: + if param_type.discriminated_subtypes: + for item in param_type.discriminated_subtypes.values(): + return self.get_sub_type(item) + return param_type + + def get_model_type(self, param_type: BaseType) -> Optional[ModelType]: + if isinstance(param_type, ModelType): + return param_type + if isinstance(param_type, CombinedType): + return param_type.target_model_subtype((ModelType,)) + return None + + def get_operation_params(self, operation: OperationType) -> Dict[str, Any]: + operation_params = {} + required_params = [p for p in operation.parameters.method if not p.optional] + for param in required_params: + model_type = self.get_model_type(param.type) + param_type = self.get_sub_type(model_type) if model_type else param.type + operation_params[param.client_name] = json_dumps_template(param_type.get_json_template_representation()) + return operation_params + + def get_test(self) -> Test: + testcases = [] + for operation_groups in self.breadth_search_operation_group: + for operation in operation_groups[-1].operations: + if operation.internal or operation.is_lro_initial_operation: + continue + operation_params = self.get_operation_params(operation) + testcase = TestCase( + operation_groups=operation_groups, + params=operation_params, + operation=operation, + is_async=self.is_async, + ) + testcases.append(testcase) + if not testcases: + raise Exception("no public operation to test") # pylint: disable=broad-exception-raised + + return Test( + client_name=self.client.name, + operation_group=self.operation_group, + testcases=testcases, + test_class_name=self.test_class_name, + is_async=self.is_async, + ) + + @property + def test_class_name(self) -> str: + test_name = TestName(self.client.name, is_async=self.is_async) + class_name = "" if self.operation_group.is_mixin else self.operation_group.class_name + return f"Test{test_name.prefix}{class_name}{test_name.async_suffix_capt}" + + def serialize_test(self) -> str: + return self.env.get_template("test.py.jinja2").render( + imports=self.import_test, + code_model=self.code_model, + test=self.get_test(), + ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/types_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/types_serializer.py new file mode 100644 index 00000000000..3e143f5209e --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/types_serializer.py @@ -0,0 +1,31 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from ..models.imports import FileImport, ImportType +from .import_serializer import FileImportSerializer +from .base_serializer import BaseSerializer + + +class TypesSerializer(BaseSerializer): + def imports(self) -> FileImport: + file_import = FileImport(self.code_model) + if self.code_model.named_unions: + file_import.add_submodule_import( + "typing", + "Union", + ImportType.STDLIB, + ) + for nu in self.code_model.named_unions: + file_import.merge(nu.imports(relative_path=".", model_typing=True, is_types_file=True)) + return file_import + + def serialize(self) -> str: + # Generate the models + template = self.env.get_template("types.py.jinja2") + return template.render( + code_model=self.code_model, + imports=FileImportSerializer(self.imports()), + serializer=self, + ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/utils.py b/packages/autorest.python/generator/pygen/codegen/serializers/utils.py new file mode 100644 index 00000000000..7cd6f7c9267 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/serializers/utils.py @@ -0,0 +1,68 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import json +from typing import Optional, List, Any +from pathlib import Path + +from ..models import Client, OperationGroup + + +def method_signature_and_response_type_annotation_template( + *, + method_signature: str, + response_type_annotation: str, +) -> str: + return f"{method_signature} -> {response_type_annotation}:" + + +def extract_sample_name(file_path: str) -> str: + file = file_path.split("specification")[-1] + return Path(file).parts[-1].replace(".json", "") + + +def strip_end(namespace: str) -> str: + return ".".join(namespace.split(".")[:-1]) + + +def get_namespace_config(namespace: str, multiapi: bool) -> str: + return strip_end(namespace) if multiapi else namespace + + +def get_namespace_from_package_name(package_name: Optional[str]) -> str: + return (package_name or "").replace("-", ".") + + +def get_all_operation_groups_recursively(clients: List[Client]) -> List[OperationGroup]: + operation_groups = [] + queue = [] + for client in clients: + queue.extend(client.operation_groups) + while queue: + operation_groups.append(queue.pop(0)) + if operation_groups[-1].operation_groups: + queue.extend(operation_groups[-1].operation_groups) + return operation_groups + + +def _improve_json_string(template_representation: str) -> Any: + origin = template_representation.split("\n") + final = [] + for line in origin: + idx0 = line.find("#") + idx1 = line.rfind('"') + modified_line = "" + if idx0 > -1 and idx1 > -1: + modified_line = line[:idx0] + line[idx1:] + " " + line[idx0:idx1] + "\n" + else: + modified_line = line + "\n" + modified_line = modified_line.replace('"', "").replace("\\", '"') + final.append(modified_line) + return "".join(final) + + +def json_dumps_template(template_representation: Any) -> Any: + # only for template use, since it wraps everything in strings + return _improve_json_string(json.dumps(template_representation, indent=4)) diff --git a/packages/autorest.python/generator/pygen/codegen/templates/client.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/client.py.jinja2 new file mode 100644 index 00000000000..57e27f1f90f --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/client.py.jinja2 @@ -0,0 +1,37 @@ +{{ serializer.class_definition }} + """{{ op_tools.wrap_string(client.description, "\n") | indent }} + + {{ op_tools.serialize_with_wrap(serializer.property_descriptions(async_mode), "\n ") | indent }} + {{ serializer.init_signature_and_response_type_annotation(async_mode) | indent }} + {% if serializer.should_init_super %} + super().__init__() + {% endif %} + {% if client.has_parameterized_host %} + {{ serializer.host_variable_name }} = {{ keywords.escape_str(client.url) }}{{ client.url_pylint_disable }} + {% endif %} + {{ serializer.initialize_config() }} + {{ op_tools.serialize(serializer.initialize_pipeline_client(async_mode)) | indent(8) }} + + {{ op_tools.serialize(serializer.serializers_and_operation_groups_properties()) | indent(8) }} + + {% set http_response = keywords.async_class + "HttpResponse" %} + {{ serializer.send_request_signature_and_response_type_annotation(async_mode) | indent }} + {{ op_tools.serialize(serializer.send_request_description(async_mode)) | indent(8) }} + request_copy = deepcopy(request) + {% if client.parameters.path %} + {{ op_tools.serialize(serializer.serialize_path()) | indent(8) }} + request_copy.url = self._client.format_url(request_copy.url, **path_format_arguments) + {% else %} + request_copy.url = self._client.format_url(request_copy.url) + {% endif %} + return self._client.send_request(request_copy, stream=stream, **kwargs) # type: ignore + + {{ keywords.def }} close(self) -> None: + {{ keywords.await }}self._client.close() + + {{ keywords.def }} __{{ keywords.async_prefix }}enter__(self){{ " -> \"" + client.name + "\"" }}: + {{ keywords.await }}self._client.__{{ keywords.async_prefix }}enter__() + return self + + {{ keywords.def }} __{{ keywords.async_prefix }}exit__(self, *exc_details: Any) -> None: + {{ keywords.await }}self._client.__{{ keywords.async_prefix }}exit__(*exc_details) diff --git a/packages/autorest.python/generator/pygen/codegen/templates/client_container.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/client_container.py.jinja2 new file mode 100644 index 00000000000..33de339affe --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/client_container.py.jinja2 @@ -0,0 +1,12 @@ +{% import 'keywords.jinja2' as keywords with context %} +{% import 'operation_tools.jinja2' as op_tools %} +{# actual template starts here #} +# coding=utf-8 +{{ code_model.options['license_header'] }} + +{{ imports }} + +{% for client in clients %} + {% set serializer = get_serializer(client) %} +{% include "client.py.jinja2" %} +{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/config.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/config.py.jinja2 new file mode 100644 index 00000000000..b267c8e97d8 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/config.py.jinja2 @@ -0,0 +1,73 @@ +class {{ client.name }}Configuration: {{ client.config.pylint_disable }} + """Configuration for {{ client.name }}. + + Note that all parameters used to create this instance are saved as instance + attributes. +{% if client.config.parameters.method | first %} + +{% endif %} + {{ op_tools.serialize_with_wrap(serializer.property_descriptions(async_mode), "\n ") | indent }} + {{ serializer.init_signature_and_response_type_annotation(async_mode) | indent }} + {% if client.config.parameters.kwargs_to_pop %} + {{ op_tools.serialize(serializer.pop_kwargs_from_signature()) | indent(8) }} + {% endif %} +{% if serializer.check_required_parameters() %} + {{ op_tools.serialize(serializer.check_required_parameters()) | indent(8) -}} +{% endif %} + +{% for parameter in client.config.parameters.method %} + self.{{ parameter.client_name }} = {{ parameter.client_name }} +{% endfor %} +{% if serializer.set_constants() %} + {{ op_tools.serialize(serializer.set_constants()) | indent(8) -}} +{% endif %} +{% if client.credential %} + {% set cred_scopes = client.credential.type if client.credential.type.policy is defined and client.credential.type.policy.credential_scopes is defined %} + {% if not cred_scopes %} + {% set cred_scopes = client.credential.type.types | selectattr("policy.credential_scopes") | first if client.credential.type.types is defined %} + {% endif %} + {% if cred_scopes %} + self.credential_scopes = kwargs.pop('credential_scopes', {{ cred_scopes.policy.credential_scopes }}) + {% endif %} +{% endif %} + kwargs.setdefault('sdk_moniker', '{{ client.config.sdk_moniker }}/{}'.format(VERSION)) + self.polling_interval = kwargs.get("polling_interval", 30) + self._configure(**kwargs) + +{% if client.credential and client.credential.type.types is defined %} + def _infer_policy(self, **kwargs): + {% for cred_type in client.credential.type.types %} + if {{ cred_type.instance_check_template.format("self.credential") }}: + return {{ cred_type.policy.call(async_mode) }} + {% endfor %} + raise TypeError(f"Unsupported credential: {self.credential}") +{% endif %} + + def _configure( + self, + **kwargs: Any + ) -> None: + self.user_agent_policy = kwargs.get('user_agent_policy') or policies.UserAgentPolicy(**kwargs) + self.headers_policy = kwargs.get('headers_policy') or policies.HeadersPolicy(**kwargs) + self.proxy_policy = kwargs.get('proxy_policy') or policies.ProxyPolicy(**kwargs) + self.logging_policy = kwargs.get('logging_policy') or policies.NetworkTraceLoggingPolicy(**kwargs) + {% if code_model.is_azure_flavor %} + self.http_logging_policy = kwargs.get('http_logging_policy') or {{ "ARM" if client.code_model.options['azure_arm'] else "policies." }}HttpLoggingPolicy(**kwargs) + self.custom_hook_policy = kwargs.get('custom_hook_policy') or policies.CustomHookPolicy(**kwargs) + self.redirect_policy = kwargs.get('redirect_policy') or policies.{{ keywords.async_class }}RedirectPolicy(**kwargs) + {% endif %} + self.retry_policy = kwargs.get('retry_policy') or policies.{{ keywords.async_class }}RetryPolicy(**kwargs) + self.authentication_policy = kwargs.get('authentication_policy') + {% if client.credential and client.credential.type.policy is defined %} + {# only adding this if credential_scopes is not passed during code generation #} + {% if client.credential.type.policy.credential_scopes is defined and client.credential.type.policy.credential_scopes | length == 0 %} + if not self.credential_scopes and not self.authentication_policy: + raise ValueError("You must provide either credential_scopes or authentication_policy as kwargs") + {% endif %} + if self.credential and not self.authentication_policy: + self.authentication_policy = {{ client.credential.type.policy.call(async_mode) }} + {% endif %} + {% if client.credential and client.credential.type.types is defined %} + if self.credential and not self.authentication_policy: + self.authentication_policy = self._infer_policy(**kwargs) + {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/config_container.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/config_container.py.jinja2 new file mode 100644 index 00000000000..9a3c263e2ff --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/config_container.py.jinja2 @@ -0,0 +1,16 @@ +{% import 'keywords.jinja2' as keywords with context %} +{% import 'operation_tools.jinja2' as op_tools %} +{# actual template starts here #} +# coding=utf-8 +{{ code_model.options['license_header'] }} + +{{ imports }} + +{% if not code_model.options['package_version'] %} +VERSION = "unknown" +{% endif %} + +{% for client in clients %} + {% set serializer = get_serializer(client) %} +{% include "config.py.jinja2" %} +{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/conftest.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/conftest.py.jinja2 new file mode 100644 index 00000000000..3d56b93351d --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/conftest.py.jinja2 @@ -0,0 +1,28 @@ +# coding=utf-8 +{{ code_model.options['license_header'] }} +import os +import pytest +from dotenv import load_dotenv +from devtools_testutils import test_proxy, add_general_regex_sanitizer, add_body_key_sanitizer, add_header_regex_sanitizer + +load_dotenv() + +# aovid record sensitive identity information in recordings +@pytest.fixture(scope="session", autouse=True) +def add_sanitizers(test_proxy): + {% for test_name in test_names %} + {% set prefix_upper = test_name.prefix|upper %} + {% set prefix_lower = test_name.prefix|lower %} + {{ prefix_lower }}_subscription_id = os.environ.get("{{ prefix_upper }}_SUBSCRIPTION_ID", "00000000-0000-0000-0000-000000000000") + {{ prefix_lower }}_tenant_id = os.environ.get("{{ prefix_upper }}_TENANT_ID", "00000000-0000-0000-0000-000000000000") + {{ prefix_lower }}_client_id = os.environ.get("{{ prefix_upper }}_CLIENT_ID", "00000000-0000-0000-0000-000000000000") + {{ prefix_lower }}_client_secret = os.environ.get("{{ prefix_upper }}_CLIENT_SECRET", "00000000-0000-0000-0000-000000000000") + add_general_regex_sanitizer(regex={{ prefix_lower }}_subscription_id, value="00000000-0000-0000-0000-000000000000") + add_general_regex_sanitizer(regex={{ prefix_lower }}_tenant_id, value="00000000-0000-0000-0000-000000000000") + add_general_regex_sanitizer(regex={{ prefix_lower }}_client_id, value="00000000-0000-0000-0000-000000000000") + add_general_regex_sanitizer(regex={{ prefix_lower }}_client_secret, value="00000000-0000-0000-0000-000000000000") + + {% endfor %} + add_header_regex_sanitizer(key="Set-Cookie", value="[set-cookie;]") + add_header_regex_sanitizer(key="Cookie", value="cookie;") + add_body_key_sanitizer(json_path="$..access_token", value="access_token") diff --git a/packages/autorest.python/generator/pygen/codegen/templates/enum.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/enum.py.jinja2 new file mode 100644 index 00000000000..047d64c30d6 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/enum.py.jinja2 @@ -0,0 +1,13 @@ + +class {{ enum.name }}({{ enum.value_type.type_annotation(is_operation_file=False) }}, Enum, metaclass=CaseInsensitiveEnumMeta): + {% if enum.yaml_data.get("description") %} + """{{ op_tools.wrap_string(enum.yaml_data["description"], "\n ") }} + """ + {% endif %} + + {% for value in enum.values %} + {{ value.name }} = {{ enum.value_type.get_declaration(value.value) }} + {% if value.description(is_operation_file=False) %} + """{{ op_tools.wrap_string(value.description(is_operation_file=False), "\n ") }}""" + {% endif %} + {% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/enum_container.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/enum_container.py.jinja2 new file mode 100644 index 00000000000..099fbb072a6 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/enum_container.py.jinja2 @@ -0,0 +1,10 @@ +{% import 'operation_tools.jinja2' as op_tools %} +# coding=utf-8 +{{ code_model.options['license_header'] }} + +from enum import Enum +from {{ code_model.core_library }}{{ "" if code_model.is_azure_flavor else ".utils" }} import CaseInsensitiveEnumMeta + +{% for enum in code_model.enums | sort %} +{% include "enum.py.jinja2" %} +{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/init.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/init.py.jinja2 new file mode 100644 index 00000000000..9f70f40bd6b --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/init.py.jinja2 @@ -0,0 +1,24 @@ +{% import 'keywords.jinja2' as keywords %} +# coding=utf-8 +{{ code_model.options['license_header'] }} + +{% if clients %} + {% for client in clients %} +from .{{ client.filename }} import {{ client.name }} + {% endfor %} +{% endif %} +{% if not async_mode and code_model.options['package_version']%} +from ._version import VERSION + +__version__ = VERSION +{% endif %} + +{{ keywords.patch_imports(try_except=True) }} +__all__ = [ + {% for client in clients %} + {{ keywords.escape_str(client.name) }}, + {% endfor %} +] +{{ keywords.extend_all }} + +_patch_sdk() diff --git a/packages/autorest.python/generator/pygen/codegen/templates/keywords.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/keywords.jinja2 new file mode 100644 index 00000000000..6ee92a41416 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/keywords.jinja2 @@ -0,0 +1,19 @@ +{% set def = "async def" if async_mode else "def" %} +{% set async_prefix = "a" if async_mode else "" %} +{% set await = "await " if async_mode else "" %} +{% set async_class = "Async" if async_mode else "" %} +{% macro escape_str(s) %}'{{ s|replace("'", "\\'") }}'{% endmacro %} +{% set kwargs_declaration = "**kwargs: Any" %} +{% set extend_all = "__all__.extend([p for p in _patch_all if p not in __all__])" %} +{% macro patch_imports(try_except=False) %} +{% set indentation = " " if try_except else "" %} +{% if try_except %} +try: +{% endif %} +{{ indentation }}from ._patch import __all__ as _patch_all +{{ indentation }}from ._patch import * # pylint: disable=unused-wildcard-import +{% if try_except %} +except ImportError: + _patch_all = [] +{% endif %} +from ._patch import patch_sdk as _patch_sdk{% endmacro %} \ No newline at end of file diff --git a/packages/autorest.python/generator/pygen/codegen/templates/lro_operation.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/lro_operation.py.jinja2 new file mode 100644 index 00000000000..bca9dcf2e4a --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/lro_operation.py.jinja2 @@ -0,0 +1,16 @@ +{% import 'operation_tools.jinja2' as op_tools with context %} +{# actual template starts here #} +{% if operation.overloads and operation.include_documentation %} +{{ op_tools.generate_overloads(operation_serializer, operation) }} +{% endif %} +{{ operation_serializer.method_signature_and_response_type_annotation(operation) }} + {{ op_tools.description(operation, operation_serializer) | indent -}} + {% if not operation.abstract %} + {% if operation_serializer.pop_kwargs_from_signature(operation) %} + {{ op_tools.serialize(operation_serializer.pop_kwargs_from_signature(operation)) | indent }} + {%- endif %} + {{ op_tools.serialize(operation_serializer.initial_call(operation)) | indent }} + {{ op_tools.serialize(operation_serializer.get_long_running_output(operation)) | indent }} + + {{ op_tools.serialize(operation_serializer.return_lro_poller(operation)) | indent }} + {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 new file mode 100644 index 00000000000..09b50a00d42 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 @@ -0,0 +1,18 @@ +{% import 'operation_tools.jinja2' as op_tools with context %} +{% import 'keywords.jinja2' as keywords with context %} +{# actual template starts here #} +{% if operation.overloads and operation.include_documentation %} +{{ op_tools.generate_overloads(operation_serializer, operation) }} +{% endif %} +{{ operation_serializer.method_signature_and_response_type_annotation(operation) }} + {{ op_tools.description(operation, operation_serializer) | indent }} + {% if not operation.abstract %} + {% if operation_serializer.pop_kwargs_from_signature(operation) %} + {{ op_tools.serialize(operation_serializer.pop_kwargs_from_signature(operation)) | indent }} + {% endif %} + {{ op_tools.serialize(operation_serializer.set_up_params_for_pager(operation)) | indent }} + + {{ op_tools.serialize(operation_serializer.initial_call(operation)) | indent }} + {{ op_tools.serialize(operation_serializer.get_long_running_output(operation)) | indent }} + {{ op_tools.serialize(operation_serializer.return_lro_poller(operation)) | indent }} + {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/macros.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/macros.jinja2 new file mode 100644 index 00000000000..64981bedb86 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/macros.jinja2 @@ -0,0 +1,12 @@ +{% macro wrap_model_string(doc_string, wrap_string, suffix_string="") %} +{% set orignal_result = doc_string | wordwrap(width=95, break_long_words=False, break_on_hyphens=False, wrapstring=wrap_string) %} +{% set list_result = orignal_result.split('\n') %} +{% for line in list_result %} + {% set suffix = suffix_string if list_result | length == loop.index %} + {% if line | length > 120 %} +{{ line + " # pylint: disable=line-too-long" }}{{ suffix }} + {% else %} +{{ line }}{{ suffix }} + {% endif %} +{% endfor %} +{% endmacro %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/metadata.json.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/metadata.json.jinja2 new file mode 100644 index 00000000000..946baa79170 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/metadata.json.jinja2 @@ -0,0 +1,167 @@ +{% import 'operation_tools.jinja2' as op_tools %} +{% import 'keywords.jinja2' as keywords %} +{ + "chosen_version": {{ chosen_version | tojson }}, + "total_api_version_list": {{ total_api_version_list | tojson }}, + "client": { + "name": {{ client.name | tojson }}, + "filename": {{ ("_" + client.legacy_filename) | tojson }}, + "description": {{ client.description | tojson }}, + "host_value": {{ (client.parameters.host.client_default_value_declaration if not client.has_parameterized_host else None) | tojson }}, + "parameterized_host_template": {{ (keywords.escape_str(client.url) if client.has_parameterized_host else None) | tojson }}, + "azure_arm": {{ client.code_model.options["azure_arm"] | tojson }}, + "has_public_lro_operations": {{ client.has_public_lro_operations | tojson }}, + "client_side_validation": {{ client.code_model.options["client_side_validation"] | tojson }}, + "sync_imports": {{ sync_client_imports | tojson }}, + "async_imports": {{ async_client_imports | tojson }} + }, + "global_parameters": { + "sync": { + {% for gp in global_parameters.method | rejectattr("client_name", "equalto", "api_version") | rejectattr("is_host") %} + {{ gp.client_name | tojson }}: { + "signature": {{ gp.method_signature(async_mode=False) | tojson }}, + "description": {{ gp.description | tojson }}, + "docstring_type": {{ gp.docstring_type(async_mode=False) | tojson }}, + "required": {{ (not gp.optional) | tojson }}, + "method_location": {{ gp.method_location | tojson }} + }{{ "," if not loop.last else "" }} + {% endfor %} + }, + "async": { + {% for gp in global_parameters.method | rejectattr("client_name", "equalto", "api_version") | rejectattr("is_host") %} + {{ gp.client_name | tojson }}: { + "signature": {{ (gp.method_signature(async_mode=True)) | tojson }}, + "description": {{ gp.description | tojson }}, + "docstring_type": {{ gp.docstring_type(async_mode=True) | tojson }}, + "required": {{ (not gp.optional) | tojson }} + }{{ "," if not loop.last else "" }} + {% endfor %} + }, + "constant": { + {% for gp in client.parameters.constant | rejectattr("client_name", "equalto", "api_version") %} + {{ gp.client_name | tojson }}: {{ gp.constant_declaration | tojson }}{{ "," if not loop.last else "" }} + {% endfor %} + }, + "call": {{ client.parameters.method | rejectattr("client_name", "equalto", "api_version") | rejectattr("is_host") | map(attribute="client_name") | join(', ') | tojson }}, + "service_client_specific": { + "sync": { + "api_version": { + "signature": "api_version: Optional[str]=None,", + "description": "API version to use if no profile is provided, or if missing in profile.", + "docstring_type": "str", + "required": false, + "method_location": "positional" + }, + {% if not client.has_parameterized_host %} + "base_url": { + "signature": {{ client.parameters.host.method_signature(async_mode=False) | tojson }}, + "description": "Service URL", + "docstring_type": "str", + "required": false, + "method_location": "positional" + }, + {% endif %} + "profile": { + "signature": "profile: KnownProfiles=KnownProfiles.default,", + "description": "A profile definition, from KnownProfiles to dict.", + "docstring_type": "azure.profiles.KnownProfiles", + "required": false, + "method_location": "positional" + } + }, + "async": { + "api_version": { + "signature": "api_version: Optional[str] = None,", + "description": "API version to use if no profile is provided, or if missing in profile.", + "docstring_type": "str", + "required": false, + "method_location": "positional" + }, + {% if not client.has_parameterized_host %} + "base_url": { + "signature": {{ client.parameters.host.method_signature(async_mode=True) | tojson }}, + "description": "Service URL", + "docstring_type": "str", + "required": false, + "method_location": "positional" + }, + {% endif %} + "profile": { + "signature": "profile: KnownProfiles = KnownProfiles.default,", + "description": "A profile definition, from KnownProfiles to dict.", + "docstring_type": "azure.profiles.KnownProfiles", + "required": false, + "method_location": "positional" + } + } + } + }, + "config": { + "credential": {{ has_credential | tojson }}, + "credential_scopes": {{ (client.credential.type.policy.credential_scopes if has_credential and client.credential.type.policy.credential_scopes is defined else None)| tojson}}, + "credential_call_sync": {{ (client.credential.type.policy.call(async_mode=False) if has_credential else None) | tojson }}, + "credential_call_async": {{ (client.credential.type.policy.call(async_mode=True) if has_credential else None) | tojson }}, + "sync_imports": {{ sync_config_imports | tojson }}, + "async_imports": {{ async_config_imports | tojson }} + }, + "operation_groups": { + {% for operation_group in client.operation_groups | rejectattr('is_mixin') %} + {{ operation_group.property_name | tojson }}: {{ operation_group.class_name | tojson }}{{ "," if not loop.last else "" }} + {% endfor %} + }{{ "," if mixin_operations }} + {% if mixin_operations %} + "operation_mixins": { + "sync_imports": {{ str(sync_mixin_imports) | tojson }}, + "async_imports": {{ str(async_mixin_imports) | tojson }}, + "sync_mixin_typing_definitions": {{ str(sync_mixin_typing_definitions) | tojson }}, + "async_mixin_typing_definitions": {{ str(async_mixin_typing_definitions) | tojson }}, + "operations": { + {% for operation in mixin_operations %} + {{ operation.name | tojson }} : { + {% set request_builder = operation.request_builder %} + "sync": { + {% set operation_serializer = get_sync_operation_serializer(operation) %} + {% if is_lro(operation) and is_paging(operation) %} + {% from "lro_paging_operation.py.jinja2" import operation_docstring with context %} + {% set sync_return_type_wrapper = [operation.get_poller(async_mode=False), operation.get_pager(async_mode=False)] %} + {% elif is_lro(operation) %} + {% from "lro_operation.py.jinja2" import operation_docstring with context %} + {% set sync_return_type_wrapper = [operation.get_poller(async_mode=False)] %} + {% elif is_paging(operation) %} + {% from "paging_operation.py.jinja2" import operation_docstring with context %} + {% set sync_return_type_wrapper = [operation.get_pager(async_mode=False)] %} + {% else %} + {% from "operation.py.jinja2" import operation_docstring with context %} + {% set sync_return_type_wrapper = "" %} + {% endif %} + "signature": {{ (operation_serializer.method_signature_and_response_type_annotation(operation, want_decorators=False) + "\n") | tojson }}, + "doc": {{ op_tools.description(operation, operation_serializer).rstrip("\n") | tojson }}, + "call": {{ operation.parameters.call | join(', ') | tojson }} + }, + "async": { + {% set coroutine = False if (is_paging(operation) and not is_lro(operation)) else True %} + {% set operation_serializer = get_async_operation_serializer(operation) %} + "coroutine": {{ coroutine | tojson }}, + {% if is_lro(operation) and is_paging(operation) %} + {% from "lro_paging_operation.py.jinja2" import operation_docstring with context %} + {% set async_return_type_wrapper = [operation.get_poller(async_mode=True), operation.get_pager(async_mode=True)] %} + {% elif is_lro(operation) %} + {% from "lro_operation.py.jinja2" import operation_docstring with context %} + {% set async_return_type_wrapper = [operation.get_poller(async_mode=True)] %} + {% elif is_paging(operation) %} + {% from "paging_operation.py.jinja2" import operation_docstring with context %} + {% set async_return_type_wrapper = [operation.get_pager(async_mode=True)] %} + {% else %} + {% from "operation.py.jinja2" import operation_docstring with context %} + {% set async_return_type_wrapper = "" %} + {% endif %} + "signature": {{ (operation_serializer.method_signature_and_response_type_annotation(operation, want_decorators=False) + "\n") | tojson }}, + "doc": {{ op_tools.description(operation, operation_serializer).rstrip("\n") | tojson }}, + "call": {{ operation.parameters.call | join(', ') | tojson }} + } + }{{ "," if not loop.last else "" }} + {% endfor %} + } + } + {% endif %} +} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/model_base.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/model_base.py.jinja2 new file mode 100644 index 00000000000..386a0a2e2eb --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/model_base.py.jinja2 @@ -0,0 +1,898 @@ +# coding=utf-8 +# -------------------------------------------------------------------------- +# Copyright (c) {{ code_model.options["company_name"] }} Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +# pylint: disable=protected-access, arguments-differ, signature-differs, broad-except + +import copy +import calendar +import decimal +import functools +import sys +import logging +import base64 +import re +import typing +import enum +import email.utils +from datetime import datetime, date, time, timedelta, timezone +from json import JSONEncoder +from typing_extensions import Self +import isodate +from {{ code_model.core_library }}.exceptions import DeserializationError +from {{ code_model.core_library }}{{ "" if code_model.is_azure_flavor else ".utils" }} import CaseInsensitiveEnumMeta +from {{ code_model.core_library }}.{{ "" if code_model.is_azure_flavor else "runtime." }}pipeline import PipelineResponse +from {{ code_model.core_library }}.serialization import _Null + +if sys.version_info >= (3, 9): + from collections.abc import MutableMapping +else: + from typing import MutableMapping + +_LOGGER = logging.getLogger(__name__) + +__all__ = ["SdkJSONEncoder", "Model", "rest_field", "rest_discriminator"] + +TZ_UTC = timezone.utc +_T = typing.TypeVar("_T") + + +def _timedelta_as_isostr(td: timedelta) -> str: + """Converts a datetime.timedelta object into an ISO 8601 formatted string, e.g. 'P4DT12H30M05S' + + Function adapted from the Tin Can Python project: https://github.com/RusticiSoftware/TinCanPython + + :param timedelta td: The timedelta to convert + :rtype: str + :return: ISO8601 version of this timedelta + """ + + # Split seconds to larger units + seconds = td.total_seconds() + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + days, hours = divmod(hours, 24) + + days, hours, minutes = list(map(int, (days, hours, minutes))) + seconds = round(seconds, 6) + + # Build date + date_str = "" + if days: + date_str = "%sD" % days + + if hours or minutes or seconds: + # Build time + time_str = "T" + + # Hours + bigger_exists = date_str or hours + if bigger_exists: + time_str += "{:02}H".format(hours) + + # Minutes + bigger_exists = bigger_exists or minutes + if bigger_exists: + time_str += "{:02}M".format(minutes) + + # Seconds + try: + if seconds.is_integer(): + seconds_string = "{:02}".format(int(seconds)) + else: + # 9 chars long w/ leading 0, 6 digits after decimal + seconds_string = "%09.6f" % seconds + # Remove trailing zeros + seconds_string = seconds_string.rstrip("0") + except AttributeError: # int.is_integer() raises + seconds_string = "{:02}".format(seconds) + + time_str += "{}S".format(seconds_string) + else: + time_str = "" + + return "P" + date_str + time_str + + +def _serialize_bytes(o, format: typing.Optional[str] = None) -> str: + encoded = base64.b64encode(o).decode() + if format == "base64url": + return encoded.strip("=").replace("+", "-").replace("/", "_") + return encoded + + +def _serialize_datetime(o, format: typing.Optional[str] = None): + if hasattr(o, "year") and hasattr(o, "hour"): + if format == "rfc7231": + return email.utils.format_datetime(o, usegmt=True) + if format == "unix-timestamp": + return int(calendar.timegm(o.utctimetuple())) + + # astimezone() fails for naive times in Python 2.7, so make make sure o is aware (tzinfo is set) + if not o.tzinfo: + iso_formatted = o.replace(tzinfo=TZ_UTC).isoformat() + else: + iso_formatted = o.astimezone(TZ_UTC).isoformat() + # Replace the trailing "+00:00" UTC offset with "Z" (RFC 3339: https://www.ietf.org/rfc/rfc3339.txt) + return iso_formatted.replace("+00:00", "Z") + # Next try datetime.date or datetime.time + return o.isoformat() + + +def _is_readonly(p): + try: + return p._visibility == ["read"] # pylint: disable=protected-access + except AttributeError: + return False + + +class SdkJSONEncoder(JSONEncoder): + """A JSON encoder that's capable of serializing datetime objects and bytes.""" + + def __init__(self, *args, exclude_readonly: bool = False, format: typing.Optional[str] = None, **kwargs): + super().__init__(*args, **kwargs) + self.exclude_readonly = exclude_readonly + self.format = format + + def default(self, o): # pylint: disable=too-many-return-statements + if _is_model(o): + if self.exclude_readonly: + readonly_props = [p._rest_name for p in o._attr_to_rest_field.values() if _is_readonly(p)] + return {k: v for k, v in o.items() if k not in readonly_props} + return dict(o.items()) + try: + return super(SdkJSONEncoder, self).default(o) + except TypeError: + if isinstance(o, _Null): + return None + if isinstance(o, decimal.Decimal): + return float(o) + if isinstance(o, (bytes, bytearray)): + return _serialize_bytes(o, self.format) + try: + # First try datetime.datetime + return _serialize_datetime(o, self.format) + except AttributeError: + pass + # Last, try datetime.timedelta + try: + return _timedelta_as_isostr(o) + except AttributeError: + # This will be raised when it hits value.total_seconds in the method above + pass + return super(SdkJSONEncoder, self).default(o) + + +_VALID_DATE = re.compile(r"\d{4}[-]\d{2}[-]\d{2}T\d{2}:\d{2}:\d{2}" + r"\.?\d*Z?[-+]?[\d{2}]?:?[\d{2}]?") +_VALID_RFC7231 = re.compile( + r"(Mon|Tue|Wed|Thu|Fri|Sat|Sun),\s\d{2}\s" + r"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT" +) + + +def _deserialize_datetime(attr: typing.Union[str, datetime]) -> datetime: + """Deserialize ISO-8601 formatted string into Datetime object. + + :param str attr: response string to be deserialized. + :rtype: ~datetime.datetime + :returns: The datetime object from that input + """ + if isinstance(attr, datetime): + # i'm already deserialized + return attr + attr = attr.upper() + match = _VALID_DATE.match(attr) + if not match: + raise ValueError("Invalid datetime string: " + attr) + + check_decimal = attr.split(".") + if len(check_decimal) > 1: + decimal_str = "" + for digit in check_decimal[1]: + if digit.isdigit(): + decimal_str += digit + else: + break + if len(decimal_str) > 6: + attr = attr.replace(decimal_str, decimal_str[0:6]) + + date_obj = isodate.parse_datetime(attr) + test_utc = date_obj.utctimetuple() + if test_utc.tm_year > 9999 or test_utc.tm_year < 1: + raise OverflowError("Hit max or min date") + return date_obj + + +def _deserialize_datetime_rfc7231(attr: typing.Union[str, datetime]) -> datetime: + """Deserialize RFC7231 formatted string into Datetime object. + + :param str attr: response string to be deserialized. + :rtype: ~datetime.datetime + :returns: The datetime object from that input + """ + if isinstance(attr, datetime): + # i'm already deserialized + return attr + match = _VALID_RFC7231.match(attr) + if not match: + raise ValueError("Invalid datetime string: " + attr) + + return email.utils.parsedate_to_datetime(attr) + + +def _deserialize_datetime_unix_timestamp(attr: typing.Union[float, datetime]) -> datetime: + """Deserialize unix timestamp into Datetime object. + + :param str attr: response string to be deserialized. + :rtype: ~datetime.datetime + :returns: The datetime object from that input + """ + if isinstance(attr, datetime): + # i'm already deserialized + return attr + return datetime.fromtimestamp(attr, TZ_UTC) + + +def _deserialize_date(attr: typing.Union[str, date]) -> date: + """Deserialize ISO-8601 formatted string into Date object. + :param str attr: response string to be deserialized. + :rtype: date + :returns: The date object from that input + """ + # This must NOT use defaultmonth/defaultday. Using None ensure this raises an exception. + if isinstance(attr, date): + return attr + return isodate.parse_date(attr, defaultmonth=None, defaultday=None) # type: ignore + + +def _deserialize_time(attr: typing.Union[str, time]) -> time: + """Deserialize ISO-8601 formatted string into time object. + + :param str attr: response string to be deserialized. + :rtype: datetime.time + :returns: The time object from that input + """ + if isinstance(attr, time): + return attr + return isodate.parse_time(attr) + + +def _deserialize_bytes(attr): + if isinstance(attr, (bytes, bytearray)): + return attr + return bytes(base64.b64decode(attr)) + + +def _deserialize_bytes_base64(attr): + if isinstance(attr, (bytes, bytearray)): + return attr + padding = "=" * (3 - (len(attr) + 3) % 4) # type: ignore + attr = attr + padding # type: ignore + encoded = attr.replace("-", "+").replace("_", "/") + return bytes(base64.b64decode(encoded)) + + +def _deserialize_duration(attr): + if isinstance(attr, timedelta): + return attr + return isodate.parse_duration(attr) + + +def _deserialize_decimal(attr): + if isinstance(attr, decimal.Decimal): + return attr + return decimal.Decimal(str(attr)) + + +_DESERIALIZE_MAPPING = { + datetime: _deserialize_datetime, + date: _deserialize_date, + time: _deserialize_time, + bytes: _deserialize_bytes, + bytearray: _deserialize_bytes, + timedelta: _deserialize_duration, + typing.Any: lambda x: x, + decimal.Decimal: _deserialize_decimal, +} + +_DESERIALIZE_MAPPING_WITHFORMAT = { + "rfc3339": _deserialize_datetime, + "rfc7231": _deserialize_datetime_rfc7231, + "unix-timestamp": _deserialize_datetime_unix_timestamp, + "base64": _deserialize_bytes, + "base64url": _deserialize_bytes_base64, +} + + +def get_deserializer(annotation: typing.Any, rf: typing.Optional["_RestField"] = None): + if rf and rf._format: + return _DESERIALIZE_MAPPING_WITHFORMAT.get(rf._format) + return _DESERIALIZE_MAPPING.get(annotation) + + +def _get_type_alias_type(module_name: str, alias_name: str): + types = { + k: v + for k, v in sys.modules[module_name].__dict__.items() + if isinstance(v, typing._GenericAlias) # type: ignore + } + if alias_name not in types: + return alias_name + return types[alias_name] + + +def _get_model(module_name: str, model_name: str): + models = { + k: v + for k, v in sys.modules[module_name].__dict__.items() + if isinstance(v, type) + } + module_end = module_name.rsplit(".", 1)[0] + models.update({ + k: v + for k, v in sys.modules[module_end].__dict__.items() + if isinstance(v, type) + }) + if isinstance(model_name, str): + model_name = model_name.split(".")[-1] + if model_name not in models: + return model_name + return models[model_name] + + +_UNSET = object() + + +class _MyMutableMapping(MutableMapping[str, typing.Any]): # pylint: disable=unsubscriptable-object + def __init__(self, data: typing.Dict[str, typing.Any]) -> None: + self._data = data + + def __contains__(self, key: typing.Any) -> bool: + return key in self._data + + def __getitem__(self, key: str) -> typing.Any: + return self._data.__getitem__(key) + + def __setitem__(self, key: str, value: typing.Any) -> None: + self._data.__setitem__(key, value) + + def __delitem__(self, key: str) -> None: + self._data.__delitem__(key) + + def __iter__(self) -> typing.Iterator[typing.Any]: + return self._data.__iter__() + + def __len__(self) -> int: + return self._data.__len__() + + def __ne__(self, other: typing.Any) -> bool: + return not self.__eq__(other) + + def keys(self) -> typing.KeysView[str]: + return self._data.keys() + + def values(self) -> typing.ValuesView[typing.Any]: + return self._data.values() + + def items(self) -> typing.ItemsView[str, typing.Any]: + return self._data.items() + + def get(self, key: str, default: typing.Any = None) -> typing.Any: + try: + return self[key] + except KeyError: + return default + + @typing.overload + def pop(self, key: str) -> typing.Any: + ... + + @typing.overload + def pop(self, key: str, default: _T) -> _T: + ... + + @typing.overload + def pop(self, key: str, default: typing.Any) -> typing.Any: + ... + + def pop(self, key: str, default: typing.Any = _UNSET) -> typing.Any: + if default is _UNSET: + return self._data.pop(key) + return self._data.pop(key, default) + + def popitem(self) -> typing.Tuple[str, typing.Any]: + return self._data.popitem() + + def clear(self) -> None: + self._data.clear() + + def update(self, *args: typing.Any, **kwargs: typing.Any) -> None: + self._data.update(*args, **kwargs) + + @typing.overload + def setdefault(self, key: str, default: None = None) -> None: + ... + + @typing.overload + def setdefault(self, key: str, default: typing.Any) -> typing.Any: + ... + + def setdefault(self, key: str, default: typing.Any = _UNSET) -> typing.Any: + if default is _UNSET: + return self._data.setdefault(key) + return self._data.setdefault(key, default) + + def __eq__(self, other: typing.Any) -> bool: + try: + other_model = self.__class__(other) + except Exception: + return False + return self._data == other_model._data + + def __repr__(self) -> str: + return str(self._data) + + +def _is_model(obj: typing.Any) -> bool: + return getattr(obj, "_is_model", False) + + +def _serialize(o, format: typing.Optional[str] = None): # pylint: disable=too-many-return-statements + if isinstance(o, list): + return [_serialize(x, format) for x in o] + if isinstance(o, dict): + return {k: _serialize(v, format) for k, v in o.items()} + if isinstance(o, set): + return {_serialize(x, format) for x in o} + if isinstance(o, tuple): + return tuple(_serialize(x, format) for x in o) + if isinstance(o, (bytes, bytearray)): + return _serialize_bytes(o, format) + if isinstance(o, decimal.Decimal): + return float(o) + if isinstance(o, enum.Enum): + return o.value + try: + # First try datetime.datetime + return _serialize_datetime(o, format) + except AttributeError: + pass + # Last, try datetime.timedelta + try: + return _timedelta_as_isostr(o) + except AttributeError: + # This will be raised when it hits value.total_seconds in the method above + pass + return o + + +def _get_rest_field( + attr_to_rest_field: typing.Dict[str, "_RestField"], rest_name: str +) -> typing.Optional["_RestField"]: + try: + return next(rf for rf in attr_to_rest_field.values() if rf._rest_name == rest_name) + except StopIteration: + return None + + +def _create_value(rf: typing.Optional["_RestField"], value: typing.Any) -> typing.Any: + if not rf: + return _serialize(value, None) + if rf._is_multipart_file_input: + return value + if rf._is_model: + return _deserialize(rf._type, value) + return _serialize(value, rf._format) + + +class Model(_MyMutableMapping): + _is_model = True + + def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + class_name = self.__class__.__name__ + if len(args) > 1: + raise TypeError(f"{class_name}.__init__() takes 2 positional arguments but {len(args) + 1} were given") + dict_to_pass = { + rest_field._rest_name: rest_field._default + for rest_field in self._attr_to_rest_field.values() + if rest_field._default is not _UNSET + } + if args: + dict_to_pass.update( + {k: _create_value(_get_rest_field(self._attr_to_rest_field, k), v) for k, v in args[0].items()} + ) + else: + non_attr_kwargs = [k for k in kwargs if k not in self._attr_to_rest_field] + if non_attr_kwargs: + # actual type errors only throw the first wrong keyword arg they see, so following that. + raise TypeError(f"{class_name}.__init__() got an unexpected keyword argument '{non_attr_kwargs[0]}'") + dict_to_pass.update( + { + self._attr_to_rest_field[k]._rest_name: _create_value(self._attr_to_rest_field[k], v) + for k, v in kwargs.items() + if v is not None + } + ) + super().__init__(dict_to_pass) + + def copy(self) -> "Model": + return Model(self.__dict__) + + def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self: # pylint: disable=unused-argument + # we know the last three classes in mro are going to be 'Model', 'dict', and 'object' + mros = cls.__mro__[:-3][::-1] # ignore model, dict, and object parents, and reverse the mro order + attr_to_rest_field: typing.Dict[str, _RestField] = { # map attribute name to rest_field property + k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type") + } + annotations = { + k: v + for mro_class in mros + if hasattr(mro_class, "__annotations__") # pylint: disable=no-member + for k, v in mro_class.__annotations__.items() # pylint: disable=no-member + } + for attr, rf in attr_to_rest_field.items(): + rf._module = cls.__module__ + if not rf._type: + rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None)) + if not rf._rest_name_input: + rf._rest_name_input = attr + cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items()) + + return super().__new__(cls) # pylint: disable=no-value-for-parameter + + def __init_subclass__(cls, discriminator: typing.Optional[str] = None) -> None: + for base in cls.__bases__: + if hasattr(base, "__mapping__"): # pylint: disable=no-member + base.__mapping__[discriminator or cls.__name__] = cls # type: ignore # pylint: disable=no-member + + @classmethod + def _get_discriminator(cls, exist_discriminators) -> typing.Optional[str]: + for v in cls.__dict__.values(): + if isinstance(v, _RestField) and v._is_discriminator and v._rest_name not in exist_discriminators: # pylint: disable=protected-access + return v._rest_name # pylint: disable=protected-access + return None + + @classmethod + def _deserialize(cls, data, exist_discriminators): + if not hasattr(cls, "__mapping__"): # pylint: disable=no-member + return cls(data) + discriminator = cls._get_discriminator(exist_discriminators) + exist_discriminators.append(discriminator) + mapped_cls = cls.__mapping__.get( + data.get(discriminator), cls + ) # pyright: ignore # pylint: disable=no-member + if mapped_cls == cls: + return cls(data) + return mapped_cls._deserialize(data, exist_discriminators) # pylint: disable=protected-access + + def as_dict(self, *, exclude_readonly: bool = False) -> typing.Dict[str, typing.Any]: + """Return a dict that can be JSONify using json.dump. + + :keyword bool exclude_readonly: Whether to remove the readonly properties. + :returns: A dict JSON compatible object + :rtype: dict + """ + + result = {} + if exclude_readonly: + readonly_props = [p._rest_name for p in self._attr_to_rest_field.values() if _is_readonly(p)] + for k, v in self.items(): + if exclude_readonly and k in readonly_props: # pyright: ignore + continue + is_multipart_file_input = False + try: + is_multipart_file_input = next(rf for rf in self._attr_to_rest_field.values() if rf._rest_name == k)._is_multipart_file_input + except StopIteration: + pass + result[k] = v if is_multipart_file_input else Model._as_dict_value(v, exclude_readonly=exclude_readonly) + return result + + @staticmethod + def _as_dict_value(v: typing.Any, exclude_readonly: bool = False) -> typing.Any: + if v is None or isinstance(v, _Null): + return None + if isinstance(v, (list, tuple, set)): + return type(v)( + Model._as_dict_value(x, exclude_readonly=exclude_readonly) + for x in v + ) + if isinstance(v, dict): + return { + dk: Model._as_dict_value(dv, exclude_readonly=exclude_readonly) + for dk, dv in v.items() + } + return v.as_dict(exclude_readonly=exclude_readonly) if hasattr(v, "as_dict") else v + +def _deserialize_model(model_deserializer: typing.Optional[typing.Callable], obj): + if _is_model(obj): + return obj + return _deserialize(model_deserializer, obj) + +def _deserialize_with_optional(if_obj_deserializer: typing.Optional[typing.Callable], obj): + if obj is None: + return obj + return _deserialize_with_callable(if_obj_deserializer, obj) + +def _deserialize_with_union(deserializers, obj): + for deserializer in deserializers: + try: + return _deserialize(deserializer, obj) + except DeserializationError: + pass + raise DeserializationError() + +def _deserialize_dict( + value_deserializer: typing.Optional[typing.Callable], + module: typing.Optional[str], + obj: typing.Dict[typing.Any, typing.Any], +): + if obj is None: + return obj + return { + k: _deserialize(value_deserializer, v, module) + for k, v in obj.items() + } + +def _deserialize_multiple_sequence( + entry_deserializers: typing.List[typing.Optional[typing.Callable]], + module: typing.Optional[str], + obj, +): + if obj is None: + return obj + return type(obj)( + _deserialize(deserializer, entry, module) + for entry, deserializer in zip(obj, entry_deserializers) + ) + +def _deserialize_sequence( + deserializer: typing.Optional[typing.Callable], + module: typing.Optional[str], + obj, +): + if obj is None: + return obj + return type(obj)(_deserialize(deserializer, entry, module) for entry in obj) + +def _sorted_annotations(types: typing.List[typing.Any]) -> typing.List[typing.Any]: + return sorted( + types, + key=lambda x: hasattr(x, "__name__") and x.__name__.lower() in ("str", "float", "int", "bool"), + ) + +def _get_deserialize_callable_from_annotation( # pylint: disable=R0911, R0915, R0912 + annotation: typing.Any, + module: typing.Optional[str], + rf: typing.Optional["_RestField"] = None, +) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]: + if not annotation or annotation in [int, float]: + return None + + # is it a type alias? + if isinstance(annotation, str): + if module is not None: + annotation = _get_type_alias_type(module, annotation) + + # is it a forward ref / in quotes? + if isinstance(annotation, (str, typing.ForwardRef)): + try: + model_name = annotation.__forward_arg__ # type: ignore + except AttributeError: + model_name = annotation + if module is not None: + annotation = _get_model(module, model_name) + + try: + if module and _is_model(annotation): + if rf: + rf._is_model = True + + return functools.partial(_deserialize_model, annotation) # pyright: ignore + except Exception: + pass + + # is it a literal? + try: + if annotation.__origin__ is typing.Literal: # pyright: ignore + return None + except AttributeError: + pass + + # is it optional? + try: + if any(a for a in annotation.__args__ if a == type(None)): # pyright: ignore + if len(annotation.__args__) <= 2: # pyright: ignore + if_obj_deserializer = _get_deserialize_callable_from_annotation( + next(a for a in annotation.__args__ if a != type(None)), module, rf # pyright: ignore + ) + + return functools.partial(_deserialize_with_optional, if_obj_deserializer) + # the type is Optional[Union[...]], we need to remove the None type from the Union + annotation_copy = copy.copy(annotation) + annotation_copy.__args__ = [a for a in annotation_copy.__args__ if a != type(None)] # pyright: ignore + return _get_deserialize_callable_from_annotation(annotation_copy, module, rf) + except AttributeError: + pass + + # is it union? + if getattr(annotation, "__origin__", None) is typing.Union: + # initial ordering is we make `string` the last deserialization option, because it is often them most generic + deserializers = [ + _get_deserialize_callable_from_annotation(arg, module, rf) + for arg in _sorted_annotations(annotation.__args__) # pyright: ignore + ] + + return functools.partial(_deserialize_with_union, deserializers) + + try: + if annotation._name == "Dict": # pyright: ignore + value_deserializer = _get_deserialize_callable_from_annotation( + annotation.__args__[1], module, rf # pyright: ignore + ) + + + return functools.partial( + _deserialize_dict, + value_deserializer, + module, + ) + except (AttributeError, IndexError): + pass + try: + if annotation._name in ["List", "Set", "Tuple", "Sequence"]: # pyright: ignore + if len(annotation.__args__) > 1: # pyright: ignore + + + entry_deserializers = [ + _get_deserialize_callable_from_annotation(dt, module, rf) for dt in annotation.__args__ # pyright: ignore + ] + return functools.partial(_deserialize_multiple_sequence, entry_deserializers, module) + deserializer = _get_deserialize_callable_from_annotation( + annotation.__args__[0], module, rf # pyright: ignore + ) + + + + return functools.partial(_deserialize_sequence, deserializer, module) + except (TypeError, IndexError, AttributeError, SyntaxError): + pass + + def _deserialize_default( + deserializer, + obj, + ): + if obj is None: + return obj + try: + return _deserialize_with_callable(deserializer, obj) + except Exception: + pass + return obj + + if get_deserializer(annotation, rf): + return functools.partial(_deserialize_default, get_deserializer(annotation, rf)) + + return functools.partial(_deserialize_default, annotation) + + +def _deserialize_with_callable( + deserializer: typing.Optional[typing.Callable[[typing.Any], typing.Any]], + value: typing.Any, +): + try: + if value is None or isinstance(value, _Null): + return None + if deserializer is None: + return value + if isinstance(deserializer, CaseInsensitiveEnumMeta): + try: + return deserializer(value) + except ValueError: + # for unknown value, return raw value + return value + if isinstance(deserializer, type) and issubclass(deserializer, Model): + return deserializer._deserialize(value, []) + return typing.cast(typing.Callable[[typing.Any], typing.Any], deserializer)(value) + except Exception as e: + raise DeserializationError() from e + + +def _deserialize( + deserializer: typing.Any, + value: typing.Any, + module: typing.Optional[str] = None, + rf: typing.Optional["_RestField"] = None, + format: typing.Optional[str] = None, +) -> typing.Any: + if isinstance(value, PipelineResponse): + value = value.http_response.json() + if rf is None and format: + rf = _RestField(format=format) + if not isinstance(deserializer, functools.partial): + deserializer = _get_deserialize_callable_from_annotation(deserializer, module, rf) + return _deserialize_with_callable(deserializer, value) + + +class _RestField: + def __init__( + self, + *, + name: typing.Optional[str] = None, + type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin + is_discriminator: bool = False, + visibility: typing.Optional[typing.List[str]] = None, + default: typing.Any = _UNSET, + format: typing.Optional[str] = None, + is_multipart_file_input: bool = False, + ): + self._type = type + self._rest_name_input = name + self._module: typing.Optional[str] = None + self._is_discriminator = is_discriminator + self._visibility = visibility + self._is_model = False + self._default = default + self._format = format + self._is_multipart_file_input = is_multipart_file_input + + @property + def _class_type(self) -> typing.Any: + return getattr(self._type, "args", [None])[0] + + @property + def _rest_name(self) -> str: + if self._rest_name_input is None: + raise ValueError("Rest name was never set") + return self._rest_name_input + + def __get__(self, obj: Model, type=None): # pylint: disable=redefined-builtin + # by this point, type and rest_name will have a value bc we default + # them in __new__ of the Model class + item = obj.get(self._rest_name) + if item is None: + return item + if self._is_model: + return item + return _deserialize(self._type, _serialize(item, self._format), rf=self) + + def __set__(self, obj: Model, value) -> None: + if value is None: + # we want to wipe out entries if users set attr to None + try: + obj.__delitem__(self._rest_name) + except KeyError: + pass + return + if self._is_model: + if not _is_model(value): + value = _deserialize(self._type, value) + obj.__setitem__(self._rest_name, value) + return + obj.__setitem__(self._rest_name, _serialize(value, self._format)) + + def _get_deserialize_callable_from_annotation( + self, annotation: typing.Any + ) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]: + return _get_deserialize_callable_from_annotation(annotation, self._module, self) + + +def rest_field( + *, + name: typing.Optional[str] = None, + type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin + visibility: typing.Optional[typing.List[str]] = None, + default: typing.Any = _UNSET, + format: typing.Optional[str] = None, + is_multipart_file_input: bool = False, +) -> typing.Any: + return _RestField(name=name, type=type, visibility=visibility, default=default, format=format, is_multipart_file_input=is_multipart_file_input) + + +def rest_discriminator( + *, + name: typing.Optional[str] = None, + type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin +) -> typing.Any: + return _RestField(name=name, type=type, is_discriminator=True) diff --git a/packages/autorest.python/generator/pygen/codegen/templates/model_container.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/model_container.py.jinja2 new file mode 100644 index 00000000000..4b824762f51 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/model_container.py.jinja2 @@ -0,0 +1,13 @@ +{% import 'operation_tools.jinja2' as op_tools %} +# coding=utf-8 +# pylint: disable=too-many-lines +{{ code_model.options['license_header'] }} + +{{ imports }} +{% for model in code_model.model_types %} +{% if model.base == "dpg" %} +{% include "model_dpg.py.jinja2" %} +{% elif model.base == "msrest" %} +{% include "model_msrest.py.jinja2" %} +{% endif %} +{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/model_dpg.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/model_dpg.py.jinja2 new file mode 100644 index 00000000000..048de17212c --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/model_dpg.py.jinja2 @@ -0,0 +1,90 @@ +{# actual template starts here #} +{% import "macros.jinja2" as macros %} + + +{{ serializer.declare_model(model) }} + """{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n ") }} + {% if model.discriminated_subtypes %} + + {{ serializer.discriminator_docstring(model) | wordwrap(width=95, break_long_words=False, break_on_hyphens=False, wrapstring='\n ') }} + {% endif %} + {% if model.has_readonly_or_constant_property %} + + Readonly variables are only populated by the server, and will be ignored when sending a request. + {% endif %} + {% if (model.properties | selectattr('optional', "equalto", false) | first) is defined %} + + All required parameters must be populated in order to send to server. + {% endif %} + + {% if model.properties != None %} + {% for p in model.properties %} + {% for line in serializer.variable_documentation_string(p) %} + {% for doc_string in line.replace('\n', '\n ').split('\n') %} + {{ macros.wrap_model_string(doc_string, '\n ') -}} + {% endfor %} + {% endfor %} + {% endfor %} + {% endif %} + """ + + {% if model.is_polymorphic %} + __mapping__: Dict[str, _model_base.Model] = {} + {% endif %} + {% for p in serializer.get_properties_to_declare(model)%} + {{ serializer.declare_property(p) }} + {% set prop_description = p.description(is_operation_file=False).replace('"', '\\"') %} + {% if prop_description %} + """{{ macros.wrap_model_string(prop_description, '\n ', '\"\"\"') -}} + {% endif %} + {% endfor %} + + {% if code_model.options["models_mode"] == "dpg" and model.flattened_property %} + __flattened_items = ["{{ model.flattened_items|join('\", \"') }}"] + {% endif %} + + {% if not model.internal and serializer.init_line(model) %} + @overload + def __init__( + self, + {% for param_signature in serializer.init_line(model) %} + {{ param_signature }} + {% endfor %} + ): + ... + + @overload + def __init__(self, mapping: Mapping[str, Any]): + """ + :param mapping: raw JSON to initialize the model. + :type mapping: Mapping[str, Any] + """ + + {% endif %} + {% set initialize_properties = serializer.initialize_properties(model) %} + {% if not model.internal and serializer.init_line(model) or initialize_properties %} + def __init__(self, *args: Any, **kwargs: Any) -> None:{{ '# pylint: disable=useless-super-delegation' if not initialize_properties else '' }} + {% for line in serializer.super_call(model) %} + {{ line }} + {% endfor %} + {% for initialize_property in initialize_properties %} + {{ initialize_property }} + {% endfor %} + {% if code_model.options["models_mode"] == "dpg" and model.flattened_property %} + {% set flattened_property_attr = model.flattened_property.client_name %} + + def __getattr__(self, name: str) -> Any: + if name in self.__flattened_items: + if self.{{ flattened_property_attr }} is None: return None + return getattr(self.{{ flattened_property_attr }}, name) + raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'") + + def __setattr__(self, key: str, value: Any) -> None: + if key in self.__flattened_items: + if self.{{ flattened_property_attr }} is None: + self.{{ flattened_property_attr }} = self._attr_to_rest_field["{{ flattened_property_attr }}"]._class_type() + setattr(self.properties, key, value) + else: + super().__setattr__(key, value) + {% endif %} + {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/model_init.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/model_init.py.jinja2 new file mode 100644 index 00000000000..f4710680849 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/model_init.py.jinja2 @@ -0,0 +1,28 @@ +{% import 'keywords.jinja2' as keywords %} +# coding=utf-8 +{{ code_model.options['license_header'] }} +{% if schemas %} + + {% for schema in schemas %} +from .{{ code_model.models_filename }} import {{ schema }} + {% endfor %} +{% endif %} +{% if enums %} + +{% for enum in enums %} +from .{{ code_model.enums_filename }} import {{ enum }} +{% endfor %} +{% endif %} +{{ keywords.patch_imports() }} +__all__ = [ + {% for schema in schemas %} + '{{ schema }}', + {% endfor %} + {% if enums %} + {% for enum in enums %} + '{{ enum }}', + {% endfor %} +{% endif %} +] +{{ keywords.extend_all }} +_patch_sdk() diff --git a/packages/autorest.python/generator/pygen/codegen/templates/model_msrest.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/model_msrest.py.jinja2 new file mode 100644 index 00000000000..15ccc8a2d90 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/model_msrest.py.jinja2 @@ -0,0 +1,92 @@ +{# actual template starts here #} +{% import "macros.jinja2" as macros %} +{% set initialize_properties = serializer.initialize_properties(model) %} +{% set exist_constant = (model.properties | selectattr('constant') | first) is defined %} + +{{ serializer.declare_model(model) }} + """{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n ") }} + {% if model.discriminated_subtypes %} + + {{ serializer.discriminator_docstring(model) | wordwrap(width=95, break_long_words=False, break_on_hyphens=False, wrapstring='\n ') }} + {% endif %} + {% if model.has_readonly_or_constant_property %} + + Variables are only populated by the server, and will be ignored when sending a request. + {% endif %} + {% if (model.properties | selectattr('optional', "equalto", false) | first) is defined %} + + All required parameters must be populated in order to send to server. + {% endif %} + + {% if model.properties != None %} + {% for p in model.properties %} + {% for line in serializer.variable_documentation_string(p) %} + {% for doc_string in line.replace('\n', '\n ').split('\n') %} + {{ macros.wrap_model_string(doc_string, '\n ') -}} + {% endfor %} + {% endfor %} + {% endfor %} + {% endif %} + """ +{% if initialize_properties or exist_constant %} + {% if (model.properties | selectattr('validation') ) | first %} + + _validation = { + {% for p in model.properties | selectattr('validation')%} + '{{ p.client_name }}': {{ str(p.validation) }}, + {% endfor %} + } + {% endif %} + + _attribute_map = { + {% if model.properties %} + {% for p in model.properties %} + {{ serializer.declare_property(p) }} + {% endfor %} + {% endif %} + } + {% if model.discriminated_subtypes %} + + _subtype_map = { + '{{ model.discriminator.client_name }}': {{ str(model.discriminated_subtypes_name_mapping) }} + } + {% endif %} + {% if model.xml_map_content %} + _xml_map = { + {{ model.xml_map_content }} + } + {% endif %} + {% if exist_constant %} + + {% for p in model.properties | selectattr('constant')%} + {{ p.client_name }} = {{ p.type.get_declaration() }} + {% endfor %} + {% endif %} + + def __init__({{ model.init_pylint_disable }} + self, + {% for param_signature in serializer.init_line(model) %} + {{ param_signature }} + {% endfor %} + **kwargs: Any + ) -> None: + """ + {% if model.properties %} + {% for p in model.properties %} + {% if p.is_input %} + {% for line in serializer.input_documentation_string(p) %} + {% for doc_string in line.replace('\n', '\n ').split('\n') %} + {{ macros.wrap_model_string(doc_string, '\n ') -}} + {% endfor %} + {% endfor %} + {% endif %} + {% endfor %} + {% endif %} + """ + {% for line in serializer.super_call(model) %} + {{ line }} + {% endfor %} + {% for initialize_property in initialize_properties %} + {{ initialize_property }} + {% endfor %} +{% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/operation.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/operation.py.jinja2 new file mode 100644 index 00000000000..aec6199880a --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/operation.py.jinja2 @@ -0,0 +1,21 @@ +{% import 'keywords.jinja2' as keywords with context %} +{% import 'operation_tools.jinja2' as op_tools %} +{# actual template starts here #} +{% if operation.overloads and operation.include_documentation %} +{{ op_tools.generate_overloads(operation_serializer, operation) }} +{% endif %} +{{ operation_serializer.method_signature_and_response_type_annotation(operation) }} +{% if operation.include_documentation %} + {{ op_tools.description(operation, operation_serializer) | indent }}{% endif %} + {% if not operation.abstract %} + {% if operation.deprecated %} + warnings.warn('Method {{operation.name}} is deprecated', DeprecationWarning) + {% endif %} + {{ op_tools.serialize(operation_serializer.error_map(operation)) | indent }} + {% if operation_serializer.pop_kwargs_from_signature(operation) %} + {{ op_tools.serialize(operation_serializer.pop_kwargs_from_signature(operation)) | indent }} + {% endif %} + {{ op_tools.serialize(operation_serializer.call_request_builder(operation)) | indent }} + {{ op_tools.serialize(operation_serializer.make_pipeline_call(operation)) | indent }} + {{ op_tools.serialize(operation_serializer.handle_response(operation)) | indent }} + {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/operation_group.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/operation_group.py.jinja2 new file mode 100644 index 00000000000..63e4385164b --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/operation_group.py.jinja2 @@ -0,0 +1,75 @@ +{% set base_class = ("(" + operation_group.base_class + ")") if operation_group.base_class else "" %} +{% macro check_abstract_methods() %} +{% if operation_group.has_abstract_operations %} + raise_if_not_implemented(self.__class__, [ + {% for operation in operation_group.operations if operation.abstract %} + '{{operation.name}}', + {% endfor %} + ]) +{% endif %} +{% endmacro %} +{% if operation_group.base_class %} +class {{ operation_group.class_name }}( {{ operation_group.pylint_disable }} + {{ operation_group.base_class }} +): +{% else %} +class {{ operation_group.class_name }}: {{ operation_group.pylint_disable }} +{% endif %} +{% if not operation_group.is_mixin %} + """ + .. warning:: + **DO NOT** instantiate this class directly. + + Instead, you should access the following operations through + :class:`{{ "~" + code_model.namespace + (".aio." if async_mode else ".") + operation_group.client.name }}`'s + :attr:`{{ operation_group.property_name }}` attribute. + """ + +{% if code_model.public_model_types and code_model.options["models_mode"] == "msrest" %} + models = _models + +{% endif %} + def __init__(self, *args, **kwargs){{ return_none_type_annotation }}: + input_args = list(args) + self._client = input_args.pop(0) if input_args else kwargs.pop("client") + self._config = input_args.pop(0) if input_args else kwargs.pop("config") + self._serialize = input_args.pop(0) if input_args else kwargs.pop("serializer") + self._deserialize = input_args.pop(0) if input_args else kwargs.pop("deserializer") + {% if code_model.options["multiapi"] %} + self._api_version = input_args.pop(0) if input_args else kwargs.pop("api_version") + {% endif %} + + {% for og in operation_group.operation_groups %} + self.{{ og.property_name }} = {{ og.class_name }}( + self._client, self._config, self._serialize, self._deserialize{{ ", self._api_version" if code_model.options["multiapi"] else "" }} + ) + {% endfor %} + +{{ check_abstract_methods() }} +{% elif operation_group.has_abstract_operations %} + + def __init__(self){{ return_none_type_annotation }}: +{{ check_abstract_methods() }} +{% endif %} +{% if operation_group.is_mixin and code_model.options["multiapi"] %} + def _api_version(self, op_name: str) -> str: # pylint: disable=unused-argument + try: + return self._config.api_version + except: # pylint: disable=bare-except + return "" +{% endif %} +{% for operation in operation_group.operations if not operation.abstract %} + +{% set request_builder = operation.request_builder %} +{% set operation_serializer = get_operation_serializer(operation) %} + {% if operation.operation_type == "lropaging" %} + {%- macro someop() %}{% include "lro_paging_operation.py.jinja2" %}{% endmacro %} + {% elif operation.operation_type == "lro" %} + {%- macro someop() %}{% include "lro_operation.py.jinja2" %}{% endmacro %} + {% elif operation.operation_type == "paging" %} + {% macro someop() %}{% include "paging_operation.py.jinja2" %}{% endmacro %} + {% else %} + {% macro someop() %}{% include "operation.py.jinja2" %}{% endmacro %} + {% endif %} + {{ someop()|indent }} +{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 new file mode 100644 index 00000000000..453087bcc18 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 @@ -0,0 +1,20 @@ +{% import 'operation_tools.jinja2' as op_tools %} +{% set operations_description = "async operations" if async_mode else "operations" %} +{% set return_none_type_annotation = " -> None" if async_mode else "" %} +# pylint: disable=too-many-lines,too-many-statements +# coding=utf-8 +{{ code_model.options['license_header'] }} +{{ imports }} +{{ unset }} +{% if code_model.options["builders_visibility"] == "embedded" and not async_mode %} +{{ op_tools.declare_serializer(code_model) }} + {% for operation_group in operation_groups %} + {% for request_builder in get_request_builders(operation_group) %} + +{% include "request_builder.py.jinja2" %} + {% endfor %} + {% endfor %} +{% endif %} +{% for operation_group in operation_groups %} + {% include "operation_group.py.jinja2" %} +{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/operation_tools.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/operation_tools.jinja2 new file mode 100644 index 00000000000..598da57e4d0 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/operation_tools.jinja2 @@ -0,0 +1,74 @@ +{% macro wrap_string(string, wrapstring, width=95) %}{{ string | replace("\\", "\\\\") | wordwrap(width=width, break_long_words=False, break_on_hyphens=False, wrapstring=wrapstring)}}{% endmacro %} + +{% macro description(builder, serializer) %} +{% set example_template = serializer.example_template(builder) %} + {% for description in serializer.description_and_summary(builder) %} + {% if description %} +{% set description = wrap_string(description, wrapstring='\n') %} + {% if serializer.line_too_long(example_template) and loop.first %} +# pylint: disable=line-too-long + {% endif %} +{{ '"""' + description if loop.first else description }} + {% else %} + + {% endif %} + {% endfor %} + {% for description in serializer.param_description_and_response_docstring(builder) %} + {% if description %} +{{ wrap_string(description, wrapstring='\n ') }} + {% else %} + + {% endif %} +{% endfor %} +{% if example_template %} + +Example: + .. code-block:: python + {% for template_line in example_template %} + {% if template_line %} + {% set wrap_amount = (template_line | length) - (template_line.lstrip() | length) + 10 %} + {{ wrap_string(template_line, wrapstring='\n' + " " * wrap_amount, width=(95 - wrap_amount)) }} + {% else %} + + {% endif %} + {% endfor %} +{% endif %} +""" +{% endmacro %} + +{% macro serialize(lines) %} +{% for line in lines %} + {% if line %} +{{ line }} + {% else %} + + {% endif %} +{% endfor %}{% endmacro %} + +{% macro serialize_with_wrap(lines, wrapstring) %} +{% for line in lines %} + {% if line %} +{{ wrap_string(line, wrapstring=wrapstring) }} + {% else %} + + {% endif %} +{% endfor %}{% endmacro %} + +{% macro declare_serializer(code_model) %} +{% if code_model.has_non_abstract_operations %} +_SERIALIZER = Serializer() + {% if not code_model.options["client_side_validation"] %} +_SERIALIZER.client_side_validation = False + {% endif %} +{% endif %} +{% endmacro %} + +{% macro generate_overloads(operation_serializer, operation) %} +{% for overload in operation.overloads %} +{{ operation_serializer.method_signature_and_response_type_annotation(overload) }} +{% if not operation.internal %} + {{ description(overload, operation_serializer) | indent }} +{% else %} + ... +{% endif %} +{% endfor %}{% endmacro %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 new file mode 100644 index 00000000000..bb38196f4ac --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 @@ -0,0 +1,17 @@ +{% import 'operation_tools.jinja2' as op_tools %} +{% import 'keywords.jinja2' as keywords %} +{# actual template starts here #} +# coding=utf-8 +{{ code_model.options['license_header'] }} + +{{ op_tools.serialize(operation_group_imports()) }} +{{ keywords.patch_imports() }} +__all__ = [ + {% for client in clients %} + {% for operation_group in client.operation_groups %} + '{{ operation_group.class_name }}', + {% endfor %} + {% endfor %} +] +{{ keywords.extend_all }} +_patch_sdk() diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 new file mode 100644 index 00000000000..bddf9a22c7b --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 @@ -0,0 +1,6 @@ +# Release History + +## 1.0.0b1 (1970-01-01) + +- Initial version + diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 new file mode 100644 index 00000000000..3499e00ae53 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 @@ -0,0 +1,21 @@ +Copyright (c) {{ code_model.options["company_name"] }} Corporation. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 new file mode 100644 index 00000000000..454a9ad2717 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 @@ -0,0 +1,8 @@ +include *.md +include LICENSE +include {{ package_name.replace('-', '/') }}/py.typed +recursive-include tests *.py +recursive-include samples *.py *.md +{%- for init_name in init_names %} +include {{ init_name }} +{%- endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 new file mode 100644 index 00000000000..fbea5913604 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 @@ -0,0 +1,107 @@ +{% if code_model.is_azure_flavor %} +{% if package_mode == "mgmtplane" -%} +# Microsoft Azure SDK for Python + +This is the Microsoft {{package_pprint_name}} Client Library. +This package has been tested with Python 3.8+. +For a more complete view of Azure libraries, see the [azure sdk python release](https://aka.ms/azsdk/python/all). + +# Usage + +To learn how to use this package, see the [quickstart guide](https://aka.ms/azsdk/python/mgmt) + +For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure) +Code samples for this package can be found at [{{package_pprint_name}}](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. +Additional code samples for different Azure services are available at [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) + +# Provide Feedback + +If you encounter any bugs or have suggestions, please file an issue in the +[Issues](https://github.com/Azure/azure-sdk-for-python/issues) +section of the project. + + +![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2F{{package_name}}%2FREADME.png) +{% else %} +# {{ package_pprint_name }} client library for Python + + +## Getting started + +### Install the package + +```bash +python -m pip install {{ package_name }} +``` + +#### Prequisites + +- Python 3.8 or later is required to use this package. +- You need an [Azure subscription][azure_sub] to use this package. +- An existing {{ package_pprint_name }} instance. + +{%- if token_credential %} +#### Create with an Azure Active Directory Credential +To use an [Azure Active Directory (AAD) token credential][authenticate_with_token], +provide an instance of the desired credential type obtained from the +[azure-identity][azure_identity_credentials] library. + +To authenticate with AAD, you must first [pip][pip] install [`azure-identity`][azure_identity_pip] + +After setup, you can choose which type of [credential][azure_identity_credentials] from azure.identity to use. +As an example, [DefaultAzureCredential][default_azure_credential] can be used to authenticate the client: + +Set the values of the client ID, tenant ID, and client secret of the AAD application as environment variables: +`AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_CLIENT_SECRET` + +Use the returned token credential to authenticate the client: + +```python +>>> from {{ namespace }} import {{ client_name }} +>>> from azure.identity import DefaultAzureCredential +>>> client = {{ client_name }}(endpoint='', credential=DefaultAzureCredential()) +``` + +## Examples + +```python +>>> from {{ namespace }} import {{ client_name }} +>>> from azure.identity import DefaultAzureCredential +>>> from {{ code_model.core_library }}.exceptions import HttpResponseError + +>>> client = {{ client_name }}(endpoint='', credential=DefaultAzureCredential()) +>>> try: + + except HttpResponseError as e: + print('service responds error: {}'.format(e.response.json())) + +``` +{%- endif %} + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require +you to agree to a Contributor License Agreement (CLA) declaring that you have +the right to, and actually do, grant us the rights to use your contribution. +For details, visit https://cla.microsoft.com. + +When you submit a pull request, a CLA-bot will automatically determine whether +you need to provide a CLA and decorate the PR appropriately (e.g., label, +comment). Simply follow the instructions provided by the bot. You will only +need to do this once across all repos using our CLA. + +This project has adopted the +[Microsoft Open Source Code of Conduct][code_of_conduct]. For more information, +see the Code of Conduct FAQ or contact opencode@microsoft.com with any +additional questions or comments. + + +[code_of_conduct]: https://opensource.microsoft.com/codeofconduct/ +[authenticate_with_token]: https://docs.microsoft.com/azure/cognitive-services/authentication?tabs=powershell#authenticate-with-an-authentication-token +[azure_identity_credentials]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/identity/azure-identity#credentials +[azure_identity_pip]: https://pypi.org/project/azure-identity/ +[default_azure_credential]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/identity/azure-identity#defaultazurecredential +[pip]: https://pypi.org/project/pip/ +[azure_sub]: https://azure.microsoft.com/free/ +{% endif %} +{% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 new file mode 100644 index 00000000000..a9782cabd58 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 @@ -0,0 +1,9 @@ +-e ../../../tools/azure-sdk-tools +../../core/azure-core +{% if token_credential -%} +../../identity/azure-identity +{% endif -%} +{% if azure_arm -%} +../../core/azure-mgmt-core +{% endif -%} +aiohttp diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 new file mode 100644 index 00000000000..db3290b7521 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 @@ -0,0 +1,110 @@ +# coding=utf-8 +{{ license_header }} +# coding: utf-8 +{% if package_mode %} +import os +import re +{% endif -%} +from setuptools import setup, find_packages + +{% set package_name = package_name or code_model.clients[0].name %} + +PACKAGE_NAME = "{{ package_name|lower }}" +{% if package_mode -%} +PACKAGE_PPRINT_NAME = "{{ package_pprint_name }}" + +# a-b-c => a/b/c +package_folder_path = PACKAGE_NAME.replace("-", "/") + +# Version extraction inspired from 'requests' +with open(os.path.join(package_folder_path, "_version.py"), "r") as fd: + version = re.search( + r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE + ).group(1) + +if not version: + raise RuntimeError("Cannot find version information") +{% set description = "\"" + code_model.options["company_name"] + " {} Client Library for Python\".format(PACKAGE_PPRINT_NAME)" %} +{% set author_email = "azpysdkhelp@microsoft.com" %} +{% set url = "https://github.com/Azure/azure-sdk-for-python/tree/main/sdk" %} +{% else %} +version = "{{ package_version }}" +{% set description = "\"%s\""|format(package_name) %} +{% set long_description = code_model.description %} +{% set author_email = "" %} +{% set url = "" %} +{% endif -%} + +setup( + name=PACKAGE_NAME, + version=version, + description={{ description }}, + {% if package_mode %} + long_description=open("README.md", "r").read(), + long_description_content_type="text/markdown", + license="MIT License", + author="{{ code_model.options["company_name"] }} Corporation", + {% endif %} + {% if code_model.is_azure_flavor %} + author_email="{{ author_email }}", + url="{{ url }}", + keywords="azure, azure sdk", + {% endif %} + {% if package_mode %} + classifiers=[ + "Development Status :: {{ dev_status }}", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", + ], + zip_safe=False, + packages=find_packages( + exclude=[ + "tests", + {% if pkgutil_names %} + # Exclude packages that will be covered by PEP420 or nspkg + {% endif %} + {%- for pkgutil_name in pkgutil_names %} + "{{ pkgutil_name }}", + {%- endfor %} + ] + ), + include_package_data=True, + package_data={ + '{{ code_model.namespace }}': ['py.typed'], + }, + {% else %} + packages=find_packages(), + include_package_data=True, + {% endif %} + install_requires=[ + {% if code_model.is_legacy %} + "msrest>=0.7.1", + {% else %} + "isodate>=0.6.1", + {% endif %} + {% if azure_arm %} + "azure-mgmt-core>=1.3.2", + {% elif code_model.is_azure_flavor %} + "azure-core>=1.30.0", + {% else %} + "corehttp[requests]", + {% endif %} + {% if code_model.need_typing_extensions %} + "typing-extensions>=4.6.0", + {% endif %} + ], + {% if package_mode %} + python_requires=">=3.8", + {% else %} + long_description="""\ + {{ code_model.description }} + """ + {% endif %} +) diff --git a/packages/autorest.python/generator/pygen/codegen/templates/paging_operation.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/paging_operation.py.jinja2 new file mode 100644 index 00000000000..d6f0d6f14dc --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/paging_operation.py.jinja2 @@ -0,0 +1,21 @@ +{% import 'operation_tools.jinja2' as op_tools with context %} +{# actual template starts here #} +{% if operation.overloads and operation.include_documentation %} +{{ op_tools.generate_overloads(operation_serializer, operation) }} +{% endif %} +{{ operation_serializer.method_signature_and_response_type_annotation(operation) }} +{% if operation.include_documentation %} + {{ op_tools.description(operation, operation_serializer) | indent }}{% endif %} + {% if not operation.abstract %} + {% if operation.deprecated %} + warnings.warn('Method {{operation.name}} is deprecated', DeprecationWarning) + {% endif %} + {% if operation_serializer.pop_kwargs_from_signature(operation) %} + {{ op_tools.serialize(operation_serializer.pop_kwargs_from_signature(operation)) | indent }} + {% endif %} + {{ op_tools.serialize(operation_serializer.set_up_params_for_pager(operation)) | indent }} + + return {{ operation.get_pager(async_mode) }}( + get_next, extract_data + ) + {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/patch.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/patch.py.jinja2 new file mode 100644 index 00000000000..87c09548714 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/patch.py.jinja2 @@ -0,0 +1,19 @@ +# ------------------------------------ +# Copyright (c) {{ code_model.options["company_name"] }} Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Customize generated code here. + +Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize +""" +{{ imports }} + +__all__: List[str] = [] # Add all objects you want publicly available to users at this package level + +def patch_sdk(): + """Do not remove from this file. + + `patch_sdk` is a last resort escape hatch that allows you to do customizations + you can't accomplish using the techniques described in + https://aka.ms/azsdk/python/dpcodegen/python/customize + """ diff --git a/packages/autorest.python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 new file mode 100644 index 00000000000..5960c353a89 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore \ No newline at end of file diff --git a/packages/autorest.python/generator/pygen/codegen/templates/request_builder.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/request_builder.py.jinja2 new file mode 100644 index 00000000000..f4810000e7b --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/request_builder.py.jinja2 @@ -0,0 +1,28 @@ +{% import 'keywords.jinja2' as keywords with context %} +{% import 'operation_tools.jinja2' as op_tools with context %} +{{ request_builder_serializer.method_signature_and_response_type_annotation(request_builder) }} +{% if code_model.options["builders_visibility"] == "public" %} + {{ op_tools.description(request_builder, request_builder_serializer) | indent }} +{% endif %} +{% if not request_builder.is_overload %} + {% if request_builder_serializer.pop_kwargs_from_signature(request_builder) %} + {{ op_tools.serialize(request_builder_serializer.pop_kwargs_from_signature(request_builder)) | indent }} + {%- endif -%} + {% if request_builder_serializer.declare_non_inputtable_constants(request_builder) %} + {{ op_tools.serialize(request_builder_serializer.declare_non_inputtable_constants(request_builder)) | indent }} + {% endif %} + # Construct URL + {{ request_builder_serializer.construct_url(request_builder) }} + {% if request_builder.parameters.path %} + {{ op_tools.serialize(request_builder_serializer.serialize_path(request_builder)) | indent }} + _url: str = _url.format(**path_format_arguments) # type: ignore + {% endif %} + + {% if request_builder.parameters.query %} + {{ op_tools.serialize(request_builder_serializer.serialize_query(request_builder)) | indent }} + {% endif %} + {% if request_builder.parameters.headers %} + {{ op_tools.serialize(request_builder_serializer.serialize_headers(request_builder)) | indent }} + {% endif %} + {{ op_tools.serialize(request_builder_serializer.create_http_request(request_builder)) | indent }} +{% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/request_builders.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/request_builders.py.jinja2 new file mode 100644 index 00000000000..3c72ec2ac49 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/request_builders.py.jinja2 @@ -0,0 +1,10 @@ +{% import 'operation_tools.jinja2' as op_tools %} +# coding=utf-8 +{{ code_model.options['license_header'] }} +{{ imports }} + +{{ op_tools.declare_serializer(code_model) }} +{% for request_builder in request_builders %} + +{% include "request_builder.py.jinja2" %} +{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/rest_init.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/rest_init.py.jinja2 new file mode 100644 index 00000000000..9833f3fdc47 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/rest_init.py.jinja2 @@ -0,0 +1,12 @@ +# coding=utf-8 +{{ code_model.options['license_header'] }} + +{% for request_builder in request_builders %} +from ._request_builders import {{ request_builder.name }} +{% endfor %} + +__all__ = [ + {% for request_builder in request_builders %} + '{{ request_builder.name }}', + {% endfor %} +] diff --git a/packages/autorest.python/generator/pygen/codegen/templates/sample.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/sample.py.jinja2 new file mode 100644 index 00000000000..421a3a7c552 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/sample.py.jinja2 @@ -0,0 +1,44 @@ +# coding=utf-8 +{% set aad_token = "DefaultAzureCredential" %} +{% set azure_key = "AzureKeyCredential" %} +{{ code_model.options['license_header'] }} + +{{ imports }} +""" +# PREREQUISITES +{% if "credential" in client_params and aad_token in client_params["credential"] %} + pip install azure-identity +{% endif %} + pip install {{ (code_model.options["package_name"] or code_model.clients[0].name)|lower }} +# USAGE + python {{ file_name }} + {% if "credential" in client_params and aad_token in client_params["credential"] %} + + Before run the sample, please set the values of the client ID, tenant ID and client secret + of the AAD application as environment variables: AZURE_CLIENT_ID, AZURE_TENANT_ID, + AZURE_CLIENT_SECRET. For more info about how to get the value, please see: + https://docs.microsoft.com/azure/active-directory/develop/howto-create-service-principal-portal + {% elif "credential" in client_params and azure_key in client_params["credential"] %} + + Before run the sample, please set environment variables AZURE_KEY with real value + which can access your service + {% endif %} +""" +def main(): + client = {{ code_model.clients[0].name }}( + {% for key,value in client_params.items() %} + {{ key }}={{ value }}, + {% endfor %} + ) + + {{ return_var }}client{{ operation_group_name }}{{ operation_name }}( + {% for key, value in operation_params.items() %} + {{ key }}={{ value|indent(8) }}, + {% endfor %} + ){{ operation_result }} + +{% if origin_file %} +# x-ms-original-file: {{ origin_file }} +{% endif %} +if __name__ == "__main__": + main() diff --git a/packages/autorest.python/generator/pygen/codegen/templates/serialization.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/serialization.py.jinja2 new file mode 100644 index 00000000000..bb5dba9d174 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/serialization.py.jinja2 @@ -0,0 +1,2004 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) {{ code_model.options["company_name"] }} Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +# pylint: skip-file +# pyright: reportUnnecessaryTypeIgnoreComment=false + +from base64 import b64decode, b64encode +import calendar +import datetime +import decimal +import email +from enum import Enum +import json +import logging +import re +import sys +import codecs +from typing import ( + Dict, + Any, + cast, + Optional, + Union, + AnyStr, + IO, + Mapping, + Callable, + TypeVar, + MutableMapping, + Type, + List, + Mapping, +) + +try: + from urllib import quote # type: ignore +except ImportError: + from urllib.parse import quote +import xml.etree.ElementTree as ET + +import isodate # type: ignore + +from {{ code_model.core_library }}.exceptions import DeserializationError, SerializationError +from {{ code_model.core_library }}.serialization import NULL as CoreNull + +_BOM = codecs.BOM_UTF8.decode(encoding="utf-8") + +ModelType = TypeVar("ModelType", bound="Model") +JSON = MutableMapping[str, Any] + + +class RawDeserializer: + + # Accept "text" because we're open minded people... + JSON_REGEXP = re.compile(r"^(application|text)/([a-z+.]+\+)?json$") + + # Name used in context + CONTEXT_NAME = "deserialized_data" + + @classmethod + def deserialize_from_text(cls, data: Optional[Union[AnyStr, IO]], content_type: Optional[str] = None) -> Any: + """Decode data according to content-type. + + Accept a stream of data as well, but will be load at once in memory for now. + + If no content-type, will return the string version (not bytes, not stream) + + :param data: Input, could be bytes or stream (will be decoded with UTF8) or text + :type data: str or bytes or IO + :param str content_type: The content type. + """ + if hasattr(data, "read"): + # Assume a stream + data = cast(IO, data).read() + + if isinstance(data, bytes): + data_as_str = data.decode(encoding="utf-8-sig") + else: + # Explain to mypy the correct type. + data_as_str = cast(str, data) + + # Remove Byte Order Mark if present in string + data_as_str = data_as_str.lstrip(_BOM) + + if content_type is None: + return data + + if cls.JSON_REGEXP.match(content_type): + try: + return json.loads(data_as_str) + except ValueError as err: + raise DeserializationError("JSON is invalid: {}".format(err), err) + elif "xml" in (content_type or []): + try: + + try: + if isinstance(data, unicode): # type: ignore + # If I'm Python 2.7 and unicode XML will scream if I try a "fromstring" on unicode string + data_as_str = data_as_str.encode(encoding="utf-8") # type: ignore + except NameError: + pass + + return ET.fromstring(data_as_str) # nosec + except ET.ParseError as err: + # It might be because the server has an issue, and returned JSON with + # content-type XML.... + # So let's try a JSON load, and if it's still broken + # let's flow the initial exception + def _json_attemp(data): + try: + return True, json.loads(data) + except ValueError: + return False, None # Don't care about this one + + success, json_result = _json_attemp(data) + if success: + return json_result + # If i'm here, it's not JSON, it's not XML, let's scream + # and raise the last context in this block (the XML exception) + # The function hack is because Py2.7 messes up with exception + # context otherwise. + _LOGGER.critical("Wasn't XML not JSON, failing") + raise DeserializationError("XML is invalid") from err + raise DeserializationError("Cannot deserialize content-type: {}".format(content_type)) + + @classmethod + def deserialize_from_http_generics(cls, body_bytes: Optional[Union[AnyStr, IO]], headers: Mapping) -> Any: + """Deserialize from HTTP response. + + Use bytes and headers to NOT use any requests/aiohttp or whatever + specific implementation. + Headers will tested for "content-type" + """ + # Try to use content-type from headers if available + content_type = None + if "content-type" in headers: + content_type = headers["content-type"].split(";")[0].strip().lower() + # Ouch, this server did not declare what it sent... + # Let's guess it's JSON... + # Also, since Autorest was considering that an empty body was a valid JSON, + # need that test as well.... + else: + content_type = "application/json" + + if body_bytes: + return cls.deserialize_from_text(body_bytes, content_type) + return None + + +_LOGGER = logging.getLogger(__name__) + +try: + _long_type = long # type: ignore +except NameError: + _long_type = int + + +class UTC(datetime.tzinfo): + """Time Zone info for handling UTC""" + + def utcoffset(self, dt): + """UTF offset for UTC is 0.""" + return datetime.timedelta(0) + + def tzname(self, dt): + """Timestamp representation.""" + return "Z" + + def dst(self, dt): + """No daylight saving for UTC.""" + return datetime.timedelta(hours=1) + + +try: + from datetime import timezone as _FixedOffset # type: ignore +except ImportError: # Python 2.7 + + class _FixedOffset(datetime.tzinfo): # type: ignore + """Fixed offset in minutes east from UTC. + Copy/pasted from Python doc + :param datetime.timedelta offset: offset in timedelta format + """ + + def __init__(self, offset): + self.__offset = offset + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return str(self.__offset.total_seconds() / 3600) + + def __repr__(self): + return "".format(self.tzname(None)) + + def dst(self, dt): + return datetime.timedelta(0) + + def __getinitargs__(self): + return (self.__offset,) + + +try: + from datetime import timezone + + TZ_UTC = timezone.utc +except ImportError: + TZ_UTC = UTC() # type: ignore + +_FLATTEN = re.compile(r"(? None: + self.additional_properties: Optional[Dict[str, Any]] = {} + for k in kwargs: + if k not in self._attribute_map: + _LOGGER.warning("%s is not a known attribute of class %s and will be ignored", k, self.__class__) + elif k in self._validation and self._validation[k].get("readonly", False): + _LOGGER.warning("Readonly attribute %s will be ignored in class %s", k, self.__class__) + else: + setattr(self, k, kwargs[k]) + + def __eq__(self, other: Any) -> bool: + """Compare objects by comparing all attributes.""" + if isinstance(other, self.__class__): + return self.__dict__ == other.__dict__ + return False + + def __ne__(self, other: Any) -> bool: + """Compare objects by comparing all attributes.""" + return not self.__eq__(other) + + def __str__(self) -> str: + return str(self.__dict__) + + @classmethod + def enable_additional_properties_sending(cls) -> None: + cls._attribute_map["additional_properties"] = {"key": "", "type": "{object}"} + + @classmethod + def is_xml_model(cls) -> bool: + try: + cls._xml_map # type: ignore + except AttributeError: + return False + return True + + @classmethod + def _create_xml_node(cls): + """Create XML node.""" + try: + xml_map = cls._xml_map # type: ignore + except AttributeError: + xml_map = {} + + return _create_xml_node(xml_map.get("name", cls.__name__), xml_map.get("prefix", None), xml_map.get("ns", None)) + + def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> JSON: + """Return the JSON that would be sent to server from this model. + + This is an alias to `as_dict(full_restapi_key_transformer, keep_readonly=False)`. + + If you want XML serialization, you can pass the kwargs is_xml=True. + + :param bool keep_readonly: If you want to serialize the readonly attributes + :returns: A dict JSON compatible object + :rtype: dict + """ + serializer = Serializer(self._infer_class_models()) + return serializer._serialize(self, keep_readonly=keep_readonly, **kwargs) # type: ignore + + def as_dict( + self, + keep_readonly: bool = True, + key_transformer: Callable[ + [str, Dict[str, Any], Any], Any + ] = attribute_transformer, + **kwargs: Any + ) -> JSON: + """Return a dict that can be serialized using json.dump. + + Advanced usage might optionally use a callback as parameter: + + .. code::python + + def my_key_transformer(key, attr_desc, value): + return key + + Key is the attribute name used in Python. Attr_desc + is a dict of metadata. Currently contains 'type' with the + msrest type and 'key' with the RestAPI encoded key. + Value is the current value in this object. + + The string returned will be used to serialize the key. + If the return type is a list, this is considered hierarchical + result dict. + + See the three examples in this file: + + - attribute_transformer + - full_restapi_key_transformer + - last_restapi_key_transformer + + If you want XML serialization, you can pass the kwargs is_xml=True. + + :param function key_transformer: A key transformer function. + :returns: A dict JSON compatible object + :rtype: dict + """ + serializer = Serializer(self._infer_class_models()) + return serializer._serialize(self, key_transformer=key_transformer, keep_readonly=keep_readonly, **kwargs) # type: ignore + + @classmethod + def _infer_class_models(cls): + try: + str_models = cls.__module__.rsplit(".", 1)[0] + models = sys.modules[str_models] + client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)} + if cls.__name__ not in client_models: + raise ValueError("Not Autorest generated code") + except Exception: + # Assume it's not Autorest generated (tests?). Add ourselves as dependencies. + client_models = {cls.__name__: cls} + return client_models + + @classmethod + def deserialize(cls: Type[ModelType], data: Any, content_type: Optional[str] = None) -> ModelType: + """Parse a str using the RestAPI syntax and return a model. + + :param str data: A str using RestAPI structure. JSON by default. + :param str content_type: JSON by default, set application/xml if XML. + :returns: An instance of this model + :raises: DeserializationError if something went wrong + """ + deserializer = Deserializer(cls._infer_class_models()) + return deserializer(cls.__name__, data, content_type=content_type) # type: ignore + + @classmethod + def from_dict( + cls: Type[ModelType], + data: Any, + key_extractors: Optional[Callable[[str, Dict[str, Any], Any], Any]] = None, + content_type: Optional[str] = None, + ) -> ModelType: + """Parse a dict using given key extractor return a model. + + By default consider key + extractors (rest_key_case_insensitive_extractor, attribute_key_case_insensitive_extractor + and last_rest_key_case_insensitive_extractor) + + :param dict data: A dict using RestAPI structure + :param str content_type: JSON by default, set application/xml if XML. + :returns: An instance of this model + :raises: DeserializationError if something went wrong + """ + deserializer = Deserializer(cls._infer_class_models()) + deserializer.key_extractors = ( # type: ignore + [ # type: ignore + attribute_key_case_insensitive_extractor, + rest_key_case_insensitive_extractor, + last_rest_key_case_insensitive_extractor, + ] + if key_extractors is None + else key_extractors + ) + return deserializer(cls.__name__, data, content_type=content_type) # type: ignore + + @classmethod + def _flatten_subtype(cls, key, objects): + if "_subtype_map" not in cls.__dict__: + return {} + result = dict(cls._subtype_map[key]) + for valuetype in cls._subtype_map[key].values(): + result.update(objects[valuetype]._flatten_subtype(key, objects)) + return result + + @classmethod + def _classify(cls, response, objects): + """Check the class _subtype_map for any child classes. + We want to ignore any inherited _subtype_maps. + Remove the polymorphic key from the initial data. + """ + for subtype_key in cls.__dict__.get("_subtype_map", {}).keys(): + subtype_value = None + + if not isinstance(response, ET.Element): + rest_api_response_key = cls._get_rest_key_parts(subtype_key)[-1] + subtype_value = response.pop(rest_api_response_key, None) or response.pop(subtype_key, None) + else: + subtype_value = xml_key_extractor(subtype_key, cls._attribute_map[subtype_key], response) + if subtype_value: + # Try to match base class. Can be class name only + # (bug to fix in Autorest to support x-ms-discriminator-name) + if cls.__name__ == subtype_value: + return cls + flatten_mapping_type = cls._flatten_subtype(subtype_key, objects) + try: + return objects[flatten_mapping_type[subtype_value]] # type: ignore + except KeyError: + _LOGGER.warning( + "Subtype value %s has no mapping, use base class %s.", + subtype_value, + cls.__name__, + ) + break + else: + _LOGGER.warning("Discriminator %s is absent or null, use base class %s.", subtype_key, cls.__name__) + break + return cls + + @classmethod + def _get_rest_key_parts(cls, attr_key): + """Get the RestAPI key of this attr, split it and decode part + :param str attr_key: Attribute key must be in attribute_map. + :returns: A list of RestAPI part + :rtype: list + """ + rest_split_key = _FLATTEN.split(cls._attribute_map[attr_key]["key"]) + return [_decode_attribute_map_key(key_part) for key_part in rest_split_key] + + +def _decode_attribute_map_key(key): + """This decode a key in an _attribute_map to the actual key we want to look at + inside the received data. + + :param str key: A key string from the generated code + """ + return key.replace("\\.", ".") + + +class Serializer(object): + """Request object model serializer.""" + + basic_types = {str: "str", int: "int", bool: "bool", float: "float"} + + _xml_basic_types_serializers = {"bool": lambda x: str(x).lower()} + days = {0: "Mon", 1: "Tue", 2: "Wed", 3: "Thu", 4: "Fri", 5: "Sat", 6: "Sun"} + months = { + 1: "Jan", + 2: "Feb", + 3: "Mar", + 4: "Apr", + 5: "May", + 6: "Jun", + 7: "Jul", + 8: "Aug", + 9: "Sep", + 10: "Oct", + 11: "Nov", + 12: "Dec", + } + validation = { + "min_length": lambda x, y: len(x) < y, + "max_length": lambda x, y: len(x) > y, + "minimum": lambda x, y: x < y, + "maximum": lambda x, y: x > y, + "minimum_ex": lambda x, y: x <= y, + "maximum_ex": lambda x, y: x >= y, + "min_items": lambda x, y: len(x) < y, + "max_items": lambda x, y: len(x) > y, + "pattern": lambda x, y: not re.match(y, x, re.UNICODE), + "unique": lambda x, y: len(x) != len(set(x)), + "multiple": lambda x, y: x % y != 0, + } + + def __init__(self, classes: Optional[Mapping[str, type]]=None): + self.serialize_type = { + "iso-8601": Serializer.serialize_iso, + "rfc-1123": Serializer.serialize_rfc, + "unix-time": Serializer.serialize_unix, + "duration": Serializer.serialize_duration, + "date": Serializer.serialize_date, + "time": Serializer.serialize_time, + "decimal": Serializer.serialize_decimal, + "long": Serializer.serialize_long, + "bytearray": Serializer.serialize_bytearray, + "base64": Serializer.serialize_base64, + "object": self.serialize_object, + "[]": self.serialize_iter, + "{}": self.serialize_dict, + } + self.dependencies: Dict[str, type] = dict(classes) if classes else {} + self.key_transformer = full_restapi_key_transformer + self.client_side_validation = True + + def _serialize(self, target_obj, data_type=None, **kwargs): + """Serialize data into a string according to type. + + :param target_obj: The data to be serialized. + :param str data_type: The type to be serialized from. + :rtype: str, dict + :raises: SerializationError if serialization fails. + """ + key_transformer = kwargs.get("key_transformer", self.key_transformer) + keep_readonly = kwargs.get("keep_readonly", False) + if target_obj is None: + return None + + attr_name = None + class_name = target_obj.__class__.__name__ + + if data_type: + return self.serialize_data(target_obj, data_type, **kwargs) + + if not hasattr(target_obj, "_attribute_map"): + data_type = type(target_obj).__name__ + if data_type in self.basic_types.values(): + return self.serialize_data(target_obj, data_type, **kwargs) + + # Force "is_xml" kwargs if we detect a XML model + try: + is_xml_model_serialization = kwargs["is_xml"] + except KeyError: + is_xml_model_serialization = kwargs.setdefault("is_xml", target_obj.is_xml_model()) + + serialized = {} + if is_xml_model_serialization: + serialized = target_obj._create_xml_node() + try: + attributes = target_obj._attribute_map + for attr, attr_desc in attributes.items(): + attr_name = attr + if not keep_readonly and target_obj._validation.get(attr_name, {}).get("readonly", False): + continue + + if attr_name == "additional_properties" and attr_desc["key"] == "": + if target_obj.additional_properties is not None: + serialized.update(target_obj.additional_properties) + continue + try: + + orig_attr = getattr(target_obj, attr) + if is_xml_model_serialization: + pass # Don't provide "transformer" for XML for now. Keep "orig_attr" + else: # JSON + keys, orig_attr = key_transformer(attr, attr_desc.copy(), orig_attr) + keys = keys if isinstance(keys, list) else [keys] + + kwargs["serialization_ctxt"] = attr_desc + new_attr = self.serialize_data(orig_attr, attr_desc["type"], **kwargs) + + if is_xml_model_serialization: + xml_desc = attr_desc.get("xml", {}) + xml_name = xml_desc.get("name", attr_desc["key"]) + xml_prefix = xml_desc.get("prefix", None) + xml_ns = xml_desc.get("ns", None) + if xml_desc.get("attr", False): + if xml_ns: + ET.register_namespace(xml_prefix, xml_ns) + xml_name = "{% raw %}{{{}}}{}{% endraw %}".format(xml_ns, xml_name) + serialized.set(xml_name, new_attr) # type: ignore + continue + if xml_desc.get("text", False): + serialized.text = new_attr # type: ignore + continue + if isinstance(new_attr, list): + serialized.extend(new_attr) # type: ignore + elif isinstance(new_attr, ET.Element): + # If the down XML has no XML/Name, we MUST replace the tag with the local tag. But keeping the namespaces. + if "name" not in getattr(orig_attr, "_xml_map", {}): + splitted_tag = new_attr.tag.split("}") + if len(splitted_tag) == 2: # Namespace + new_attr.tag = "}".join([splitted_tag[0], xml_name]) + else: + new_attr.tag = xml_name + serialized.append(new_attr) # type: ignore + else: # That's a basic type + # Integrate namespace if necessary + local_node = _create_xml_node(xml_name, xml_prefix, xml_ns) + local_node.text = str(new_attr) + serialized.append(local_node) # type: ignore + else: # JSON + for k in reversed(keys): # type: ignore + new_attr = {k: new_attr} + + _new_attr = new_attr + _serialized = serialized + for k in keys: # type: ignore + if k not in _serialized: + _serialized.update(_new_attr) # type: ignore + _new_attr = _new_attr[k] # type: ignore + _serialized = _serialized[k] + except ValueError as err: + if isinstance(err, SerializationError): + raise + + except (AttributeError, KeyError, TypeError) as err: + msg = "Attribute {} in object {} cannot be serialized.\n{}".format(attr_name, class_name, str(target_obj)) + raise SerializationError(msg) from err + else: + return serialized + + def body(self, data, data_type, **kwargs): + """Serialize data intended for a request body. + + :param data: The data to be serialized. + :param str data_type: The type to be serialized from. + :rtype: dict + :raises: SerializationError if serialization fails. + :raises: ValueError if data is None + """ + + # Just in case this is a dict + internal_data_type_str = data_type.strip("[]{}") + internal_data_type = self.dependencies.get(internal_data_type_str, None) + try: + is_xml_model_serialization = kwargs["is_xml"] + except KeyError: + if internal_data_type and issubclass(internal_data_type, Model): + is_xml_model_serialization = kwargs.setdefault("is_xml", internal_data_type.is_xml_model()) + else: + is_xml_model_serialization = False + if internal_data_type and not isinstance(internal_data_type, Enum): + try: + deserializer = Deserializer(self.dependencies) + # Since it's on serialization, it's almost sure that format is not JSON REST + # We're not able to deal with additional properties for now. + deserializer.additional_properties_detection = False + if is_xml_model_serialization: + deserializer.key_extractors = [ # type: ignore + attribute_key_case_insensitive_extractor, + ] + else: + deserializer.key_extractors = [ + rest_key_case_insensitive_extractor, + attribute_key_case_insensitive_extractor, + last_rest_key_case_insensitive_extractor, + ] + data = deserializer._deserialize(data_type, data) + except DeserializationError as err: + raise SerializationError("Unable to build a model: " + str(err)) from err + + return self._serialize(data, data_type, **kwargs) + + def url(self, name, data, data_type, **kwargs): + """Serialize data intended for a URL path. + + :param data: The data to be serialized. + :param str data_type: The type to be serialized from. + :rtype: str + :raises: TypeError if serialization fails. + :raises: ValueError if data is None + """ + try: + output = self.serialize_data(data, data_type, **kwargs) + if data_type == "bool": + output = json.dumps(output) + + if kwargs.get("skip_quote") is True: + output = str(output) + output = output.replace("{", quote("{")).replace("}", quote("}")) + else: + output = quote(str(output), safe="") + except SerializationError: + raise TypeError("{} must be type {}.".format(name, data_type)) + else: + return output + + def query(self, name, data, data_type, **kwargs): + """Serialize data intended for a URL query. + + :param data: The data to be serialized. + :param str data_type: The type to be serialized from. + :keyword bool skip_quote: Whether to skip quote the serialized result. + Defaults to False. + :rtype: str, list + :raises: TypeError if serialization fails. + :raises: ValueError if data is None + """ + try: + # Treat the list aside, since we don't want to encode the div separator + if data_type.startswith("["): + internal_data_type = data_type[1:-1] + do_quote = not kwargs.get('skip_quote', False) + return self.serialize_iter(data, internal_data_type, do_quote=do_quote, **kwargs) + + # Not a list, regular serialization + output = self.serialize_data(data, data_type, **kwargs) + if data_type == "bool": + output = json.dumps(output) + if kwargs.get("skip_quote") is True: + output = str(output) + else: + output = quote(str(output), safe="") + except SerializationError: + raise TypeError("{} must be type {}.".format(name, data_type)) + else: + return str(output) + + def header(self, name, data, data_type, **kwargs): + """Serialize data intended for a request header. + + :param data: The data to be serialized. + :param str data_type: The type to be serialized from. + :rtype: str + :raises: TypeError if serialization fails. + :raises: ValueError if data is None + """ + try: + if data_type in ["[str]"]: + data = ["" if d is None else d for d in data] + + output = self.serialize_data(data, data_type, **kwargs) + if data_type == "bool": + output = json.dumps(output) + except SerializationError: + raise TypeError("{} must be type {}.".format(name, data_type)) + else: + return str(output) + + def serialize_data(self, data, data_type, **kwargs): + """Serialize generic data according to supplied data type. + + :param data: The data to be serialized. + :param str data_type: The type to be serialized from. + :param bool required: Whether it's essential that the data not be + empty or None + :raises: AttributeError if required data is None. + :raises: ValueError if data is None + :raises: SerializationError if serialization fails. + """ + if data is None: + raise ValueError("No value for given attribute") + + try: + if data is CoreNull: + return None + if data_type in self.basic_types.values(): + return self.serialize_basic(data, data_type, **kwargs) + + elif data_type in self.serialize_type: + return self.serialize_type[data_type](data, **kwargs) + + # If dependencies is empty, try with current data class + # It has to be a subclass of Enum anyway + enum_type = self.dependencies.get(data_type, data.__class__) + if issubclass(enum_type, Enum): + return Serializer.serialize_enum(data, enum_obj=enum_type) + + iter_type = data_type[0] + data_type[-1] + if iter_type in self.serialize_type: + return self.serialize_type[iter_type](data, data_type[1:-1], **kwargs) + + except (ValueError, TypeError) as err: + msg = "Unable to serialize value: {!r} as type: {!r}." + raise SerializationError(msg.format(data, data_type)) from err + else: + return self._serialize(data, **kwargs) + + @classmethod + def _get_custom_serializers(cls, data_type, **kwargs): + custom_serializer = kwargs.get("basic_types_serializers", {}).get(data_type) + if custom_serializer: + return custom_serializer + if kwargs.get("is_xml", False): + return cls._xml_basic_types_serializers.get(data_type) + + @classmethod + def serialize_basic(cls, data, data_type, **kwargs): + """Serialize basic builting data type. + Serializes objects to str, int, float or bool. + + Possible kwargs: + - basic_types_serializers dict[str, callable] : If set, use the callable as serializer + - is_xml bool : If set, use xml_basic_types_serializers + + :param data: Object to be serialized. + :param str data_type: Type of object in the iterable. + """ + custom_serializer = cls._get_custom_serializers(data_type, **kwargs) + if custom_serializer: + return custom_serializer(data) + if data_type == "str": + return cls.serialize_unicode(data) + return eval(data_type)(data) # nosec + + @classmethod + def serialize_unicode(cls, data): + """Special handling for serializing unicode strings in Py2. + Encode to UTF-8 if unicode, otherwise handle as a str. + + :param data: Object to be serialized. + :rtype: str + """ + try: # If I received an enum, return its value + return data.value + except AttributeError: + pass + + try: + if isinstance(data, unicode): # type: ignore + # Don't change it, JSON and XML ElementTree are totally able + # to serialize correctly u'' strings + return data + except NameError: + return str(data) + else: + return str(data) + + def serialize_iter(self, data, iter_type, div=None, **kwargs): + """Serialize iterable. + + Supported kwargs: + - serialization_ctxt dict : The current entry of _attribute_map, or same format. + serialization_ctxt['type'] should be same as data_type. + - is_xml bool : If set, serialize as XML + + :param list attr: Object to be serialized. + :param str iter_type: Type of object in the iterable. + :param bool required: Whether the objects in the iterable must + not be None or empty. + :param str div: If set, this str will be used to combine the elements + in the iterable into a combined string. Default is 'None'. + :keyword bool do_quote: Whether to quote the serialized result of each iterable element. + Defaults to False. + :rtype: list, str + """ + if isinstance(data, str): + raise SerializationError("Refuse str type as a valid iter type.") + + serialization_ctxt = kwargs.get("serialization_ctxt", {}) + is_xml = kwargs.get("is_xml", False) + + serialized = [] + for d in data: + try: + serialized.append(self.serialize_data(d, iter_type, **kwargs)) + except ValueError as err: + if isinstance(err, SerializationError): + raise + serialized.append(None) + + if kwargs.get('do_quote', False): + serialized = [ + '' if s is None else quote(str(s), safe='') + for s + in serialized + ] + + if div: + serialized = ["" if s is None else str(s) for s in serialized] + serialized = div.join(serialized) + + if "xml" in serialization_ctxt or is_xml: + # XML serialization is more complicated + xml_desc = serialization_ctxt.get("xml", {}) + xml_name = xml_desc.get("name") + if not xml_name: + xml_name = serialization_ctxt["key"] + + # Create a wrap node if necessary (use the fact that Element and list have "append") + is_wrapped = xml_desc.get("wrapped", False) + node_name = xml_desc.get("itemsName", xml_name) + if is_wrapped: + final_result = _create_xml_node(xml_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) + else: + final_result = [] + # All list elements to "local_node" + for el in serialized: + if isinstance(el, ET.Element): + el_node = el + else: + el_node = _create_xml_node(node_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) + if el is not None: # Otherwise it writes "None" :-p + el_node.text = str(el) + final_result.append(el_node) + return final_result + return serialized + + def serialize_dict(self, attr, dict_type, **kwargs): + """Serialize a dictionary of objects. + + :param dict attr: Object to be serialized. + :param str dict_type: Type of object in the dictionary. + :param bool required: Whether the objects in the dictionary must + not be None or empty. + :rtype: dict + """ + serialization_ctxt = kwargs.get("serialization_ctxt", {}) + serialized = {} + for key, value in attr.items(): + try: + serialized[self.serialize_unicode(key)] = self.serialize_data(value, dict_type, **kwargs) + except ValueError as err: + if isinstance(err, SerializationError): + raise + serialized[self.serialize_unicode(key)] = None + + if "xml" in serialization_ctxt: + # XML serialization is more complicated + xml_desc = serialization_ctxt["xml"] + xml_name = xml_desc["name"] + + final_result = _create_xml_node(xml_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) + for key, value in serialized.items(): + ET.SubElement(final_result, key).text = value + return final_result + + return serialized + + def serialize_object(self, attr, **kwargs): + """Serialize a generic object. + This will be handled as a dictionary. If object passed in is not + a basic type (str, int, float, dict, list) it will simply be + cast to str. + + :param dict attr: Object to be serialized. + :rtype: dict or str + """ + if attr is None: + return None + if isinstance(attr, ET.Element): + return attr + obj_type = type(attr) + if obj_type in self.basic_types: + return self.serialize_basic(attr, self.basic_types[obj_type], **kwargs) + if obj_type is _long_type: + return self.serialize_long(attr) + if obj_type is str: + return self.serialize_unicode(attr) + if obj_type is datetime.datetime: + return self.serialize_iso(attr) + if obj_type is datetime.date: + return self.serialize_date(attr) + if obj_type is datetime.time: + return self.serialize_time(attr) + if obj_type is datetime.timedelta: + return self.serialize_duration(attr) + if obj_type is decimal.Decimal: + return self.serialize_decimal(attr) + + # If it's a model or I know this dependency, serialize as a Model + elif obj_type in self.dependencies.values() or isinstance(attr, Model): + return self._serialize(attr) + + if obj_type == dict: + serialized = {} + for key, value in attr.items(): + try: + serialized[self.serialize_unicode(key)] = self.serialize_object(value, **kwargs) + except ValueError: + serialized[self.serialize_unicode(key)] = None + return serialized + + if obj_type == list: + serialized = [] + for obj in attr: + try: + serialized.append(self.serialize_object(obj, **kwargs)) + except ValueError: + pass + return serialized + return str(attr) + + @staticmethod + def serialize_enum(attr, enum_obj=None): + try: + result = attr.value + except AttributeError: + result = attr + try: + enum_obj(result) # type: ignore + return result + except ValueError: + for enum_value in enum_obj: # type: ignore + if enum_value.value.lower() == str(attr).lower(): + return enum_value.value + error = "{!r} is not valid value for enum {!r}" + raise SerializationError(error.format(attr, enum_obj)) + + @staticmethod + def serialize_bytearray(attr, **kwargs): + """Serialize bytearray into base-64 string. + + :param attr: Object to be serialized. + :rtype: str + """ + return b64encode(attr).decode() + + @staticmethod + def serialize_base64(attr, **kwargs): + """Serialize str into base-64 string. + + :param attr: Object to be serialized. + :rtype: str + """ + encoded = b64encode(attr).decode("ascii") + return encoded.strip("=").replace("+", "-").replace("/", "_") + + @staticmethod + def serialize_decimal(attr, **kwargs): + """Serialize Decimal object to float. + + :param attr: Object to be serialized. + :rtype: float + """ + return float(attr) + + @staticmethod + def serialize_long(attr, **kwargs): + """Serialize long (Py2) or int (Py3). + + :param attr: Object to be serialized. + :rtype: int/long + """ + return _long_type(attr) + + @staticmethod + def serialize_date(attr, **kwargs): + """Serialize Date object into ISO-8601 formatted string. + + :param Date attr: Object to be serialized. + :rtype: str + """ + if isinstance(attr, str): + attr = isodate.parse_date(attr) + t = "{:04}-{:02}-{:02}".format(attr.year, attr.month, attr.day) + return t + + @staticmethod + def serialize_time(attr, **kwargs): + """Serialize Time object into ISO-8601 formatted string. + + :param datetime.time attr: Object to be serialized. + :rtype: str + """ + if isinstance(attr, str): + attr = isodate.parse_time(attr) + t = "{:02}:{:02}:{:02}".format(attr.hour, attr.minute, attr.second) + if attr.microsecond: + t += ".{:02}".format(attr.microsecond) + return t + + @staticmethod + def serialize_duration(attr, **kwargs): + """Serialize TimeDelta object into ISO-8601 formatted string. + + :param TimeDelta attr: Object to be serialized. + :rtype: str + """ + if isinstance(attr, str): + attr = isodate.parse_duration(attr) + return isodate.duration_isoformat(attr) + + @staticmethod + def serialize_rfc(attr, **kwargs): + """Serialize Datetime object into RFC-1123 formatted string. + + :param Datetime attr: Object to be serialized. + :rtype: str + :raises: TypeError if format invalid. + """ + try: + if not attr.tzinfo: + _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") + utc = attr.utctimetuple() + except AttributeError: + raise TypeError("RFC1123 object must be valid Datetime object.") + + return "{}, {:02} {} {:04} {:02}:{:02}:{:02} GMT".format( + Serializer.days[utc.tm_wday], + utc.tm_mday, + Serializer.months[utc.tm_mon], + utc.tm_year, + utc.tm_hour, + utc.tm_min, + utc.tm_sec, + ) + + @staticmethod + def serialize_iso(attr, **kwargs): + """Serialize Datetime object into ISO-8601 formatted string. + + :param Datetime attr: Object to be serialized. + :rtype: str + :raises: SerializationError if format invalid. + """ + if isinstance(attr, str): + attr = isodate.parse_datetime(attr) + try: + if not attr.tzinfo: + _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") + utc = attr.utctimetuple() + if utc.tm_year > 9999 or utc.tm_year < 1: + raise OverflowError("Hit max or min date") + + microseconds = str(attr.microsecond).rjust(6, "0").rstrip("0").ljust(3, "0") + if microseconds: + microseconds = "." + microseconds + date = "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}".format( + utc.tm_year, utc.tm_mon, utc.tm_mday, utc.tm_hour, utc.tm_min, utc.tm_sec + ) + return date + microseconds + "Z" + except (ValueError, OverflowError) as err: + msg = "Unable to serialize datetime object." + raise SerializationError(msg) from err + except AttributeError as err: + msg = "ISO-8601 object must be valid Datetime object." + raise TypeError(msg) from err + + @staticmethod + def serialize_unix(attr, **kwargs): + """Serialize Datetime object into IntTime format. + This is represented as seconds. + + :param Datetime attr: Object to be serialized. + :rtype: int + :raises: SerializationError if format invalid + """ + if isinstance(attr, int): + return attr + try: + if not attr.tzinfo: + _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") + return int(calendar.timegm(attr.utctimetuple())) + except AttributeError: + raise TypeError("Unix time object must be valid Datetime object.") + + +def rest_key_extractor(attr, attr_desc, data): + key = attr_desc["key"] + working_data = data + + while "." in key: + # Need the cast, as for some reasons "split" is typed as list[str | Any] + dict_keys = cast(List[str], _FLATTEN.split(key)) + if len(dict_keys) == 1: + key = _decode_attribute_map_key(dict_keys[0]) + break + working_key = _decode_attribute_map_key(dict_keys[0]) + working_data = working_data.get(working_key, data) + if working_data is None: + # If at any point while following flatten JSON path see None, it means + # that all properties under are None as well + return None + key = ".".join(dict_keys[1:]) + + return working_data.get(key) + + +def rest_key_case_insensitive_extractor(attr, attr_desc, data): + key = attr_desc["key"] + working_data = data + + while "." in key: + dict_keys = _FLATTEN.split(key) + if len(dict_keys) == 1: + key = _decode_attribute_map_key(dict_keys[0]) + break + working_key = _decode_attribute_map_key(dict_keys[0]) + working_data = attribute_key_case_insensitive_extractor(working_key, None, working_data) + if working_data is None: + # If at any point while following flatten JSON path see None, it means + # that all properties under are None as well + return None + key = ".".join(dict_keys[1:]) + + if working_data: + return attribute_key_case_insensitive_extractor(key, None, working_data) + + +def last_rest_key_extractor(attr, attr_desc, data): + """Extract the attribute in "data" based on the last part of the JSON path key.""" + key = attr_desc["key"] + dict_keys = _FLATTEN.split(key) + return attribute_key_extractor(dict_keys[-1], None, data) + + +def last_rest_key_case_insensitive_extractor(attr, attr_desc, data): + """Extract the attribute in "data" based on the last part of the JSON path key. + + This is the case insensitive version of "last_rest_key_extractor" + """ + key = attr_desc["key"] + dict_keys = _FLATTEN.split(key) + return attribute_key_case_insensitive_extractor(dict_keys[-1], None, data) + + +def attribute_key_extractor(attr, _, data): + return data.get(attr) + + +def attribute_key_case_insensitive_extractor(attr, _, data): + found_key = None + lower_attr = attr.lower() + for key in data: + if lower_attr == key.lower(): + found_key = key + break + + return data.get(found_key) + + +def _extract_name_from_internal_type(internal_type): + """Given an internal type XML description, extract correct XML name with namespace. + + :param dict internal_type: An model type + :rtype: tuple + :returns: A tuple XML name + namespace dict + """ + internal_type_xml_map = getattr(internal_type, "_xml_map", {}) + xml_name = internal_type_xml_map.get("name", internal_type.__name__) + xml_ns = internal_type_xml_map.get("ns", None) + if xml_ns: + xml_name = "{% raw %}{{{}}}{}{% endraw %}".format(xml_ns, xml_name) + return xml_name + + +def xml_key_extractor(attr, attr_desc, data): + if isinstance(data, dict): + return None + + # Test if this model is XML ready first + if not isinstance(data, ET.Element): + return None + + xml_desc = attr_desc.get("xml", {}) + xml_name = xml_desc.get("name", attr_desc["key"]) + + # Look for a children + is_iter_type = attr_desc["type"].startswith("[") + is_wrapped = xml_desc.get("wrapped", False) + internal_type = attr_desc.get("internalType", None) + internal_type_xml_map = getattr(internal_type, "_xml_map", {}) + + # Integrate namespace if necessary + xml_ns = xml_desc.get("ns", internal_type_xml_map.get("ns", None)) + if xml_ns: + xml_name = "{% raw %}{{{}}}{}{% endraw %}".format(xml_ns, xml_name) + + # If it's an attribute, that's simple + if xml_desc.get("attr", False): + return data.get(xml_name) + + # If it's x-ms-text, that's simple too + if xml_desc.get("text", False): + return data.text + + # Scenario where I take the local name: + # - Wrapped node + # - Internal type is an enum (considered basic types) + # - Internal type has no XML/Name node + if is_wrapped or (internal_type and (issubclass(internal_type, Enum) or "name" not in internal_type_xml_map)): + children = data.findall(xml_name) + # If internal type has a local name and it's not a list, I use that name + elif not is_iter_type and internal_type and "name" in internal_type_xml_map: + xml_name = _extract_name_from_internal_type(internal_type) + children = data.findall(xml_name) + # That's an array + else: + if internal_type: # Complex type, ignore itemsName and use the complex type name + items_name = _extract_name_from_internal_type(internal_type) + else: + items_name = xml_desc.get("itemsName", xml_name) + children = data.findall(items_name) + + if len(children) == 0: + if is_iter_type: + if is_wrapped: + return None # is_wrapped no node, we want None + else: + return [] # not wrapped, assume empty list + return None # Assume it's not there, maybe an optional node. + + # If is_iter_type and not wrapped, return all found children + if is_iter_type: + if not is_wrapped: + return children + else: # Iter and wrapped, should have found one node only (the wrap one) + if len(children) != 1: + raise DeserializationError( + "Tried to deserialize an array not wrapped, and found several nodes '{}'. Maybe you should declare this array as wrapped?".format( + xml_name + ) + ) + return list(children[0]) # Might be empty list and that's ok. + + # Here it's not a itertype, we should have found one element only or empty + if len(children) > 1: + raise DeserializationError("Find several XML '{}' where it was not expected".format(xml_name)) + return children[0] + + +class Deserializer(object): + """Response object model deserializer. + + :param dict classes: Class type dictionary for deserializing complex types. + :ivar list key_extractors: Ordered list of extractors to be used by this deserializer. + """ + + basic_types = {str: "str", int: "int", bool: "bool", float: "float"} + + valid_date = re.compile(r"\d{4}[-]\d{2}[-]\d{2}T\d{2}:\d{2}:\d{2}" r"\.?\d*Z?[-+]?[\d{2}]?:?[\d{2}]?") + + def __init__(self, classes: Optional[Mapping[str, type]]=None): + self.deserialize_type = { + "iso-8601": Deserializer.deserialize_iso, + "rfc-1123": Deserializer.deserialize_rfc, + "unix-time": Deserializer.deserialize_unix, + "duration": Deserializer.deserialize_duration, + "date": Deserializer.deserialize_date, + "time": Deserializer.deserialize_time, + "decimal": Deserializer.deserialize_decimal, + "long": Deserializer.deserialize_long, + "bytearray": Deserializer.deserialize_bytearray, + "base64": Deserializer.deserialize_base64, + "object": self.deserialize_object, + "[]": self.deserialize_iter, + "{}": self.deserialize_dict, + } + self.deserialize_expected_types = { + "duration": (isodate.Duration, datetime.timedelta), + "iso-8601": (datetime.datetime), + } + self.dependencies: Dict[str, type] = dict(classes) if classes else {} + self.key_extractors = [rest_key_extractor, xml_key_extractor] + # Additional properties only works if the "rest_key_extractor" is used to + # extract the keys. Making it to work whatever the key extractor is too much + # complicated, with no real scenario for now. + # So adding a flag to disable additional properties detection. This flag should be + # used if your expect the deserialization to NOT come from a JSON REST syntax. + # Otherwise, result are unexpected + self.additional_properties_detection = True + + def __call__(self, target_obj, response_data, content_type=None): + """Call the deserializer to process a REST response. + + :param str target_obj: Target data type to deserialize to. + :param requests.Response response_data: REST response object. + :param str content_type: Swagger "produces" if available. + :raises: DeserializationError if deserialization fails. + :return: Deserialized object. + """ + data = self._unpack_content(response_data, content_type) + return self._deserialize(target_obj, data) + + def _deserialize(self, target_obj, data): + """Call the deserializer on a model. + + Data needs to be already deserialized as JSON or XML ElementTree + + :param str target_obj: Target data type to deserialize to. + :param object data: Object to deserialize. + :raises: DeserializationError if deserialization fails. + :return: Deserialized object. + """ + # This is already a model, go recursive just in case + if hasattr(data, "_attribute_map"): + constants = [name for name, config in getattr(data, "_validation", {}).items() if config.get("constant")] + try: + for attr, mapconfig in data._attribute_map.items(): + if attr in constants: + continue + value = getattr(data, attr) + if value is None: + continue + local_type = mapconfig["type"] + internal_data_type = local_type.strip("[]{}") + if internal_data_type not in self.dependencies or isinstance(internal_data_type, Enum): + continue + setattr(data, attr, self._deserialize(local_type, value)) + return data + except AttributeError: + return + + response, class_name = self._classify_target(target_obj, data) + + if isinstance(response, str): + return self.deserialize_data(data, response) + elif isinstance(response, type) and issubclass(response, Enum): + return self.deserialize_enum(data, response) + + if data is None or data is CoreNull: + return data + try: + attributes = response._attribute_map # type: ignore + d_attrs = {} + for attr, attr_desc in attributes.items(): + # Check empty string. If it's not empty, someone has a real "additionalProperties"... + if attr == "additional_properties" and attr_desc["key"] == "": + continue + raw_value = None + # Enhance attr_desc with some dynamic data + attr_desc = attr_desc.copy() # Do a copy, do not change the real one + internal_data_type = attr_desc["type"].strip("[]{}") + if internal_data_type in self.dependencies: + attr_desc["internalType"] = self.dependencies[internal_data_type] + + for key_extractor in self.key_extractors: + found_value = key_extractor(attr, attr_desc, data) + if found_value is not None: + if raw_value is not None and raw_value != found_value: + msg = ( + "Ignoring extracted value '%s' from %s for key '%s'" + " (duplicate extraction, follow extractors order)" + ) + _LOGGER.warning(msg, found_value, key_extractor, attr) + continue + raw_value = found_value + + value = self.deserialize_data(raw_value, attr_desc["type"]) + d_attrs[attr] = value + except (AttributeError, TypeError, KeyError) as err: + msg = "Unable to deserialize to object: " + class_name # type: ignore + raise DeserializationError(msg) from err + else: + additional_properties = self._build_additional_properties(attributes, data) + return self._instantiate_model(response, d_attrs, additional_properties) + + def _build_additional_properties(self, attribute_map, data): + if not self.additional_properties_detection: + return None + if "additional_properties" in attribute_map and attribute_map.get("additional_properties", {}).get("key") != "": + # Check empty string. If it's not empty, someone has a real "additionalProperties" + return None + if isinstance(data, ET.Element): + data = {el.tag: el.text for el in data} + + known_keys = { + _decode_attribute_map_key(_FLATTEN.split(desc["key"])[0]) + for desc in attribute_map.values() + if desc["key"] != "" + } + present_keys = set(data.keys()) + missing_keys = present_keys - known_keys + return {key: data[key] for key in missing_keys} + + def _classify_target(self, target, data): + """Check to see whether the deserialization target object can + be classified into a subclass. + Once classification has been determined, initialize object. + + :param str target: The target object type to deserialize to. + :param str/dict data: The response data to deserialize. + """ + if target is None: + return None, None + + if isinstance(target, str): + try: + target = self.dependencies[target] + except KeyError: + return target, target + + try: + target = target._classify(data, self.dependencies) # type: ignore + except AttributeError: + pass # Target is not a Model, no classify + return target, target.__class__.__name__ # type: ignore + + def failsafe_deserialize(self, target_obj, data, content_type=None): + """Ignores any errors encountered in deserialization, + and falls back to not deserializing the object. Recommended + for use in error deserialization, as we want to return the + HttpResponseError to users, and not have them deal with + a deserialization error. + + :param str target_obj: The target object type to deserialize to. + :param str/dict data: The response data to deserialize. + :param str content_type: Swagger "produces" if available. + """ + try: + return self(target_obj, data, content_type=content_type) + except: + _LOGGER.debug( + "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True + ) + return None + + @staticmethod + def _unpack_content(raw_data, content_type=None): + """Extract the correct structure for deserialization. + + If raw_data is a PipelineResponse, try to extract the result of RawDeserializer. + if we can't, raise. Your Pipeline should have a RawDeserializer. + + If not a pipeline response and raw_data is bytes or string, use content-type + to decode it. If no content-type, try JSON. + + If raw_data is something else, bypass all logic and return it directly. + + :param raw_data: Data to be processed. + :param content_type: How to parse if raw_data is a string/bytes. + :raises JSONDecodeError: If JSON is requested and parsing is impossible. + :raises UnicodeDecodeError: If bytes is not UTF8 + """ + # Assume this is enough to detect a Pipeline Response without importing it + context = getattr(raw_data, "context", {}) + if context: + if RawDeserializer.CONTEXT_NAME in context: + return context[RawDeserializer.CONTEXT_NAME] + raise ValueError("This pipeline didn't have the RawDeserializer policy; can't deserialize") + + # Assume this is enough to recognize universal_http.ClientResponse without importing it + if hasattr(raw_data, "body"): + return RawDeserializer.deserialize_from_http_generics(raw_data.text(), raw_data.headers) + + # Assume this enough to recognize requests.Response without importing it. + if hasattr(raw_data, "_content_consumed"): + return RawDeserializer.deserialize_from_http_generics(raw_data.text, raw_data.headers) + + if isinstance(raw_data, (str, bytes)) or hasattr(raw_data, "read"): + return RawDeserializer.deserialize_from_text(raw_data, content_type) # type: ignore + return raw_data + + def _instantiate_model(self, response, attrs, additional_properties=None): + """Instantiate a response model passing in deserialized args. + + :param response: The response model class. + :param d_attrs: The deserialized response attributes. + """ + if callable(response): + subtype = getattr(response, "_subtype_map", {}) + try: + readonly = [k for k, v in response._validation.items() if v.get("readonly")] + const = [k for k, v in response._validation.items() if v.get("constant")] + kwargs = {k: v for k, v in attrs.items() if k not in subtype and k not in readonly + const} + response_obj = response(**kwargs) + for attr in readonly: + setattr(response_obj, attr, attrs.get(attr)) + if additional_properties: + response_obj.additional_properties = additional_properties + return response_obj + except TypeError as err: + msg = "Unable to deserialize {} into model {}. ".format(kwargs, response) # type: ignore + raise DeserializationError(msg + str(err)) + else: + try: + for attr, value in attrs.items(): + setattr(response, attr, value) + return response + except Exception as exp: + msg = "Unable to populate response model. " + msg += "Type: {}, Error: {}".format(type(response), exp) + raise DeserializationError(msg) + + def deserialize_data(self, data, data_type): + """Process data for deserialization according to data type. + + :param str data: The response string to be deserialized. + :param str data_type: The type to deserialize to. + :raises: DeserializationError if deserialization fails. + :return: Deserialized object. + """ + if data is None: + return data + + try: + if not data_type: + return data + if data_type in self.basic_types.values(): + return self.deserialize_basic(data, data_type) + if data_type in self.deserialize_type: + if isinstance(data, self.deserialize_expected_types.get(data_type, tuple())): + return data + + is_a_text_parsing_type = lambda x: x not in ["object", "[]", r"{}"] + if isinstance(data, ET.Element) and is_a_text_parsing_type(data_type) and not data.text: + return None + data_val = self.deserialize_type[data_type](data) + return data_val + + iter_type = data_type[0] + data_type[-1] + if iter_type in self.deserialize_type: + return self.deserialize_type[iter_type](data, data_type[1:-1]) + + obj_type = self.dependencies[data_type] + if issubclass(obj_type, Enum): + if isinstance(data, ET.Element): + data = data.text + return self.deserialize_enum(data, obj_type) + + except (ValueError, TypeError, AttributeError) as err: + msg = "Unable to deserialize response data." + msg += " Data: {}, {}".format(data, data_type) + raise DeserializationError(msg) from err + else: + return self._deserialize(obj_type, data) + + def deserialize_iter(self, attr, iter_type): + """Deserialize an iterable. + + :param list attr: Iterable to be deserialized. + :param str iter_type: The type of object in the iterable. + :rtype: list + """ + if attr is None: + return None + if isinstance(attr, ET.Element): # If I receive an element here, get the children + attr = list(attr) + if not isinstance(attr, (list, set)): + raise DeserializationError("Cannot deserialize as [{}] an object of type {}".format(iter_type, type(attr))) + return [self.deserialize_data(a, iter_type) for a in attr] + + def deserialize_dict(self, attr, dict_type): + """Deserialize a dictionary. + + :param dict/list attr: Dictionary to be deserialized. Also accepts + a list of key, value pairs. + :param str dict_type: The object type of the items in the dictionary. + :rtype: dict + """ + if isinstance(attr, list): + return {x["key"]: self.deserialize_data(x["value"], dict_type) for x in attr} + + if isinstance(attr, ET.Element): + # Transform value into {"Key": "value"} + attr = {el.tag: el.text for el in attr} + return {k: self.deserialize_data(v, dict_type) for k, v in attr.items()} + + def deserialize_object(self, attr, **kwargs): + """Deserialize a generic object. + This will be handled as a dictionary. + + :param dict attr: Dictionary to be deserialized. + :rtype: dict + :raises: TypeError if non-builtin datatype encountered. + """ + if attr is None: + return None + if isinstance(attr, ET.Element): + # Do no recurse on XML, just return the tree as-is + return attr + if isinstance(attr, str): + return self.deserialize_basic(attr, "str") + obj_type = type(attr) + if obj_type in self.basic_types: + return self.deserialize_basic(attr, self.basic_types[obj_type]) + if obj_type is _long_type: + return self.deserialize_long(attr) + + if obj_type == dict: + deserialized = {} + for key, value in attr.items(): + try: + deserialized[key] = self.deserialize_object(value, **kwargs) + except ValueError: + deserialized[key] = None + return deserialized + + if obj_type == list: + deserialized = [] + for obj in attr: + try: + deserialized.append(self.deserialize_object(obj, **kwargs)) + except ValueError: + pass + return deserialized + + else: + error = "Cannot deserialize generic object with type: " + raise TypeError(error + str(obj_type)) + + def deserialize_basic(self, attr, data_type): + """Deserialize basic builtin data type from string. + Will attempt to convert to str, int, float and bool. + This function will also accept '1', '0', 'true' and 'false' as + valid bool values. + + :param str attr: response string to be deserialized. + :param str data_type: deserialization data type. + :rtype: str, int, float or bool + :raises: TypeError if string format is not valid. + """ + # If we're here, data is supposed to be a basic type. + # If it's still an XML node, take the text + if isinstance(attr, ET.Element): + attr = attr.text + if not attr: + if data_type == "str": + # None or '', node is empty string. + return "" + else: + # None or '', node with a strong type is None. + # Don't try to model "empty bool" or "empty int" + return None + + if data_type == "bool": + if attr in [True, False, 1, 0]: + return bool(attr) + elif isinstance(attr, str): + if attr.lower() in ["true", "1"]: + return True + elif attr.lower() in ["false", "0"]: + return False + raise TypeError("Invalid boolean value: {}".format(attr)) + + if data_type == "str": + return self.deserialize_unicode(attr) + return eval(data_type)(attr) # nosec + + @staticmethod + def deserialize_unicode(data): + """Preserve unicode objects in Python 2, otherwise return data + as a string. + + :param str data: response string to be deserialized. + :rtype: str or unicode + """ + # We might be here because we have an enum modeled as string, + # and we try to deserialize a partial dict with enum inside + if isinstance(data, Enum): + return data + + # Consider this is real string + try: + if isinstance(data, unicode): # type: ignore + return data + except NameError: + return str(data) + else: + return str(data) + + @staticmethod + def deserialize_enum(data, enum_obj): + """Deserialize string into enum object. + + If the string is not a valid enum value it will be returned as-is + and a warning will be logged. + + :param str data: Response string to be deserialized. If this value is + None or invalid it will be returned as-is. + :param Enum enum_obj: Enum object to deserialize to. + :rtype: Enum + """ + if isinstance(data, enum_obj) or data is None: + return data + if isinstance(data, Enum): + data = data.value + if isinstance(data, int): + # Workaround. We might consider remove it in the future. + try: + return list(enum_obj.__members__.values())[data] + except IndexError: + error = "{!r} is not a valid index for enum {!r}" + raise DeserializationError(error.format(data, enum_obj)) + try: + return enum_obj(str(data)) + except ValueError: + for enum_value in enum_obj: + if enum_value.value.lower() == str(data).lower(): + return enum_value + # We don't fail anymore for unknown value, we deserialize as a string + _LOGGER.warning("Deserializer is not able to find %s as valid enum in %s", data, enum_obj) + return Deserializer.deserialize_unicode(data) + + @staticmethod + def deserialize_bytearray(attr): + """Deserialize string into bytearray. + + :param str attr: response string to be deserialized. + :rtype: bytearray + :raises: TypeError if string format invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + return bytearray(b64decode(attr)) # type: ignore + + @staticmethod + def deserialize_base64(attr): + """Deserialize base64 encoded string into string. + + :param str attr: response string to be deserialized. + :rtype: bytearray + :raises: TypeError if string format invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + padding = "=" * (3 - (len(attr) + 3) % 4) # type: ignore + attr = attr + padding # type: ignore + encoded = attr.replace("-", "+").replace("_", "/") + return b64decode(encoded) + + @staticmethod + def deserialize_decimal(attr): + """Deserialize string into Decimal object. + + :param str attr: response string to be deserialized. + :rtype: Decimal + :raises: DeserializationError if string format invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + try: + return decimal.Decimal(str(attr)) # type: ignore + except decimal.DecimalException as err: + msg = "Invalid decimal {}".format(attr) + raise DeserializationError(msg) from err + + @staticmethod + def deserialize_long(attr): + """Deserialize string into long (Py2) or int (Py3). + + :param str attr: response string to be deserialized. + :rtype: long or int + :raises: ValueError if string format invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + return _long_type(attr) # type: ignore + + @staticmethod + def deserialize_duration(attr): + """Deserialize ISO-8601 formatted string into TimeDelta object. + + :param str attr: response string to be deserialized. + :rtype: TimeDelta + :raises: DeserializationError if string format invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + try: + duration = isodate.parse_duration(attr) + except (ValueError, OverflowError, AttributeError) as err: + msg = "Cannot deserialize duration object." + raise DeserializationError(msg) from err + else: + return duration + + @staticmethod + def deserialize_date(attr): + """Deserialize ISO-8601 formatted string into Date object. + + :param str attr: response string to be deserialized. + :rtype: Date + :raises: DeserializationError if string format invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + if re.search(r"[^\W\d_]", attr, re.I + re.U): # type: ignore + raise DeserializationError("Date must have only digits and -. Received: %s" % attr) + # This must NOT use defaultmonth/defaultday. Using None ensure this raises an exception. + return isodate.parse_date(attr, defaultmonth=0, defaultday=0) + + @staticmethod + def deserialize_time(attr): + """Deserialize ISO-8601 formatted string into time object. + + :param str attr: response string to be deserialized. + :rtype: datetime.time + :raises: DeserializationError if string format invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + if re.search(r"[^\W\d_]", attr, re.I + re.U): # type: ignore + raise DeserializationError("Date must have only digits and -. Received: %s" % attr) + return isodate.parse_time(attr) + + @staticmethod + def deserialize_rfc(attr): + """Deserialize RFC-1123 formatted string into Datetime object. + + :param str attr: response string to be deserialized. + :rtype: Datetime + :raises: DeserializationError if string format invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + try: + parsed_date = email.utils.parsedate_tz(attr) # type: ignore + date_obj = datetime.datetime( + *parsed_date[:6], tzinfo=_FixedOffset(datetime.timedelta(minutes=(parsed_date[9] or 0) / 60)) + ) + if not date_obj.tzinfo: + date_obj = date_obj.astimezone(tz=TZ_UTC) + except ValueError as err: + msg = "Cannot deserialize to rfc datetime object." + raise DeserializationError(msg) from err + else: + return date_obj + + @staticmethod + def deserialize_iso(attr): + """Deserialize ISO-8601 formatted string into Datetime object. + + :param str attr: response string to be deserialized. + :rtype: Datetime + :raises: DeserializationError if string format invalid. + """ + if isinstance(attr, ET.Element): + attr = attr.text + try: + attr = attr.upper() # type: ignore + match = Deserializer.valid_date.match(attr) + if not match: + raise ValueError("Invalid datetime string: " + attr) + + check_decimal = attr.split(".") + if len(check_decimal) > 1: + decimal_str = "" + for digit in check_decimal[1]: + if digit.isdigit(): + decimal_str += digit + else: + break + if len(decimal_str) > 6: + attr = attr.replace(decimal_str, decimal_str[0:6]) + + date_obj = isodate.parse_datetime(attr) + test_utc = date_obj.utctimetuple() + if test_utc.tm_year > 9999 or test_utc.tm_year < 1: + raise OverflowError("Hit max or min date") + except (ValueError, OverflowError, AttributeError) as err: + msg = "Cannot deserialize datetime object." + raise DeserializationError(msg) from err + else: + return date_obj + + @staticmethod + def deserialize_unix(attr): + """Serialize Datetime object into IntTime format. + This is represented as seconds. + + :param int attr: Object to be serialized. + :rtype: Datetime + :raises: DeserializationError if format invalid + """ + if isinstance(attr, ET.Element): + attr = int(attr.text) # type: ignore + try: + attr = int(attr) + date_obj = datetime.datetime.fromtimestamp(attr, TZ_UTC) + except ValueError as err: + msg = "Cannot deserialize to unix datetime object." + raise DeserializationError(msg) from err + else: + return date_obj diff --git a/packages/autorest.python/generator/pygen/codegen/templates/test.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/test.py.jinja2 new file mode 100644 index 00000000000..7664fa914c6 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/test.py.jinja2 @@ -0,0 +1,26 @@ +{% set prefix_lower = test.prefix|lower %} +{% set async = "async " if test.is_async else "" %} +{% set async_suffix = "_async" if test.is_async else "" %} +# coding=utf-8 +{{ code_model.options['license_header'] }} +import pytest +{{ imports }} + + +@pytest.mark.skip("you may need to update the auto-generated test case before run it") +class {{ test.test_class_name }}({{ test.base_test_class_name }}): +{% for testcase in test.testcases %} + @{{ test.preparer_name }}() + @recorded_by_proxy{{ async_suffix }} + {{ async }}def test_{{ testcase.operation.name }}(self, {{ prefix_lower }}_endpoint): + client = self.{{ test.create_client_name }}(endpoint={{ prefix_lower }}_endpoint) + {{testcase.response }}client{{ testcase.operation_group_prefix }}.{{ testcase.operation.name }}( + {% for key, value in testcase.params.items() %} + {{ key }}={{ value|indent(12) }}, + {% endfor %} + ){{ testcase.operation_suffix }} + {{ testcase.extra_operation }} + # please add some check logic here by yourself + # ... + +{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/testpreparer.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/testpreparer.py.jinja2 new file mode 100644 index 00000000000..b3b15f37276 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/testpreparer.py.jinja2 @@ -0,0 +1,26 @@ +# coding=utf-8 +{{ code_model.options['license_header'] }} +{{ imports }} + +{% for test_name in test_names %} +{% set extra_async = ", is_async=True" if test_name.is_async else ""%} +{% set prefix_lower = test_name.prefix|lower %} +class {{ test_name.base_test_class_name }}(AzureRecordedTestCase): + + def {{ test_name.create_client_name }}(self, endpoint): + credential = self.get_credential({{ test_name.client_name }}{{ extra_async }}) + return self.create_client_from_credential( + {{ test_name.client_name }}, + credential=credential, + endpoint=endpoint, + ) + +{% if not test_name.is_async %} +{{ test_name.preparer_name }} = functools.partial( + PowerShellPreparer, + "{{ prefix_lower }}", + {{ prefix_lower }}_endpoint="https://fake_{{ prefix_lower }}_endpoint.com" +) +{% endif %} + +{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/types.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/types.py.jinja2 new file mode 100644 index 00000000000..2dc930871fc --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/types.py.jinja2 @@ -0,0 +1,8 @@ +# coding=utf-8 +# pylint: disable=too-many-lines +{{ code_model.options['license_header'] }} + +{{ imports }} +{% for nu in code_model.named_unions %} +{{nu.name}} = {{nu.type_definition()}} +{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/validation.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/validation.py.jinja2 new file mode 100644 index 00000000000..ebc4b243881 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/validation.py.jinja2 @@ -0,0 +1,38 @@ +{{ code_model.options['license_header'] }} +import functools + +def api_version_validation(**kwargs): + params_added_on = kwargs.pop("params_added_on", {}) + method_added_on = kwargs.pop("method_added_on", "") + + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + # this assumes the client has an _api_version attribute + client = args[0] + client_api_version = client._config.api_version # pylint: disable=protected-access + except AttributeError: + return func(*args, **kwargs) + + if method_added_on > client_api_version: + raise ValueError( + f"'{func.__name__}' is not available in API version " + f"{client_api_version}. Pass service API version {method_added_on} or newer to your client." + ) + + unsupported = { + parameter: api_version + for api_version, parameters in params_added_on.items() + for parameter in parameters + if parameter in kwargs and api_version > client_api_version + } + if unsupported: + raise ValueError("".join([ + f"'{param}' is not available in API version {client_api_version}. " + f"Use service API version {version} or newer.\n" + for param, version in unsupported.items() + ])) + return func(*args, **kwargs) + return wrapper + return decorator diff --git a/packages/autorest.python/generator/pygen/codegen/templates/vendor.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/vendor.py.jinja2 new file mode 100644 index 00000000000..bc15b14dd56 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/vendor.py.jinja2 @@ -0,0 +1,98 @@ +{% import 'keywords.jinja2' as keywords with context %} +{{ code_model.options['license_header'] }} + +{{ imports }} + +{% if code_model.need_mixin_abc %} + {% for client in clients | selectattr("has_mixin") %} +{% set pylint_disable = "# pylint: disable=name-too-long" if (client.name | length) + ("MixinABC" | length) > 40 else "" %} +class {{ client.name }}MixinABC( {{ pylint_disable }} + ABC +): + """DO NOT use this class. It is for internal typing use only.""" + _client: "{{ keywords.async_class }}PipelineClient" + _config: {{ client.name }}Configuration + _serialize: "Serializer" + _deserialize: "Deserializer" + {% endfor %} +{% endif %} +{% if code_model.has_abstract_operations %} + +def raise_if_not_implemented(cls, abstract_methods): + not_implemented = [f for f in abstract_methods if not callable(getattr(cls, f, None))] + if not_implemented: + raise NotImplementedError("The following methods on operation group '{}' are not implemented: '{}'." + " Please refer to https://aka.ms/azsdk/python/dpcodegen/python/customize to learn how to customize.".format( + cls.__name__, '\', \''.join(not_implemented)) + ) +{% endif %} + +{% if code_model.has_etag %} +def quote_etag(etag: Optional[str]) -> Optional[str]: + if not etag or etag == "*": + return etag + if etag.startswith("W/"): + return etag + if etag.startswith('"') and etag.endswith('"'): + return etag + if etag.startswith("'") and etag.endswith("'"): + return etag + return '"' + etag + '"' + + +def prep_if_match(etag: Optional[str], match_condition: Optional[MatchConditions]) -> Optional[str]: + if match_condition == MatchConditions.IfNotModified: + if_match = quote_etag(etag) if etag else None + return if_match + if match_condition == MatchConditions.IfPresent: + return "*" + return None + + +def prep_if_none_match(etag: Optional[str], match_condition: Optional[MatchConditions]) -> Optional[str]: + if match_condition == MatchConditions.IfModified: + if_none_match = quote_etag(etag) if etag else None + return if_none_match + if match_condition == MatchConditions.IfMissing: + return "*" + return None +{% endif %} +{% if code_model.has_form_data and code_model.options["models_mode"] == "dpg" and not async_mode %} +# file-like tuple could be `(filename, IO (or bytes))` or `(filename, IO (or bytes), content_type)` +FileContent = Union[str, bytes, IO[str], IO[bytes]] + +FileType = Union[ + # file (or bytes) + FileContent, + # (filename, file (or bytes)) + Tuple[Optional[str], FileContent], + # (filename, file (or bytes), content_type) + Tuple[Optional[str], FileContent, Optional[str]], +] + +FilesType = Union[Mapping[str, FileType], Sequence[Tuple[str, FileType]]] + +def serialize_multipart_data_entry(data_entry: Any) -> Any: + if isinstance(data_entry, (list, tuple, dict, Model)): + return json.dumps(data_entry, cls=SdkJSONEncoder, exclude_readonly=True) + return data_entry + +def prepare_multipart_form_data( + body: Mapping[str, Any], multipart_fields: List[str], data_fields: List[str] +) -> Tuple[List[FileType], Dict[str, Any]]: + files: List[FileType] = [] + data: Dict[str, Any] = {} + for multipart_field in multipart_fields: + multipart_entry = body.get(multipart_field) + if isinstance(multipart_entry, list): + files.extend([(multipart_field, e) for e in multipart_entry ]) + elif multipart_entry: + files.append((multipart_field, multipart_entry)) + + for data_field in data_fields: + data_entry = body.get(data_field) + if data_entry: + data[data_field] = serialize_multipart_data_entry(data_entry) + + return files, data +{% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/version.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/version.py.jinja2 new file mode 100644 index 00000000000..2ea06fccbe3 --- /dev/null +++ b/packages/autorest.python/generator/pygen/codegen/templates/version.py.jinja2 @@ -0,0 +1,4 @@ +# coding=utf-8 +{{ code_model.options['license_header'] }} + +VERSION = "{{ code_model.options['package_version'] }}" diff --git a/packages/typespec-python/generator/pygen/m2r/__init__.py b/packages/autorest.python/generator/pygen/m2r/__init__.py similarity index 100% rename from packages/typespec-python/generator/pygen/m2r/__init__.py rename to packages/autorest.python/generator/pygen/m2r/__init__.py diff --git a/packages/autorest.python/generator/pygen/postprocess/__init__.py b/packages/autorest.python/generator/pygen/postprocess/__init__.py new file mode 100644 index 00000000000..771177261cb --- /dev/null +++ b/packages/autorest.python/generator/pygen/postprocess/__init__.py @@ -0,0 +1,183 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Tuple, Any, Dict +from pathlib import Path +import os +import shutil +from venv import EnvBuilder +import black +from black.report import NothingChanged +from .venvtools import ExtendedEnvBuilder, python_run + +from .. import Plugin + +_BLACK_MODE = black.Mode() # pyright: ignore [reportPrivateImportUsage] +_BLACK_MODE.line_length = 120 + + +def format_file(file: Path, file_content: str) -> str: + if not file.suffix == ".py": + return file_content + try: + file_content = black.format_file_contents(file_content, fast=True, mode=_BLACK_MODE) + except NothingChanged: + pass + return file_content + + +class PostProcessPlugin(Plugin): # pylint: disable=abstract-method + def __init__(self, **kwargs: Any): + super().__init__(**kwargs) + output_folder_uri = self.options["outputFolderUri"] + if output_folder_uri.startswith("file:"): + output_folder_uri = output_folder_uri[5:] + if os.name == "nt" and output_folder_uri.startswith("///"): + output_folder_uri = output_folder_uri[3:] + self.output_folder = Path(output_folder_uri) # path to where the setup.py is + self.setup_venv() + + # set up the venv + # base folder is where the code starts, i.e. where we + self.base_folder, self.namespace = self.get_namespace(self.output_folder, "") + + def setup_venv(self): + venv_path = self.output_folder / Path(".temp_folder") / Path("temp_venv") + + if venv_path.exists(): + env_builder = EnvBuilder(with_pip=True) + self.venv_context = env_builder.ensure_directories(venv_path) + else: + env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True) + env_builder.create(venv_path) + self.venv_context = env_builder.context + python_run( + self.venv_context, + "pip", + ["install", "-e", str(self.output_folder)], + directory=self.output_folder, + ) + + def get_namespace(self, dir: Path, namespace: str) -> Tuple[Path, str]: + try: + init_file = next(d for d in dir.iterdir() if d.name == "__init__.py") + # we don't care about pkgutil inits, we skip over them + file_content = self.read_file(init_file.relative_to(self.output_folder)) + if "pkgutil" not in file_content: + return dir, namespace + except StopIteration: + pass + + try: + # first, see if we can get a folder that has the same name as the current output folder + start = self.output_folder.stem.split("-")[0] + next_dir = next(d for d in dir.iterdir() if d.is_dir() and d.name == start) + except StopIteration: + invalid_start_chars = [".", "_"] + invalid_dirs = [ + "swagger", + "out", + "tests", + "samples", + ] + + next_dir = next( + d + for d in dir.iterdir() + if d.is_dir() + and not str(d).endswith("egg-info") + and d.name[0] not in invalid_start_chars + and d.name not in invalid_dirs + ) + + namespace = f"{namespace}.{next_dir.name}" if namespace else next_dir.name + return self.get_namespace(next_dir, namespace) + + def process(self) -> bool: + folders = [f for f in self.base_folder.glob("**/*") if f.is_dir() and not f.stem.startswith("__")] + # will always have the root + self.fix_imports_in_init( + generated_file_name="_client", + folder_path=self.base_folder, + namespace=self.namespace, + ) + try: + aio_folder = next(f for f in folders if f.stem == "aio") + self.fix_imports_in_init( + generated_file_name="_client", + folder_path=aio_folder, + namespace=f"{self.namespace}.aio", + ) + except StopIteration: + pass + + try: + models_folder = next(f for f in folders if f.stem == "models") + self.fix_imports_in_init( + generated_file_name="_models", + folder_path=models_folder, + namespace=f"{self.namespace}.models", + ) + except StopIteration: + pass + operations_folders = [f for f in folders if f.stem in ["operations", "_operations"]] + for operations_folder in operations_folders: + sub_namespace = ".".join(str(operations_folder.relative_to(self.base_folder)).split(os.sep)) + self.fix_imports_in_init( + generated_file_name="_operations", + folder_path=operations_folder, + namespace=f"{self.namespace}.{sub_namespace}", + ) + shutil.rmtree(f"{str(self.output_folder)}/.temp_folder") + return True + + def fix_imports_in_init(self, generated_file_name: str, folder_path: Path, namespace: str) -> None: + customized_objects_str = python_run( + self.venv_context, + command=[namespace, str(self.output_folder)], + module="get_all", + ) + + if not customized_objects_str: + return + customized_objects = {k: None for k in customized_objects_str.split(",")}.keys() # filter out duplicates + file = (folder_path / "__init__.py").relative_to(self.output_folder) + file_content = self.read_file(file).replace("\r\n", "\n") + added_objs = [] + for obj in customized_objects: + if f" import {obj}\n" in file_content: + # means we're overriding a generated model + file_content = file_content.replace( + f"from .{generated_file_name} import {obj}\n", + f"from ._patch import {obj}\n", + ) + else: + added_objs.append(obj) + file_content = file_content.replace( + "try:\n from ._patch import __all__ as _patch_all\n " + "from ._patch import * # pylint: disable=unused-wildcard-import" + "\nexcept ImportError:\n _patch_all = []", + "", + ) + file_content = file_content.replace("from ._patch import __all__ as _patch_all", "") + file_content = file_content.replace( + "from ._patch import * # pylint: disable=unused-wildcard-import\n", + "", + ) + file_content = file_content.replace("__all__.extend([p for p in _patch_all if p not in __all__])", "") + if added_objs: + # add import + patch_sdk_import = "from ._patch import patch_sdk as _patch_sdk" + imports = "\n".join([f"from ._patch import {obj}" for obj in added_objs]) + if imports: + replacement = f"{imports}\n{patch_sdk_import}" + else: + replacement = patch_sdk_import + file_content = file_content.replace(patch_sdk_import, replacement) + # add to __all__ + added_objs_all = "\n".join([f' "{obj}",' for obj in added_objs]) + "\n" + file_content = file_content.replace("__all__ = [", f"__all__ = [\n{added_objs_all}", 1) + formatted_file = format_file(file, file_content) + self.write_file(file, formatted_file) diff --git a/packages/autorest.python/generator/pygen/postprocess/get_all.py b/packages/autorest.python/generator/pygen/postprocess/get_all.py new file mode 100644 index 00000000000..4206e36a9c4 --- /dev/null +++ b/packages/autorest.python/generator/pygen/postprocess/get_all.py @@ -0,0 +1,19 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys +import importlib + + +def main(namespace): + sdk = importlib.import_module(namespace) + return sdk._patch.__all__ # pylint: disable=protected-access + + +if __name__ == "__main__": + patched = ",".join(main(sys.argv[1])) + output_folder = sys.argv[2] + with open(f"{output_folder}/.temp_folder/patched.txt", "w", encoding="utf-8-sig") as f: + f.write(patched) diff --git a/packages/autorest.python/generator/pygen/postprocess/venvtools.py b/packages/autorest.python/generator/pygen/postprocess/venvtools.py new file mode 100644 index 00000000000..482cb9ee0ff --- /dev/null +++ b/packages/autorest.python/generator/pygen/postprocess/venvtools.py @@ -0,0 +1,77 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Optional +import subprocess +import venv +import sys +from pathlib import Path + + +_ROOT_DIR = Path(__file__).parent + + +class ExtendedEnvBuilder(venv.EnvBuilder): + """An extended env builder which saves the context, to have access + easily to bin path and such. + """ + + def __init__(self, *args, **kwargs): + self.context = None + if sys.version_info < (3, 9, 0): + # Not supported on Python 3.8, and we don't need it + kwargs.pop("upgrade_deps", None) + super().__init__(*args, **kwargs) + + def ensure_directories(self, env_dir): + self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir) + return self.context + + +def create( + env_dir, + system_site_packages=False, + clear=False, + symlinks=False, + with_pip=False, + prompt=None, + upgrade_deps=False, +): + """Create a virtual environment in a directory.""" + builder = ExtendedEnvBuilder( + system_site_packages=system_site_packages, + clear=clear, + symlinks=symlinks, + with_pip=with_pip, + prompt=prompt, + upgrade_deps=upgrade_deps, + ) + builder.create(env_dir) + return builder.context + + +def python_run( # pylint: disable=inconsistent-return-statements + venv_context, module, command, directory=_ROOT_DIR +) -> Optional[str]: + try: + cmd_line = [ + venv_context.env_exe, + "-m", + module, + ] + command + print("Executing: {}".format(" ".join(cmd_line))) + subprocess.run( + cmd_line, + cwd=directory, + check=True, + stdout=False, + ) + if module == "get_all": + with open(f"{command[1]}/.temp_folder/patched.txt", "r", encoding="utf-8-sig") as f: + return f.read() + except subprocess.CalledProcessError as err: + print(err) + sys.exit(1) + return None diff --git a/packages/autorest.python/generator/pygen/preprocess/__init__.py b/packages/autorest.python/generator/pygen/preprocess/__init__.py new file mode 100644 index 00000000000..d823a910872 --- /dev/null +++ b/packages/autorest.python/generator/pygen/preprocess/__init__.py @@ -0,0 +1,483 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""The preprocessing autorest plugin. +""" +import copy +from typing import Callable, Dict, Any, List, Optional + +from ..utils import to_snake_case +from .helpers import ( + add_redefined_builtin_info, + pad_builtin_namespaces, + pad_special_chars, +) +from .python_mappings import CADL_RESERVED_WORDS, RESERVED_WORDS, PadType + +from .. import YamlUpdatePlugin +from ..utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES + + +def update_overload_section( + overload: Dict[str, Any], + yaml_data: Dict[str, Any], + section: str, +): + try: + for overload_s, original_s in zip(overload[section], yaml_data[section]): + if overload_s.get("type"): + overload_s["type"] = original_s["type"] + if overload_s.get("headers"): + for overload_h, original_h in zip(overload_s["headers"], original_s["headers"]): + if overload_h.get("type"): + overload_h["type"] = original_h["type"] + except KeyError as exc: + raise ValueError(overload["name"]) from exc + + +def add_overload(yaml_data: Dict[str, Any], body_type: Dict[str, Any], for_flatten_params=False): + overload = copy.deepcopy(yaml_data) + overload["isOverload"] = True + overload["bodyParameter"]["type"] = body_type + overload["bodyParameter"]["defaultToUnsetSentinel"] = False + overload["overloads"] = [] + if yaml_data.get("initialOperation"): + overload["initialOperation"] = yaml_data["initialOperation"] + + if for_flatten_params: + overload["bodyParameter"]["flattened"] = True + else: + overload["parameters"] = [p for p in overload["parameters"] if not p.get("inFlattenedBody")] + # for yaml sync, we need to make sure all of the responses, parameters, and exceptions' types have the same yaml id + for overload_p, original_p in zip(overload["parameters"], yaml_data["parameters"]): + overload_p["type"] = original_p["type"] + update_overload_section(overload, yaml_data, "responses") + update_overload_section(overload, yaml_data, "exceptions") + + # update content type to be an overloads content type + content_type_param = next(p for p in overload["parameters"] if p["wireName"].lower() == "content-type") + content_type_param["inOverload"] = True + content_type_param["inDocstring"] = True + body_type_description = get_body_type_for_description(overload["bodyParameter"]) + content_type_param["description"] = ( + f"Body Parameter content-type. Content type parameter for {body_type_description} body." + ) + content_types = yaml_data["bodyParameter"]["contentTypes"] + if body_type["type"] == "binary" and len(content_types) > 1: + content_types = "'" + "', '".join(content_types) + "'" + content_type_param["description"] += f" Known values are: {content_types}." + overload["bodyParameter"]["inOverload"] = True + for parameter in overload["parameters"]: + parameter["inOverload"] = True + parameter["defaultToUnsetSentinel"] = False + return overload + + +def add_overloads_for_body_param(yaml_data: Dict[str, Any]) -> None: + """If we added a body parameter type, add overloads for that type""" + body_parameter = yaml_data["bodyParameter"] + if not ( + body_parameter["type"]["type"] == "combined" + and len(yaml_data["bodyParameter"]["type"]["types"]) > len(yaml_data["overloads"]) + ): + return + for body_type in body_parameter["type"]["types"]: + if any(o for o in yaml_data["overloads"] if id(o["bodyParameter"]["type"]) == id(body_type)): + continue + yaml_data["overloads"].append(add_overload(yaml_data, body_type)) + if body_type.get("type") == "model" and body_type.get("base") == "json": + yaml_data["overloads"].append(add_overload(yaml_data, body_type, for_flatten_params=True)) + content_type_param = next(p for p in yaml_data["parameters"] if p["wireName"].lower() == "content-type") + content_type_param["inOverload"] = False + content_type_param["inOverriden"] = True + content_type_param["inDocstring"] = True + content_type_param["clientDefaultValue"] = ( + None # make it none bc it will be overriden, we depend on default of overloads + ) + content_type_param["optional"] = True + + +def update_description(description: Optional[str], default_description: str = "") -> str: + if not description: + description = default_description + description.rstrip(" ") + if description and description[-1] != ".": + description += "." + return description + + +def update_operation_group_class_name(prefix: str, class_name: str) -> str: + if class_name == "": + return prefix + "OperationsMixin" + if class_name == "Operations": + return "Operations" + return class_name + "Operations" + + +def update_paging_response(yaml_data: Dict[str, Any]) -> None: + yaml_data["discriminator"] = "paging" + + +HEADERS_HIDE_IN_METHOD = ( + "repeatability-request-id", + "repeatability-first-sent", + "x-ms-client-request-id", + "client-request-id", + "return-client-request-id", +) +HEADERS_CONVERT_IN_METHOD = { + "if-match": { + "clientName": "etag", + "wireName": "etag", + "description": "check if resource is changed. Set None to skip checking etag.", + }, + "if-none-match": { + "clientName": "match_condition", + "wireName": "match-condition", + "description": "The match condition to use upon the etag.", + "type": { + "type": "sdkcore", + "name": "MatchConditions", + }, + }, +} + + +def get_wire_name_lower(parameter: Dict[str, Any]) -> str: + return (parameter.get("wireName") or "").lower() + + +def headers_convert(yaml_data: Dict[str, Any], replace_data: Any) -> None: + if isinstance(replace_data, dict): + for k, v in replace_data.items(): + yaml_data[k] = v + + +def has_json_content_type(yaml_data: Dict[str, Any]) -> bool: + return any(ct for ct in yaml_data.get("contentTypes", []) if JSON_REGEXP.match(ct)) + + +def has_multi_part_content_type(yaml_data: Dict[str, Any]) -> bool: + return any(ct for ct in yaml_data.get("contentTypes", []) if ct == "multipart/form-data") + + +class PreProcessPlugin(YamlUpdatePlugin): # pylint: disable=abstract-method + """Add Python naming information.""" + + @property + def azure_arm(self) -> bool: + return self.options.get("azure-arm", False) + + @property + def version_tolerant(self) -> bool: + return self.options.get("version-tolerant", True) + + @property + def models_mode(self) -> Optional[str]: + return self.options.get("models-mode", "dpg" if self.is_cadl else None) + + @property + def is_cadl(self) -> bool: + return self.options.get("cadl_file", False) + + def add_body_param_type( + self, + code_model: Dict[str, Any], + body_parameter: Dict[str, Any], + ): + # only add overload for special content type + if ( # pylint: disable=too-many-boolean-expressions + body_parameter + and body_parameter["type"]["type"] in ("model", "dict", "list") + and ( + has_json_content_type(body_parameter) or (self.is_cadl and has_multi_part_content_type(body_parameter)) + ) + and not body_parameter["type"].get("xmlMetadata") + and not any(t for t in ["flattened", "groupedBy"] if body_parameter.get(t)) + ): + origin_type = body_parameter["type"]["type"] + is_dpg_model = body_parameter["type"].get("base") == "dpg" + body_parameter["type"] = { + "type": "combined", + "types": [body_parameter["type"]], + } + # don't add binary overload for multipart content type + if not (self.is_cadl and has_multi_part_content_type(body_parameter)): + body_parameter["type"]["types"].append(KNOWN_TYPES["binary"]) + + if origin_type == "model" and is_dpg_model and self.models_mode == "dpg": + body_parameter["type"]["types"].insert(1, KNOWN_TYPES["any-object"]) + code_model["types"].append(body_parameter["type"]) + + def pad_reserved_words(self, name: str, pad_type: PadType): + # we want to pad hidden variables as well + if not name: + # we'll pass in empty operation groups sometime etc. + return name + + if self.is_cadl: + reserved_words = {k: (v + CADL_RESERVED_WORDS.get(k, [])) for k, v in RESERVED_WORDS.items()} + else: + reserved_words = RESERVED_WORDS + name = pad_special_chars(name) + name_prefix = "_" if name[0] == "_" else "" + name = name[1:] if name[0] == "_" else name + if name.lower() in reserved_words[pad_type]: + return name_prefix + name + pad_type + return name_prefix + name + + def update_types(self, yaml_data: List[Dict[str, Any]]) -> None: + for type in yaml_data: + for property in type.get("properties", []): + property["description"] = update_description(property.get("description", "")) + property["clientName"] = self.pad_reserved_words(property["clientName"].lower(), PadType.PROPERTY) + add_redefined_builtin_info(property["clientName"], property) + if type.get("name"): + name = self.pad_reserved_words(type["name"], PadType.MODEL) + type["name"] = name[0].upper() + name[1:] + type["description"] = update_description(type.get("description", ""), type["name"]) + type["snakeCaseName"] = to_snake_case(type["name"]) + if type.get("values"): + # we're enums + for value in type["values"]: + padded_name = self.pad_reserved_words(value["name"].lower(), PadType.ENUM).upper() + if padded_name[0] in "0123456789": + padded_name = "ENUM_" + padded_name + value["name"] = padded_name + + # add type for reference + for v in HEADERS_CONVERT_IN_METHOD.values(): + if isinstance(v, dict) and "type" in v: + yaml_data.append(v["type"]) + + def update_client(self, yaml_data: Dict[str, Any]) -> None: + yaml_data["description"] = update_description(yaml_data["description"], default_description=yaml_data["name"]) + yaml_data["legacyFilename"] = to_snake_case(yaml_data["name"].replace(" ", "_")) + parameters = yaml_data["parameters"] + for parameter in parameters: + self.update_parameter(parameter) + if parameter["clientName"] == "credential": + policy = parameter["type"].get("policy") + if policy and policy["type"] == "BearerTokenCredentialPolicy" and self.azure_arm: + policy["type"] = "ARMChallengeAuthenticationPolicy" + policy["credentialScopes"] = ["https://management.azure.com/.default"] + if ( + (not self.version_tolerant or self.azure_arm) + and parameters + and parameters[-1]["clientName"] == "credential" + ): + # we need to move credential to the front in mgmt mode for backcompat reasons + yaml_data["parameters"] = [parameters[-1]] + parameters[:-1] + prop_name = yaml_data["name"] + if prop_name.endswith("Client"): + prop_name = prop_name[: len(prop_name) - len("Client")] + yaml_data["builderPadName"] = to_snake_case(prop_name) + for og in yaml_data["operationGroups"]: + for o in og["operations"]: + property_if_match = None + property_if_none_match = None + for p in o["parameters"]: + wire_name_lower = get_wire_name_lower(p) + if p["location"] == "header" and wire_name_lower == "client-request-id": + yaml_data["requestIdHeaderName"] = wire_name_lower + if self.version_tolerant and p["location"] == "header": + if wire_name_lower == "if-match": + property_if_match = p + elif wire_name_lower == "if-none-match": + property_if_none_match = p + # pylint: disable=line-too-long + # some service(e.g. https://github.com/Azure/azure-rest-api-specs/blob/main/specification/cosmos-db/data-plane/Microsoft.Tables/preview/2019-02-02/table.json) + # only has one, so we need to add "if-none-match" or "if-match" if it's missing + if not property_if_match and property_if_none_match: + property_if_match = property_if_none_match.copy() + property_if_match["wireName"] = "if-match" + if not property_if_none_match and property_if_match: + property_if_none_match = property_if_match.copy() + property_if_none_match["wireName"] = "if-none-match" + + if property_if_match and property_if_none_match: + # arrange if-match and if-none-match to the end of parameters + o["parameters"] = [ + item + for item in o["parameters"] + if get_wire_name_lower(item) not in ("if-match", "if-none-match") + ] + [property_if_match, property_if_none_match] + + o["hasEtag"] = True + yaml_data["hasEtag"] = True + + def get_operation_updater(self, yaml_data: Dict[str, Any]) -> Callable[[Dict[str, Any], Dict[str, Any]], None]: + if yaml_data["discriminator"] == "lropaging": + return self.update_lro_paging_operation + if yaml_data["discriminator"] == "lro": + return self.update_lro_operation + if yaml_data["discriminator"] == "paging": + return self.update_paging_operation + return self.update_operation + + def update_parameter(self, yaml_data: Dict[str, Any]) -> None: + yaml_data["description"] = update_description(yaml_data.get("description", "")) + if not (yaml_data["location"] == "header" and yaml_data["clientName"] in ("content_type", "accept")): + yaml_data["clientName"] = self.pad_reserved_words(yaml_data["clientName"].lower(), PadType.PARAMETER) + if yaml_data.get("propertyToParameterName"): + # need to create a new one with padded keys and values + yaml_data["propertyToParameterName"] = { + self.pad_reserved_words(prop, PadType.PROPERTY): self.pad_reserved_words( + param_name, PadType.PARAMETER + ).lower() + for prop, param_name in yaml_data["propertyToParameterName"].items() + } + wire_name_lower = (yaml_data.get("wireName") or "").lower() + if yaml_data["location"] == "header" and ( + wire_name_lower in HEADERS_HIDE_IN_METHOD or yaml_data.get("clientDefaultValue") == "multipart/form-data" + ): + yaml_data["hideInMethod"] = True + if self.version_tolerant and yaml_data["location"] == "header" and wire_name_lower in HEADERS_CONVERT_IN_METHOD: + headers_convert(yaml_data, HEADERS_CONVERT_IN_METHOD[wire_name_lower]) + if wire_name_lower in ["$host", "content-type", "accept"] and yaml_data["type"]["type"] == "constant": + yaml_data["clientDefaultValue"] = yaml_data["type"]["value"] + + def update_operation( + self, + code_model: Dict[str, Any], + yaml_data: Dict[str, Any], + *, + is_overload: bool = False, + ) -> None: + yaml_data["groupName"] = self.pad_reserved_words(yaml_data["groupName"], PadType.OPERATION_GROUP) + yaml_data["groupName"] = to_snake_case(yaml_data["groupName"]) + yaml_data["name"] = yaml_data["name"].lower() + yaml_data["name"] = self.pad_reserved_words(yaml_data["name"], PadType.METHOD) + yaml_data["description"] = update_description(yaml_data["description"], yaml_data["name"]) + yaml_data["summary"] = update_description(yaml_data.get("summary", "")) + body_parameter = yaml_data.get("bodyParameter") + for parameter in yaml_data["parameters"]: + self.update_parameter(parameter) + if yaml_data.get("bodyParameter"): + self.update_parameter(yaml_data["bodyParameter"]) + for entry in yaml_data["bodyParameter"].get("entries", []): + self.update_parameter(entry) + for overload in yaml_data.get("overloads", []): + self.update_operation(code_model, overload, is_overload=True) + for response in yaml_data.get("responses", []): + response["discriminator"] = "operation" + if body_parameter and not is_overload: + # if we have a JSON body, we add a binary overload + self.add_body_param_type(code_model, body_parameter) + add_overloads_for_body_param(yaml_data) + + def _update_lro_operation_helper(self, yaml_data: Dict[str, Any]) -> None: + for response in yaml_data.get("responses", []): + response["discriminator"] = "lro" + response["pollerSync"] = response.get("pollerSync") or "azure.core.polling.LROPoller" + response["pollerAsync"] = response.get("pollerAsync") or "azure.core.polling.AsyncLROPoller" + if not response.get("pollingMethodSync"): + response["pollingMethodSync"] = ( + "azure.mgmt.core.polling.arm_polling.ARMPolling" + if self.azure_arm + else "azure.core.polling.base_polling.LROBasePolling" + ) + if not response.get("pollingMethodAsync"): + response["pollingMethodAsync"] = ( + "azure.mgmt.core.polling.async_arm_polling.AsyncARMPolling" + if self.azure_arm + else "azure.core.polling.async_base_polling.AsyncLROBasePolling" + ) + + def update_lro_paging_operation( + self, + code_model: Dict[str, Any], + yaml_data: Dict[str, Any], + is_overload: bool = False, + item_type: Optional[Dict[str, Any]] = None, + ) -> None: + self.update_lro_operation(code_model, yaml_data, is_overload=is_overload) + self.update_paging_operation(code_model, yaml_data, is_overload=is_overload, item_type=item_type) + yaml_data["discriminator"] = "lropaging" + for response in yaml_data.get("responses", []): + response["discriminator"] = "lropaging" + for overload in yaml_data.get("overloads", []): + self.update_lro_paging_operation( + code_model, + overload, + is_overload=True, + item_type=yaml_data["responses"][0]["itemType"], + ) + + def update_lro_operation( + self, + code_model: Dict[str, Any], + yaml_data: Dict[str, Any], + is_overload: bool = False, + ) -> None: + self.update_operation(code_model, yaml_data, is_overload=is_overload) + self.update_operation(code_model, yaml_data["initialOperation"], is_overload=is_overload) + self._update_lro_operation_helper(yaml_data) + for overload in yaml_data.get("overloads", []): + self._update_lro_operation_helper(overload) + self.update_operation(code_model, overload["initialOperation"], is_overload=True) + + def update_paging_operation( + self, + code_model: Dict[str, Any], + yaml_data: Dict[str, Any], + is_overload: bool = False, + item_type: Optional[Dict[str, Any]] = None, + ) -> None: + self.update_operation(code_model, yaml_data, is_overload=is_overload) + item_type = item_type or yaml_data["itemType"]["elementType"] + if yaml_data.get("nextOperation"): + yaml_data["nextOperation"]["groupName"] = self.pad_reserved_words( + yaml_data["nextOperation"]["groupName"], PadType.OPERATION_GROUP + ) + yaml_data["nextOperation"]["groupName"] = to_snake_case(yaml_data["nextOperation"]["groupName"]) + for response in yaml_data["nextOperation"].get("responses", []): + update_paging_response(response) + response["itemType"] = item_type + for response in yaml_data.get("responses", []): + update_paging_response(response) + response["itemType"] = item_type + for overload in yaml_data.get("overloads", []): + self.update_paging_operation(code_model, overload, is_overload=True, item_type=item_type) + + def update_operation_groups(self, code_model: Dict[str, Any], client: Dict[str, Any]) -> None: + operation_groups_yaml_data = client["operationGroups"] + for operation_group in operation_groups_yaml_data: + operation_group["identifyName"] = self.pad_reserved_words( + operation_group.get("name", operation_group["propertyName"]), + PadType.OPERATION_GROUP, + ) + operation_group["identifyName"] = to_snake_case(operation_group["identifyName"]) + operation_group["propertyName"] = self.pad_reserved_words( + operation_group["propertyName"], PadType.OPERATION_GROUP + ) + operation_group["propertyName"] = to_snake_case(operation_group["propertyName"]) + operation_group["className"] = update_operation_group_class_name( + client["name"], operation_group["className"] + ) + for operation in operation_group["operations"]: + self.get_operation_updater(operation)(code_model, operation) + + if operation_group.get("operationGroups"): + self.update_operation_groups(code_model, operation_group) + + def update_yaml(self, yaml_data: Dict[str, Any]) -> None: + """Convert in place the YAML str.""" + self.update_types(yaml_data["types"]) + for client in yaml_data["clients"]: + self.update_client(client) + self.update_operation_groups(yaml_data, client) + for clients in yaml_data["subnamespaceToClients"].values(): + for client in clients: + self.update_client(client) + self.update_operation_groups(yaml_data, client) + if yaml_data.get("namespace"): + yaml_data["namespace"] = pad_builtin_namespaces(yaml_data["namespace"]) + + +if __name__ == "__main__": + # CADL pipeline will call this + args, unknown_args = parse_args() + PreProcessPlugin(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process() diff --git a/packages/autorest.python/generator/pygen/preprocess/helpers.py b/packages/autorest.python/generator/pygen/preprocess/helpers.py new file mode 100644 index 00000000000..cb9a664a776 --- /dev/null +++ b/packages/autorest.python/generator/pygen/preprocess/helpers.py @@ -0,0 +1,27 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import re +from typing import Any, Dict +from .python_mappings import ( + REDEFINED_BUILTINS, + BUILTIN_PACKAGES, +) + + +def add_redefined_builtin_info(name: str, yaml_data: Dict[str, Any]) -> None: + if name in REDEFINED_BUILTINS: + yaml_data["pylintDisable"] = "redefined-builtin" + + +def pad_builtin_namespaces(namespace: str) -> str: + items = namespace.split(".") + if items[0] in BUILTIN_PACKAGES: + items[0] = items[0] + "_" + return ".".join(items) + + +def pad_special_chars(name: str) -> str: + return re.sub(r"[^A-z0-9_]", "_", name) diff --git a/packages/autorest.python/generator/pygen/preprocess/python_mappings.py b/packages/autorest.python/generator/pygen/preprocess/python_mappings.py new file mode 100644 index 00000000000..4edebd7120b --- /dev/null +++ b/packages/autorest.python/generator/pygen/preprocess/python_mappings.py @@ -0,0 +1,222 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from enum import Enum + +basic_latin_chars = { + " ": "Space", + "!": "ExclamationMark", + '"': "QuotationMark", + "#": "NumberSign", + "$": "DollarSign", + "%": "PercentSign", + "&": "Ampersand", + "'": "Apostrophe", + "(": "LeftParenthesis", + ")": "RightParenthesis", + "*": "Asterisk", + "+": "PlusSign", + ",": "Comma", + "-": "HyphenMinus", + ".": "FullStop", + "/": "Slash", + "0": "Zero", + "1": "One", + "2": "Two", + "3": "Three", + "4": "Four", + "5": "Five", + "6": "Six", + "7": "Seven", + "8": "Eight", + "9": "Nine", + ":": "Colon", + ";": "Semicolon", + "<": "LessThanSign", + "=": "EqualSign", + ">": "GreaterThanSign", + "?": "QuestionMark", + "@": "AtSign", + "[": "LeftSquareBracket", + "\\": "Backslash", + "]": "RightSquareBracket", + "^": "CircumflexAccent", + "`": "GraveAccent", + "{": "LeftCurlyBracket", + "|": "VerticalBar", + "}": "RightCurlyBracket", + "~": "Tilde", +} + + +class PadType(str, Enum): + MODEL = "Model" + METHOD = "_method" + PARAMETER = "_parameter" + ENUM = "_enum" + PROPERTY = "_property" + OPERATION_GROUP = "Operations" + + +_always_reserved = [ + "and", + "as", + "assert", + "break", + "class", + "continue", + "def", + "del", + "elif", + "else", + "except", + "exec", + "finally", + "for", + "from", + "global", + "if", + "import", + "in", + "is", + "lambda", + "not", + "or", + "pass", + "raise", + "return", + "try", + "while", + "with", + "yield", + "async", + "await", + "int", +] + +RESERVED_MODEL_PROPERTIES = [ + "keys", + "items", + "values", + "popitem", + "clear", + "update", + "setdefault", + "pop", + "get", +] + +RESERVED_WORDS = { + PadType.METHOD: [*_always_reserved], + PadType.PARAMETER: [ + "self", + # these are kwargs we've reserved for our generated operations + "content_type", + "accept", + "cls", + "polling", + "continuation_token", # for LRO calls + # these are transport kwargs + # https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/CLIENT_LIBRARY_DEVELOPER.md#transport + "connection_timeout", + "connection_verify", + "connection_cert", + "connection_data_block_size", + "use_env_settings", + # the following aren't in the readme, but Xiang said these are also transport kwargs + "read_timeout", + "proxies", + "cookies", + # these are policy kwargs + # https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/CLIENT_LIBRARY_DEVELOPER.md#available-policies + "base_headers", + "headers", + "request_id", + "auto_request_id", + "base_user_agent", + "user_agent", + "user_agent_overwrite", + "user_agent_use_env", + "user_agent", + "sdk_moniker", + "logging_enable", + "logger", + "response_encoding", + "proxies", + "raw_request_hook", + "raw_response_hook", + "network_span_namer", + "tracing_attributes", + "permit_redirects", + "redirect_max", + "redirect_remove_headers", + "redirect_on_status_codes", + "permit_redirects", + "redirect_max", + "redirect_remove_headers", + "redirect_on_status_codes", + "retry_total", + "retry_connect", + "retry_read", + "retry_status", + "retry_backoff_factor", + "retry_backoff_max", + "retry_mode", + "retry_on_status_codes", + "retry_total", + "retry_connect", + "retry_read", + "retry_status", + "retry_backoff_factor", + "retry_backoff_max", + "retry_mode", + "retry_on_status_codes", + *_always_reserved, + ], + PadType.MODEL: [*_always_reserved], + PadType.PROPERTY: ["self", *_always_reserved], + PadType.ENUM: ["mro", *_always_reserved], + PadType.OPERATION_GROUP: [*_always_reserved], +} + +CADL_RESERVED_WORDS = { + PadType.PARAMETER: ["stream"], + PadType.PROPERTY: RESERVED_MODEL_PROPERTIES, +} + +REDEFINED_BUILTINS = [ # we don't pad, but we need to do lint ignores + "id", + "min", + "max", + "filter", + "property", +] + +BUILTIN_PACKAGES = [ + "array", + "atexit", + "binascii", + "builtins", + "cmath", + "errno", + "faulthandler", + "fcntl", + "gc", + "grp", + "itertools", + "marshal", + "math", + "posix", + "pwd", + "pyexpat", + "select", + "spwd", + "sys", + "syslog", + "time", + "unicodedata", + "xxsubtype", + "zlib", +] diff --git a/packages/autorest.python/generator/pygen/utils.py b/packages/autorest.python/generator/pygen/utils.py new file mode 100644 index 00000000000..95bf493bde8 --- /dev/null +++ b/packages/autorest.python/generator/pygen/utils.py @@ -0,0 +1,149 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict, Tuple, List +import re +import argparse + + +def update_enum_value(name: str, value: Any, description: str, enum_type: Dict[str, Any]) -> Dict[str, Any]: + return { + "name": name, + "type": "enumvalue", + "value": value, + "description": description, + "enumType": enum_type, + "valueType": enum_type["valueType"], + } + + +def to_snake_case(name: str) -> str: + def replace_upper_characters(m) -> str: + match_str = m.group().lower() + if m.start() > 0 and name[m.start() - 1] == "_": + # we are good if a '_' already exists + return match_str + # the first letter should not have _ + prefix = "_" if m.start() > 0 else "" + + # we will add an extra _ if there are multiple upper case chars together + next_non_upper_case_char_location = m.start() + len(match_str) + if ( + len(match_str) > 2 + and len(name) - next_non_upper_case_char_location > 1 + and name[next_non_upper_case_char_location].isalpha() + ): + return prefix + match_str[: len(match_str) - 1] + "_" + match_str[len(match_str) - 1] + + return prefix + match_str + + result = re.sub("[A-Z]+", replace_upper_characters, name) + return result.replace(" ", "_").replace("__", "_").replace("-", "") + + +def parse_args( + need_cadl_file: bool = True, +) -> Tuple[argparse.Namespace, Dict[str, Any]]: + parser = argparse.ArgumentParser( + description="Run mypy against target folder. Add a local custom plugin to the path prior to execution. " + ) + parser.add_argument( + "--output-folder", + dest="output_folder", + help="Output folder for generated SDK", + required=True, + ) + parser.add_argument( + "--cadl-file", + dest="cadl_file", + help="Serialized cadl file", + required=need_cadl_file, + ) + parser.add_argument( + "--debug", + dest="debug", + help="Debug mode", + required=False, + action="store", + ) + args, unknown_args = parser.parse_known_args() + + def _get_value(value: Any) -> Any: + if value == "true": + return True + if value == "false": + return False + try: + return int(value) + except ValueError: + pass + return value + + unknown_args_ret = { + ua.strip("--").split("=", maxsplit=1)[0]: _get_value( # pylint: disable=bad-str-strip-call + ua.strip("--").split("=", maxsplit=1)[1] # pylint: disable=bad-str-strip-call + ) + for ua in unknown_args + } + return args, unknown_args_ret + + +def get_body_type_for_description(body_parameter: Dict[str, Any]) -> str: + if body_parameter["type"]["type"] == "binary": + return "binary" + if body_parameter["type"]["type"] == "string": + return "string" + return "JSON" + + +# used if we want to get a string / binary type etc +KNOWN_TYPES: Dict[str, Dict[str, Any]] = { + "string": {"type": "string"}, + "binary": {"type": "binary"}, + "anydict": {"type": "dict", "elementType": {"type": "any"}}, + "any-object": {"type": "any-object"}, +} + +JSON_REGEXP = re.compile(r"^(application|text)/(.+\+)?json$") + + +def build_policies( + is_arm: bool, + async_mode: bool, + *, + is_azure_flavor: bool = False, + tracing: bool = True, +) -> List[str]: + if is_azure_flavor: + # for Azure + async_prefix = "Async" if async_mode else "" + policies = [ + "policies.RequestIdPolicy(**kwargs)", + "self._config.headers_policy", + "self._config.user_agent_policy", + "self._config.proxy_policy", + "policies.ContentDecodePolicy(**kwargs)", + (f"{async_prefix}ARMAutoResourceProviderRegistrationPolicy()" if is_arm else None), + "self._config.redirect_policy", + "self._config.retry_policy", + "self._config.authentication_policy", + "self._config.custom_hook_policy", + "self._config.logging_policy", + "policies.DistributedTracingPolicy(**kwargs)" if tracing else None, + "policies.SensitiveHeaderCleanupPolicy(**kwargs) if self._config.redirect_policy else None", + "self._config.http_logging_policy", + ] + else: + # for non-Azure + policies = [ + "self._config.headers_policy", + "self._config.user_agent_policy", + "self._config.proxy_policy", + "policies.ContentDecodePolicy(**kwargs)", + "self._config.retry_policy", + "self._config.authentication_policy", + "self._config.logging_policy", + ] + return [p for p in policies if p] diff --git a/packages/autorest.python/generator/requirements.txt b/packages/autorest.python/generator/requirements.txt new file mode 100644 index 00000000000..bebbbb5a645 --- /dev/null +++ b/packages/autorest.python/generator/requirements.txt @@ -0,0 +1,12 @@ +black==24.4.0 +click==8.1.3 +docutils==0.19 +Jinja2==3.1.4 +m2r2==0.3.3 +MarkupSafe==2.1.2 +mistune==0.8.4 +pathspec==0.11.1 +platformdirs==3.2.0 +PyYAML==6.0.1 +tomli==2.0.1 +setuptools==69.2.0 diff --git a/packages/autorest.python/generator/setup.py b/packages/autorest.python/generator/setup.py new file mode 100644 index 00000000000..b19d4f2692e --- /dev/null +++ b/packages/autorest.python/generator/setup.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + + +import os +import re + +from setuptools import setup, find_packages + + +# Version extraction inspired from 'requests' +with open(os.path.join("pygen", "_version.py"), "r") as fd: + version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) + +if not version: + raise RuntimeError("Cannot find version information") + +setup( + name="pygen", + version=version, + description="Core Library for Python Generation", + long_description=open("README.md", "r").read(), + long_description_content_type="text/markdown", + license="MIT License", + author="Microsoft Corporation", + author_email="azpysdkhelp@microsoft.com", + url="https://github.com/Azure/autorest.python/packages/core", + classifiers=[ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", + ], + packages=find_packages( + exclude=[ + "test", + ] + ), + install_requires=[ + "Jinja2 >= 2.11", # I need "include" and auto-context + blank line are not indented by default + "pyyaml", + "m2r2", + "black", + ], +) diff --git a/packages/typespec-python/generator/pygen/black.py b/packages/typespec-python/generator/pygen/black.py new file mode 100644 index 00000000000..b1d48aff821 --- /dev/null +++ b/packages/typespec-python/generator/pygen/black.py @@ -0,0 +1,71 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from pathlib import Path +import os +from typing import Any, Dict +import black +from black.report import NothingChanged + +from . import Plugin +from .utils import parse_args + +_LOGGER = logging.getLogger("blib2to3") + +_BLACK_MODE = black.Mode() # pyright: ignore [reportPrivateImportUsage] +_BLACK_MODE.line_length = 120 + + +class BlackScriptPlugin(Plugin): # pylint: disable=abstract-method + def __init__(self, **kwargs): + super().__init__(**kwargs) + output_folder = self.options.get("output_folder", str(self.output_folder)) + if output_folder.startswith("file:"): + output_folder = output_folder[5:] + if os.name == "nt" and output_folder.startswith("///"): + output_folder = output_folder[3:] + self.output_folder = Path(output_folder) + + def process(self) -> bool: + # apply format_file on every .py file in the output folder + list( + map( + self.format_file, + [ + Path(f) + for f in self.list_file() + if all( + item not in f + for item in ( + "__pycache__", + "node_modules", + ".tox", + ".mypy_cache", + ) + ) + and Path(f).suffix == ".py" + ], + ) + ) + return True + + def format_file(self, file: Path) -> None: + try: + file_content = self.read_file(file) + file_content = black.format_file_contents(file_content, fast=True, mode=_BLACK_MODE) + except NothingChanged: + pass + except: # pylint: disable=bare-except + _LOGGER.error("Error: failed to format %s", file) + raise + else: + self.write_file(file, file_content) + + +if __name__ == "__main__": + # CADL pipeline will call this + args, unknown_args = parse_args(need_cadl_file=False) + BlackScriptPlugin(output_folder=args.output_folder, **unknown_args).process() diff --git a/packages/typespec-python/generator/pygen/m2r.py b/packages/typespec-python/generator/pygen/m2r.py new file mode 100644 index 00000000000..d709863e7c4 --- /dev/null +++ b/packages/typespec-python/generator/pygen/m2r.py @@ -0,0 +1,65 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""An MD to RST plugin. +""" +import logging +from typing import Any, Dict, Set, Union + +import m2r2 + +from . import YamlUpdatePlugin +from .utils import parse_args + + +_LOGGER = logging.getLogger(__name__) + + +class GeneratorRenderer(m2r2.RestRenderer): + """Redefine the concept of inline HTML in the renderer, we don't want to define a new format + in the description/summary. + """ + + def inline_html(self, html: str) -> str: + """Do not render inline HTML with a role definition.""" + return f":code:`{html}`" + + +class M2R(YamlUpdatePlugin): # pylint: disable=abstract-method + """A plugin to convert any description and summary from MD to RST.""" + + def update_yaml(self, yaml_data: Dict[str, Any]) -> None: + """Convert in place the YAML str.""" + self._convert_docstring_no_cycles(yaml_data, set()) + + def _convert_docstring_no_cycles(self, yaml_data: Union[Dict[str, Any], str], node_list: Set[int]) -> None: + """Walk the YAML tree to convert MD to RST.""" + if id(yaml_data) in node_list: + return + node_list.add(id(yaml_data)) + + if isinstance(yaml_data, list): + for elt in yaml_data: + self._convert_docstring_no_cycles(elt, node_list) + elif isinstance(yaml_data, dict): + for key, value in yaml_data.items(): + if key in ["description", "summary"]: + yaml_data[key] = self.convert_to_rst(value) + continue + self._convert_docstring_no_cycles(value, node_list) + + @staticmethod + def convert_to_rst(string_to_convert: str) -> str: + """Convert that string from MD to RST.""" + try: + return m2r2.convert(string_to_convert, renderer=GeneratorRenderer()).strip() + except Exception: # pylint: disable=broad-except + return string_to_convert + + +if __name__ == "__main__": + # CADL pipeline will call this + args, unknown_args = parse_args() + M2R(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process() diff --git a/packages/typespec-python/scripts/run_tsp.py b/packages/typespec-python/scripts/run_tsp.py index c9d360706fe..8f68f260cbb 100644 --- a/packages/typespec-python/scripts/run_tsp.py +++ b/packages/typespec-python/scripts/run_tsp.py @@ -34,7 +34,7 @@ breakpoint() # pylint: disable=undefined-variable # run m2r - python_run(venv_context, "m2r.__init__", command=sys.argv[1:]) + python_run(venv_context, "m2r", command=sys.argv[1:]) python_run(venv_context, "preprocess.__init__", command=sys.argv[1:]) python_run(venv_context, "codegen.__init__", command=sys.argv[1:]) - python_run(venv_context, "black.__init__", command=sys.argv[1:]) + python_run(venv_context, "black", command=sys.argv[1:]) From 8fb27e1f473ff7523d45914e134a424cc44363f9 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 6 Jun 2024 18:14:15 -0400 Subject: [PATCH 66/75] remove generator from autorest.python committed code, can generate typespec --- .gitignore | 2 +- packages/autorest.python/generator/LICENSE | 21 - packages/autorest.python/generator/README.md | 1 - .../generator/dev_requirements.txt | 4 - .../generator/pygen/__init__.py | 107 - .../generator/pygen/_version.py | 7 - .../generator/pygen/black/__init__.py | 71 - .../generator/pygen/codegen/__init__.py | 334 --- .../generator/pygen/codegen/_utils.py | 16 - .../pygen/codegen/models/__init__.py | 202 -- .../generator/pygen/codegen/models/base.py | 188 -- .../pygen/codegen/models/base_builder.py | 119 - .../generator/pygen/codegen/models/client.py | 422 ---- .../pygen/codegen/models/code_model.py | 245 -- .../pygen/codegen/models/combined_type.py | 153 -- .../pygen/codegen/models/constant_type.py | 134 -- .../pygen/codegen/models/credential_types.py | 223 -- .../pygen/codegen/models/dictionary_type.py | 131 -- .../pygen/codegen/models/enum_type.py | 246 -- .../generator/pygen/codegen/models/imports.py | 291 --- .../pygen/codegen/models/list_type.py | 147 -- .../pygen/codegen/models/lro_operation.py | 143 -- .../codegen/models/lro_paging_operation.py | 32 - .../pygen/codegen/models/model_type.py | 350 --- .../pygen/codegen/models/operation.py | 510 ----- .../pygen/codegen/models/operation_group.py | 184 -- .../pygen/codegen/models/paging_operation.py | 156 -- .../pygen/codegen/models/parameter.py | 402 ---- .../pygen/codegen/models/parameter_list.py | 390 ---- .../pygen/codegen/models/primitive_types.py | 640 ------ .../pygen/codegen/models/property.py | 182 -- .../pygen/codegen/models/request_builder.py | 189 -- .../models/request_builder_parameter.py | 115 - .../pygen/codegen/models/response.py | 348 --- .../generator/pygen/codegen/models/utils.py | 23 - .../pygen/codegen/serializers/__init__.py | 570 ----- .../codegen/serializers/base_serializer.py | 21 - .../codegen/serializers/builder_serializer.py | 1453 ------------ .../codegen/serializers/client_serializer.py | 295 --- .../codegen/serializers/enum_serializer.py | 15 - .../codegen/serializers/general_serializer.py | 212 -- .../codegen/serializers/import_serializer.py | 127 -- .../serializers/metadata_serializer.py | 198 -- .../serializers/model_init_serializer.py | 33 - .../codegen/serializers/model_serializer.py | 287 --- .../operation_groups_serializer.py | 89 - .../serializers/operations_init_serializer.py | 44 - .../serializers/parameter_serializer.py | 221 -- .../codegen/serializers/patch_serializer.py | 19 - .../request_builders_serializer.py | 52 - .../codegen/serializers/sample_serializer.py | 163 -- .../codegen/serializers/test_serializer.py | 263 --- .../codegen/serializers/types_serializer.py | 31 - .../pygen/codegen/serializers/utils.py | 68 - .../pygen/codegen/templates/client.py.jinja2 | 37 - .../templates/client_container.py.jinja2 | 12 - .../pygen/codegen/templates/config.py.jinja2 | 73 - .../templates/config_container.py.jinja2 | 16 - .../codegen/templates/conftest.py.jinja2 | 28 - .../pygen/codegen/templates/enum.py.jinja2 | 13 - .../templates/enum_container.py.jinja2 | 10 - .../pygen/codegen/templates/init.py.jinja2 | 24 - .../pygen/codegen/templates/keywords.jinja2 | 19 - .../codegen/templates/lro_operation.py.jinja2 | 16 - .../templates/lro_paging_operation.py.jinja2 | 18 - .../pygen/codegen/templates/macros.jinja2 | 12 - .../codegen/templates/metadata.json.jinja2 | 167 -- .../codegen/templates/model_base.py.jinja2 | 898 -------- .../templates/model_container.py.jinja2 | 13 - .../codegen/templates/model_dpg.py.jinja2 | 90 - .../codegen/templates/model_init.py.jinja2 | 28 - .../codegen/templates/model_msrest.py.jinja2 | 92 - .../codegen/templates/operation.py.jinja2 | 21 - .../templates/operation_group.py.jinja2 | 75 - .../operation_groups_container.py.jinja2 | 20 - .../codegen/templates/operation_tools.jinja2 | 74 - .../operations_folder_init.py.jinja2 | 17 - .../packaging_templates/CHANGELOG.md.jinja2 | 6 - .../packaging_templates/LICENSE.jinja2 | 21 - .../packaging_templates/MANIFEST.in.jinja2 | 8 - .../packaging_templates/README.md.jinja2 | 107 - .../dev_requirements.txt.jinja2 | 9 - .../packaging_templates/setup.py.jinja2 | 110 - .../templates/paging_operation.py.jinja2 | 21 - .../pygen/codegen/templates/patch.py.jinja2 | 19 - .../codegen/templates/pkgutil_init.py.jinja2 | 1 - .../templates/request_builder.py.jinja2 | 28 - .../templates/request_builders.py.jinja2 | 10 - .../codegen/templates/rest_init.py.jinja2 | 12 - .../pygen/codegen/templates/sample.py.jinja2 | 44 - .../codegen/templates/serialization.py.jinja2 | 2004 ----------------- .../pygen/codegen/templates/test.py.jinja2 | 26 - .../codegen/templates/testpreparer.py.jinja2 | 26 - .../pygen/codegen/templates/types.py.jinja2 | 8 - .../codegen/templates/validation.py.jinja2 | 38 - .../pygen/codegen/templates/vendor.py.jinja2 | 98 - .../pygen/codegen/templates/version.py.jinja2 | 4 - .../generator/pygen/m2r/__init__.py | 65 - .../generator/pygen/postprocess/__init__.py | 183 -- .../generator/pygen/postprocess/get_all.py | 19 - .../generator/pygen/postprocess/venvtools.py | 77 - .../generator/pygen/preprocess/__init__.py | 483 ---- .../generator/pygen/preprocess/helpers.py | 27 - .../pygen/preprocess/python_mappings.py | 222 -- .../autorest.python/generator/pygen/utils.py | 149 -- .../generator/requirements.txt | 12 - packages/autorest.python/generator/setup.py | 55 - packages/typespec-python/scripts/prepare.py | 1 + packages/typespec-python/scripts/run_tsp.py | 8 +- packages/typespec-python/scripts/venvtools.py | 2 +- 110 files changed, 7 insertions(+), 16558 deletions(-) delete mode 100644 packages/autorest.python/generator/LICENSE delete mode 100644 packages/autorest.python/generator/README.md delete mode 100644 packages/autorest.python/generator/dev_requirements.txt delete mode 100644 packages/autorest.python/generator/pygen/__init__.py delete mode 100644 packages/autorest.python/generator/pygen/_version.py delete mode 100644 packages/autorest.python/generator/pygen/black/__init__.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/__init__.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/_utils.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/__init__.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/base.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/base_builder.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/client.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/code_model.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/combined_type.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/constant_type.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/credential_types.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/dictionary_type.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/enum_type.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/imports.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/list_type.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/lro_operation.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/lro_paging_operation.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/model_type.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/operation.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/operation_group.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/paging_operation.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/parameter.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/parameter_list.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/primitive_types.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/property.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/request_builder.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/request_builder_parameter.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/response.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/models/utils.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/__init__.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/base_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/builder_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/client_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/enum_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/general_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/import_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/metadata_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/model_init_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/model_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/operation_groups_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/operations_init_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/parameter_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/patch_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/request_builders_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/sample_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/test_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/types_serializer.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/serializers/utils.py delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/client.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/client_container.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/config.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/config_container.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/conftest.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/enum.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/enum_container.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/init.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/keywords.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/lro_operation.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/macros.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/metadata.json.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/model_base.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/model_container.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/model_dpg.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/model_init.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/model_msrest.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/operation.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/operation_group.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/operation_tools.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/paging_operation.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/patch.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/request_builder.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/request_builders.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/rest_init.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/sample.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/serialization.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/test.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/testpreparer.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/types.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/validation.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/vendor.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/codegen/templates/version.py.jinja2 delete mode 100644 packages/autorest.python/generator/pygen/m2r/__init__.py delete mode 100644 packages/autorest.python/generator/pygen/postprocess/__init__.py delete mode 100644 packages/autorest.python/generator/pygen/postprocess/get_all.py delete mode 100644 packages/autorest.python/generator/pygen/postprocess/venvtools.py delete mode 100644 packages/autorest.python/generator/pygen/preprocess/__init__.py delete mode 100644 packages/autorest.python/generator/pygen/preprocess/helpers.py delete mode 100644 packages/autorest.python/generator/pygen/preprocess/python_mappings.py delete mode 100644 packages/autorest.python/generator/pygen/utils.py delete mode 100644 packages/autorest.python/generator/requirements.txt delete mode 100644 packages/autorest.python/generator/setup.py diff --git a/.gitignore b/.gitignore index 4a669c950e6..2bbf0a3b855 100644 --- a/.gitignore +++ b/.gitignore @@ -125,4 +125,4 @@ node_modules/ # Generated test folders test/services/*/_generated -**/autorest.python/pygen +**/autorest.python/generator diff --git a/packages/autorest.python/generator/LICENSE b/packages/autorest.python/generator/LICENSE deleted file mode 100644 index 21071075c24..00000000000 --- a/packages/autorest.python/generator/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ - MIT License - - Copyright (c) Microsoft Corporation. All rights reserved. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE diff --git a/packages/autorest.python/generator/README.md b/packages/autorest.python/generator/README.md deleted file mode 100644 index 3d8a65a8919..00000000000 --- a/packages/autorest.python/generator/README.md +++ /dev/null @@ -1 +0,0 @@ -# Core Library for Python Generation diff --git a/packages/autorest.python/generator/dev_requirements.txt b/packages/autorest.python/generator/dev_requirements.txt deleted file mode 100644 index 32f2ff33251..00000000000 --- a/packages/autorest.python/generator/dev_requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ --r ../../../eng/requirements.txt --r ../../../eng/dev_requirements.txt -ptvsd==4.3.2 -types-PyYAML==6.0.12.8 diff --git a/packages/autorest.python/generator/pygen/__init__.py b/packages/autorest.python/generator/pygen/__init__.py deleted file mode 100644 index 6050cbec00f..00000000000 --- a/packages/autorest.python/generator/pygen/__init__.py +++ /dev/null @@ -1,107 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import logging -from pathlib import Path -import json -from abc import ABC, abstractmethod -from typing import Any, Dict, Union, List - -import yaml - -from ._version import VERSION - - -__version__ = VERSION -_LOGGER = logging.getLogger(__name__) - - -class ReaderAndWriter: - def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None: - self.output_folder = Path(output_folder) - self._list_file: List[str] = [] - try: - with open( - Path(self.output_folder) / Path("..") / Path("python.json"), - "r", - encoding="utf-8-sig", - ) as fd: - python_json = json.load(fd) - except Exception: # pylint: disable=broad-except - python_json = {} - self.options = kwargs - if python_json: - _LOGGER.warning("Loading python.json file. This behavior will be depreacted") - self.options.update(python_json) - - def read_file(self, path: Union[str, Path]) -> str: - """Directly reading from disk""" - # make path relative to output folder - try: - with open(self.output_folder / Path(path), "r", encoding="utf-8-sig") as fd: - return fd.read() - except FileNotFoundError: - return "" - - def write_file(self, filename: Union[str, Path], file_content: str) -> None: - """Directly writing to disk""" - file_folder = Path(filename).parent - if not Path.is_dir(self.output_folder / file_folder): - Path.mkdir(self.output_folder / file_folder, parents=True) - with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd: - fd.write(file_content) - - def list_file(self) -> List[str]: - return [str(f) for f in self.output_folder.glob("**/*") if f.is_file()] - - -class Plugin(ReaderAndWriter, ABC): - """A base class for autorest plugin. - - :param autorestapi: An autorest API instance - """ - - @abstractmethod - def process(self) -> bool: - """The plugin process. - - :rtype: bool - :returns: True if everything's ok, False optherwise - :raises Exception: Could raise any exception, stacktrace will be sent to autorest API - """ - raise NotImplementedError() - - -class YamlUpdatePlugin(Plugin): - """A plugin that update the YAML as input.""" - - def get_yaml(self) -> Dict[str, Any]: - # cadl file doesn't have to be relative to output folder - with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd: - return yaml.safe_load(fd.read()) - - def write_yaml(self, yaml_string: str) -> None: - with open(self.options["cadl_file"], "w", encoding="utf-8-sig") as fd: - fd.write(yaml_string) - - def process(self) -> bool: - # List the input file, should be only one - yaml_data = self.get_yaml() - - self.update_yaml(yaml_data) - - yaml_string = yaml.safe_dump(yaml_data) - - self.write_yaml(yaml_string) - return True - - @abstractmethod - def update_yaml(self, yaml_data: Dict[str, Any]) -> None: - """The code-model-v4-no-tags yaml model tree. - - :rtype: updated yaml - :raises Exception: Could raise any exception, stacktrace will be sent to autorest API - """ - raise NotImplementedError() diff --git a/packages/autorest.python/generator/pygen/_version.py b/packages/autorest.python/generator/pygen/_version.py deleted file mode 100644 index aadc0f31ff8..00000000000 --- a/packages/autorest.python/generator/pygen/_version.py +++ /dev/null @@ -1,7 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -VERSION = "0.1.0" diff --git a/packages/autorest.python/generator/pygen/black/__init__.py b/packages/autorest.python/generator/pygen/black/__init__.py deleted file mode 100644 index 82033d53b17..00000000000 --- a/packages/autorest.python/generator/pygen/black/__init__.py +++ /dev/null @@ -1,71 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import logging -from pathlib import Path -import os -from typing import Any, Dict -import black -from black.report import NothingChanged - -from .. import Plugin -from ..utils import parse_args - -_LOGGER = logging.getLogger("blib2to3") - -_BLACK_MODE = black.Mode() # pyright: ignore [reportPrivateImportUsage] -_BLACK_MODE.line_length = 120 - - -class BlackScriptPlugin(Plugin): # pylint: disable=abstract-method - def __init__(self, **kwargs): - super().__init__(**kwargs) - output_folder = self.options.get("output_folder", str(self.output_folder)) - if output_folder.startswith("file:"): - output_folder = output_folder[5:] - if os.name == "nt" and output_folder.startswith("///"): - output_folder = output_folder[3:] - self.output_folder = Path(output_folder) - - def process(self) -> bool: - # apply format_file on every .py file in the output folder - list( - map( - self.format_file, - [ - Path(f) - for f in self.list_file() - if all( - item not in f - for item in ( - "__pycache__", - "node_modules", - ".tox", - ".mypy_cache", - ) - ) - and Path(f).suffix == ".py" - ], - ) - ) - return True - - def format_file(self, file: Path) -> None: - try: - file_content = self.read_file(file) - file_content = black.format_file_contents(file_content, fast=True, mode=_BLACK_MODE) - except NothingChanged: - pass - except: # pylint: disable=bare-except - _LOGGER.error("Error: failed to format %s", file) - raise - else: - self.write_file(file, file_content) - - -if __name__ == "__main__": - # CADL pipeline will call this - args, unknown_args = parse_args(need_cadl_file=False) - BlackScriptPlugin(output_folder=args.output_folder, **unknown_args).process() diff --git a/packages/autorest.python/generator/pygen/codegen/__init__.py b/packages/autorest.python/generator/pygen/codegen/__init__.py deleted file mode 100644 index ea4716f86e6..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/__init__.py +++ /dev/null @@ -1,334 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import logging -from typing import Dict, Any, Union, Optional -from pathlib import Path -import yaml - - -from .. import Plugin -from ..utils import parse_args -from .models.code_model import CodeModel -from .serializers import JinjaSerializer -from ._utils import DEFAULT_HEADER_TEXT, VALID_PACKAGE_MODE, TYPESPEC_PACKAGE_MODE - - -def _default_pprint(package_name: str) -> str: - return " ".join([i.capitalize() for i in package_name.split("-")]) - - -_LOGGER = logging.getLogger(__name__) - - -class OptionsRetriever: - OPTIONS_TO_DEFAULT = { - "azure-arm": False, - "flavor": "azure", # need to default to azure in shared code so we don't break swagger generation - "no-async": False, - "low-level-client": False, - "version-tolerant": True, - "keep-version-file": False, - "no-namespace-folders": False, - "basic-setup-py": False, - "client-side-validation": False, - "multiapi": False, - "polymorphic-examples": 5, - "generate-sample": False, - "generate-test": False, - "from-typespec": False, - "emit-cross-language-definition-file": False, - } - - @property - def is_azure_flavor(self) -> bool: - return self.flavor == "azure" - - def __init__(self, options: Dict[str, Any]) -> None: - self.options = options - - def __getattr__(self, prop: str) -> Any: - key = prop.replace("_", "-") - return self.options.get(key, self.OPTIONS_TO_DEFAULT.get(key)) - - @property - def company_name(self) -> str: - return self.options.get("company-name", "Microsoft" if self.is_azure_flavor else "") - - @property - def license_header(self) -> str: - license_header = self.options.get( - "header-text", - (DEFAULT_HEADER_TEXT.format(company_name=self.company_name) if self.company_name else ""), - ) - if license_header: - license_header = license_header.replace("\n", "\n# ") - license_header = ( - "# --------------------------------------------------------------------------\n# " + license_header - ) - license_header += "\n# --------------------------------------------------------------------------" - return license_header - - @property - def show_operations(self) -> bool: - return self.options.get("show-operations", not self.low_level_client) - - @property - def _models_mode_default(self) -> str: - models_mode_default = "none" if self.low_level_client or self.version_tolerant else "msrest" - if self.options.get("cadl_file") is not None: - models_mode_default = "dpg" - return models_mode_default - - @property - def original_models_mode(self) -> str: - return self.options.get("models-mode", self._models_mode_default) - - @property - def models_mode(self) -> Union[str, bool]: - # switch to falsy value for easier code writing - return False if self.original_models_mode == "none" else self.original_models_mode - - @property - def tracing(self) -> bool: - return self.options.get( - "tracing", - self.show_operations and self.is_azure_flavor, - ) - - @property - def show_send_request(self) -> bool: - return self.options.get( - "show-send-request", - self._low_level_or_version_tolerant, - ) - - @property - def _low_level_or_version_tolerant(self) -> bool: - return self.low_level_client or self.version_tolerant - - @property - def only_path_and_body_params_positional(self) -> bool: - return self.options.get( - "only-path-and-body-params-positional", - self._low_level_or_version_tolerant, - ) - - @property - def combine_operation_files(self) -> bool: - return self.options.get( - "combine-operation-files", - self.version_tolerant, - ) - - @property - def package_pprint_name(self) -> str: - return self.options.get("package-pprint-name") or _default_pprint(str(self.package_name)) - - @property - def default_optional_constants_to_none(self) -> bool: - return self.options.get( - "default-optional-constants-to-none", - self._low_level_or_version_tolerant, - ) - - @property - def builders_visibility(self) -> str: - builders_visibility = self.options.get("builders-visibility") - if builders_visibility is None: - return "public" if self.low_level_client else "embedded" - return builders_visibility.lower() - - @property - def head_as_boolean(self) -> bool: - head_as_boolean = self.options.get("head-as-boolean", True) - # Force some options in ARM MODE - return True if self.azure_arm else head_as_boolean - - @property - def package_mode(self) -> str: - return self.options.get("packaging-files-dir") or self.options.get("package-mode", "") - - @property - def packaging_files_config(self) -> Optional[Dict[str, Any]]: - packaging_files_config = self.options.get("packaging-files-config") - if packaging_files_config is None: - return None - # packaging-files-config is either a string or a dict - # if it's a string, we can split on the comma to get the dict - # otherwise we just return - try: - return {k.strip(): v.strip() for k, v in [i.split(":") for i in packaging_files_config.split("|")]} - except AttributeError: - return packaging_files_config - - -class CodeGenerator(Plugin): - def __init__(self, *args, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self.options_retriever = OptionsRetriever(self.options) - - def _validate_code_model_options(self) -> None: - if self.options_retriever.builders_visibility not in [ - "public", - "hidden", - "embedded", - ]: - raise ValueError("The value of --builders-visibility must be either 'public', 'hidden', or 'embedded'") - - if self.options_retriever.original_models_mode not in ["msrest", "dpg", "none"]: - raise ValueError( - "--models-mode can only be 'msrest', 'dpg' or 'none'. " - "Pass in 'msrest' if you want msrest models, or " - "'none' if you don't want any." - ) - - if not self.options_retriever.show_operations and self.options_retriever.builders_visibility == "embedded": - raise ValueError( - "Can not embed builders without operations. " - "Either set --show-operations to True, or change the value of --builders-visibility " - "to 'public' or 'hidden'." - ) - - if self.options_retriever.basic_setup_py and not self.options_retriever.package_version: - raise ValueError("--basic-setup-py must be used with --package-version") - - if self.options_retriever.package_mode and not self.options_retriever.package_version: - raise ValueError("--package-mode must be used with --package-version") - - if not self.options_retriever.show_operations and self.options_retriever.combine_operation_files: - raise ValueError( - "Can not combine operation files if you are not showing operations. " - "If you want operation files, pass in flag --show-operations" - ) - - if self.options_retriever.package_mode: - if ( - ( - self.options_retriever.package_mode not in TYPESPEC_PACKAGE_MODE - and self.options_retriever.from_typespec - ) - or ( - self.options_retriever.package_mode not in VALID_PACKAGE_MODE - and not self.options_retriever.from_typespec - ) - ) and not Path(self.options_retriever.package_mode).exists(): - raise ValueError( - f"--package-mode can only be {' or '.join(TYPESPEC_PACKAGE_MODE)} or directory which contains template files" # pylint: disable=line-too-long - ) - - if self.options_retriever.multiapi and self.options_retriever.version_tolerant: - raise ValueError( - "Can not currently generate version tolerant multiapi SDKs. " - "We are working on creating a new multiapi SDK for version tolerant and it is not available yet." - ) - - if self.options_retriever.client_side_validation and self.options_retriever.version_tolerant: - raise ValueError("Can not generate version tolerant with --client-side-validation. ") - - if not (self.options_retriever.azure_arm or self.options_retriever.version_tolerant): - _LOGGER.warning( - "You are generating with options that would not allow the SDK to be shipped as an official Azure SDK. " - "Please read https://aka.ms/azsdk/dpcodegen for more details." - ) - - if not self.options_retriever.is_azure_flavor and self.options_retriever.tracing: - raise ValueError("Can only have tracing turned on for Azure SDKs.") - - @staticmethod - def remove_cloud_errors(yaml_data: Dict[str, Any]) -> None: - for client in yaml_data["clients"]: - for group in client["operationGroups"]: - for operation in group["operations"]: - if not operation.get("exceptions"): - continue - i = 0 - while i < len(operation["exceptions"]): - exception = operation["exceptions"][i] - if ( - exception.get("schema") - and exception["schema"]["language"]["default"]["name"] == "CloudError" - ): - del operation["exceptions"][i] - i -= 1 - i += 1 - if yaml_data.get("schemas") and yaml_data["schemas"].get("objects"): - for i in range(len(yaml_data["schemas"]["objects"])): - obj_schema = yaml_data["schemas"]["objects"][i] - if obj_schema["language"]["default"]["name"] == "CloudError": - del yaml_data["schemas"]["objects"][i] - break - - def _build_code_model_options(self) -> Dict[str, Any]: - flags = [ - "azure_arm", - "head_as_boolean", - "license_header", - "keep_version_file", - "no_async", - "no_namespace_folders", - "basic_setup_py", - "package_name", - "package_version", - "client_side_validation", - "tracing", - "multiapi", - "polymorphic_examples", - "models_mode", - "builders_visibility", - "show_operations", - "show_send_request", - "only_path_and_body_params_positional", - "version_tolerant", - "low_level_client", - "combine_operation_files", - "package_mode", - "package_pprint_name", - "packaging_files_config", - "default_optional_constants_to_none", - "generate_sample", - "generate_test", - "default_api_version", - "from_typespec", - "flavor", - "company_name", - "emit_cross_language_definition_file", - ] - return {f: getattr(self.options_retriever, f) for f in flags} - - def get_yaml(self) -> Dict[str, Any]: - # cadl file doesn't have to be relative to output folder - with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd: - return yaml.safe_load(fd.read()) - - def get_serializer(self, code_model: CodeModel): - return JinjaSerializer(code_model, output_folder=self.output_folder) - - def process(self) -> bool: - # List the input file, should be only one - self._validate_code_model_options() - options = self._build_code_model_options() - yaml_data = self.get_yaml() - - if self.options_retriever.azure_arm: - self.remove_cloud_errors(yaml_data) - - code_model = CodeModel(yaml_data=yaml_data, options=options) - if not self.options_retriever.is_azure_flavor and any(client.lro_operations for client in code_model.clients): - raise ValueError("Only support LROs for Azure SDKs") - serializer = self.get_serializer(code_model) - serializer.serialize() - - return True - - -if __name__ == "__main__": - # CADL pipeline will call this - parsed_args, unknown_args = parse_args() - CodeGenerator( - output_folder=parsed_args.output_folder, - cadl_file=parsed_args.cadl_file, - **unknown_args, - ).process() diff --git a/packages/autorest.python/generator/pygen/codegen/_utils.py b/packages/autorest.python/generator/pygen/codegen/_utils.py deleted file mode 100644 index 38d881e8197..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/_utils.py +++ /dev/null @@ -1,16 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -DEFAULT_HEADER_TEXT = ( - "Copyright (c) {company_name} Corporation. All rights reserved.\n" - "Licensed under the MIT License. See License.txt in the project root for license information.\n" - "Code generated by {company_name} (R) Python Code Generator.\n" - "Changes may cause incorrect behavior and will be lost if the code is regenerated." -) - -SWAGGER_PACKAGE_MODE = ["mgmtplane", "dataplane"] # for backward compatibility -TYPESPEC_PACKAGE_MODE = ["azure-mgmt", "azure-dataplane", "generic"] -VALID_PACKAGE_MODE = SWAGGER_PACKAGE_MODE + TYPESPEC_PACKAGE_MODE diff --git a/packages/autorest.python/generator/pygen/codegen/models/__init__.py b/packages/autorest.python/generator/pygen/codegen/models/__init__.py deleted file mode 100644 index 7f9a8e0507c..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/__init__.py +++ /dev/null @@ -1,202 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import logging -from typing import Any, Dict, Union, Optional -from .base import BaseModel -from .base_builder import BaseBuilder, ParameterListType -from .code_model import CodeModel -from .client import Client -from .model_type import ModelType, JSONModelType, DPGModelType, MsrestModelType -from .dictionary_type import DictionaryType -from .list_type import ListType -from .combined_type import CombinedType -from .primitive_types import ( - ByteArraySchema, - DateType, - DatetimeType, - DurationType, - IntegerType, - FloatType, - StringType, - TimeType, - AnyType, - PrimitiveType, - BinaryType, - BooleanType, - AnyObjectType, - UnixTimeType, - SdkCoreType, - DecimalType, -) -from .enum_type import EnumType, EnumValue -from .base import BaseType -from .constant_type import ConstantType -from .imports import FileImport, ImportType, TypingSection -from .lro_operation import LROOperation -from .paging_operation import PagingOperation -from .parameter import ( - Parameter, - ParameterMethodLocation, - ParameterLocation, - BodyParameter, - ParameterDelimeter, - ClientParameter, - ConfigParameter, -) -from .operation import Operation -from .property import Property -from .operation_group import OperationGroup -from .response import Response -from .parameter_list import ( - ParameterList, - ClientGlobalParameterList, - ConfigGlobalParameterList, -) -from .request_builder import ( - RequestBuilder, - OverloadedRequestBuilder, - RequestBuilderBase, -) -from .lro_paging_operation import LROPagingOperation -from .request_builder_parameter import ( - RequestBuilderParameter, - RequestBuilderBodyParameter, -) -from .credential_types import ( - TokenCredentialType, - KeyCredentialType, - ARMChallengeAuthenticationPolicyType, - BearerTokenCredentialPolicyType, - KeyCredentialPolicyType, - CredentialType, -) - -__all__ = [ - "KeyCredentialPolicyType", - "AnyType", - "BaseModel", - "BaseType", - "CodeModel", - "Client", - "ConstantType", - "ModelType", - "DictionaryType", - "ListType", - "EnumType", - "EnumValue", - "FileImport", - "ImportType", - "TypingSection", - "PrimitiveType", - "LROOperation", - "Operation", - "PagingOperation", - "Parameter", - "ParameterList", - "OperationGroup", - "Property", - "RequestBuilder", - "Response", - "TokenCredentialType", - "LROPagingOperation", - "BaseBuilder", - "RequestBuilderParameter", - "BinaryType", - "ClientGlobalParameterList", - "ConfigGlobalParameterList", - "ParameterMethodLocation", - "ParameterLocation", - "OverloadedRequestBuilder", - "RequestBuilderBase", - "BodyParameter", - "RequestBuilderBodyParameter", - "ParameterDelimeter", - "CredentialType", - "ClientParameter", - "ConfigParameter", - "ParameterListType", -] - -TYPE_TO_OBJECT = { - "integer": IntegerType, - "float": FloatType, - "decimal": DecimalType, - "string": StringType, - "list": ListType, - "dict": DictionaryType, - "constant": ConstantType, - "enum": EnumType, - "enumvalue": EnumValue, - "binary": BinaryType, - "any": AnyType, - "utcDateTime": DatetimeType, - "offsetDateTime": DatetimeType, - "plainTime": TimeType, - "duration": DurationType, - "plainDate": DateType, - "bytes": ByteArraySchema, - "boolean": BooleanType, - "combined": CombinedType, - "OAuth2": TokenCredentialType, - "Key": KeyCredentialType, - "ARMChallengeAuthenticationPolicy": ARMChallengeAuthenticationPolicyType, - "BearerTokenCredentialPolicy": BearerTokenCredentialPolicyType, - "KeyCredentialPolicy": KeyCredentialPolicyType, - "any-object": AnyObjectType, - "unixtime": UnixTimeType, - "credential": StringType, - "sdkcore": SdkCoreType, -} -_LOGGER = logging.getLogger(__name__) - - -def build_type(yaml_data: Dict[str, Any], code_model: CodeModel) -> BaseType: - yaml_id = id(yaml_data) - try: - return code_model.lookup_type(yaml_id) - except KeyError: - # Not created yet, let's create it and add it to the index - pass - response: Optional[BaseType] = None - if yaml_data["type"] == "model": - # need to special case model to avoid recursion - if yaml_data["base"] == "json" or not code_model.options["models_mode"]: - model_type = JSONModelType - elif yaml_data["base"] == "dpg": - model_type = DPGModelType # type: ignore - else: - model_type = MsrestModelType # type: ignore - response = model_type(yaml_data, code_model) - code_model.types_map[yaml_id] = response - response.fill_instance_from_yaml(yaml_data, code_model) - elif yaml_data["type"] == "enum": - # avoid recursion because we add the parent enum type to the enum value - response = EnumType( - yaml_data, - code_model, - values=[], - value_type=build_type(yaml_data["valueType"], code_model), - ) - code_model.types_map[yaml_id] = response - response.fill_instance_from_yaml(yaml_data, code_model) - else: - object_type = yaml_data.get("type") - if object_type not in TYPE_TO_OBJECT: - _LOGGER.warning( - 'Unrecognized definition type "%s" is found, falling back it as "string"! ', - yaml_data["type"], - ) - object_type = "string" - response = TYPE_TO_OBJECT[object_type].from_yaml(yaml_data, code_model) # type: ignore - if response is None: - raise ValueError("response can not be None") - code_model.types_map[yaml_id] = response - return response - - -RequestBuilderType = Union[RequestBuilder, OverloadedRequestBuilder] -ParameterType = Union[Parameter, RequestBuilderParameter, ClientParameter, ConfigParameter] -OperationType = Union[Operation, LROOperation, PagingOperation, LROPagingOperation] diff --git a/packages/autorest.python/generator/pygen/codegen/models/base.py b/packages/autorest.python/generator/pygen/codegen/models/base.py deleted file mode 100644 index 7a44fb0b283..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/base.py +++ /dev/null @@ -1,188 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any, Dict, TYPE_CHECKING, List, Optional -from abc import ABC, abstractmethod -from .imports import FileImport - - -if TYPE_CHECKING: - from .code_model import CodeModel - from .model_type import ModelType - - -class BaseModel: - """This is the base class for model representations that are based on some YAML data. - - :param yaml_data: the yaml data for this schema - :type yaml_data: dict[str, Any] - """ - - def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - self.yaml_data = yaml_data - self.code_model = code_model - - @property - def id(self) -> int: - return id(self.yaml_data) - - def __repr__(self): - return f"<{self.__class__.__name__}>" - - -class BaseType(BaseModel, ABC): # pylint: disable=too-many-public-methods - """This is the base class for all types. - - :param yaml_data: the yaml data for this schema - :type yaml_data: dict[str, Any] - """ - - def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - super().__init__(yaml_data, code_model) - self.type = yaml_data["type"] # the type discriminator - self.api_versions: List[str] = yaml_data.get("apiVersions", []) # api versions this type is in. - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "BaseType": - return cls(yaml_data=yaml_data, code_model=code_model) - - def imports(self, **kwargs) -> FileImport: # pylint: disable=unused-argument - return FileImport(self.code_model) - - def imports_for_multiapi(self, **kwargs: Any) -> FileImport: - return self.imports(**kwargs) - - def imports_for_sample(self) -> FileImport: - return self.imports() - - @staticmethod - def serialize_sample_value(value: Any) -> str: - return repr(value) - - @property - def xml_metadata(self) -> Dict[str, Any]: - """XML metadata for the type, if the type has it.""" - return self.yaml_data.get("xmlMetadata", {}) - - @property - def is_xml(self) -> bool: - """Whether the type is an XML type or not. Most likely not.""" - return bool(self.xml_metadata) - - @property - def xml_serialization_ctxt(self) -> Optional[str]: - """Return the serialization context in case this schema is used in an operation.""" - attrs_list = [] - if self.xml_metadata.get("name"): - attrs_list.append(f"'name': '{self.xml_metadata['name']}'") - if self.xml_metadata.get("attribute", False): - attrs_list.append("'attr': True") - if self.xml_metadata.get("prefix", False): - attrs_list.append(f"'prefix': '{self.xml_metadata['prefix']}'") - if self.xml_metadata.get("namespace", False): - attrs_list.append(f"'ns': '{self.xml_metadata['namespace']}'") - if self.xml_metadata.get("text"): - attrs_list.append("'text': True") - return ", ".join(attrs_list) - - @property - def serialization_type(self) -> str: - """The tag recognized by 'msrest' as a serialization/deserialization. - - 'str', 'int', 'float', 'bool' or - https://github.com/Azure/msrest-for-python/blob/b505e3627b547bd8fdc38327e86c70bdb16df061/msrest/serialization.py#L407-L416 - - or the object schema name (e.g. DotSalmon). - - If list: '[str]' - If dict: '{str}' - """ - raise NotImplementedError() - - @property - def msrest_deserialization_key(self) -> str: - return self.serialization_type - - @property - def client_default_value(self) -> Any: - """Whether there's a client default value for this type""" - return self.yaml_data.get("clientDefaultValue") - - @abstractmethod - def description(self, *, is_operation_file: bool) -> str: - """The description""" - - @abstractmethod - def docstring_text(self, **kwargs: Any) -> str: - """The names used in rtype documentation""" - - @abstractmethod - def docstring_type(self, **kwargs: Any) -> str: - """The python type used for RST syntax input. - - Special case for enum, for instance: 'str or ~namespace.EnumName' - """ - - @abstractmethod - def type_annotation(self, **kwargs: Any) -> str: - """The python type used for type annotation - - Special case for enum, for instance: Union[str, "EnumName"] - """ - - @property - def validation(self) -> Optional[Dict[str, Any]]: - """Whether there's any validation constraints on this type. - - Even though we generate validation maps if there are validation constraints, - only SDKs with client-side-validate=true (0.001% libraries, if any) actually raise in this case. - """ - return None - - def get_declaration(self, value: Any) -> str: - """Return the current value from YAML as a Python string that represents the constant. - - Example, if schema is "bytearray" and value is "foo", - should return bytearray("foo", encoding="utf-8") - as a string. - - This is important for constant serialization. - - By default, return value, since it works sometimes (integer) - """ - return str(value) - - @abstractmethod - def get_json_template_representation( - self, - *, - optional: bool = True, - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - """Template of what this schema would look like as JSON input""" - - def get_polymorphic_subtypes( - self, polymorphic_subtypes: List["ModelType"] # pylint: disable=unused-argument - ) -> None: - return None - - @property - @abstractmethod - def instance_check_template(self) -> str: - """Template of what an instance check of a variable for this type would look like""" - - @property - def serialization_constraints(self) -> List[str]: - """Whether there are any serialization constraints when serializing this type.""" - return [] - - @property - def type_description(self) -> str: - return self.type_annotation() - - @property - def is_form_data(self) -> bool: - return False diff --git a/packages/autorest.python/generator/pygen/codegen/models/base_builder.py b/packages/autorest.python/generator/pygen/codegen/models/base_builder.py deleted file mode 100644 index 7bd5b2f4279..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/base_builder.py +++ /dev/null @@ -1,119 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import logging -from typing import ( - List, - Dict, - Any, - Generic, - TypeVar, - Optional, - Union, - TYPE_CHECKING, - cast, -) -from abc import abstractmethod - -from .base import BaseModel -from .parameter_list import ( - ParameterList, - RequestBuilderParameterList, - OverloadedRequestBuilderParameterList, -) - -ParameterListType = TypeVar( - "ParameterListType", - bound=Union[ - ParameterList, - RequestBuilderParameterList, - OverloadedRequestBuilderParameterList, - ], -) -if TYPE_CHECKING: - from .code_model import CodeModel - from .client import Client - from .operation import Operation - from .request_builder import RequestBuilder - - -OverloadListType = TypeVar("OverloadListType", bound=Union[List["Operation"], List["RequestBuilder"]]) - -_LOGGER = logging.getLogger(__name__) - - -class BaseBuilder( - Generic[ParameterListType, OverloadListType], BaseModel -): # pylint: disable=too-many-instance-attributes - """Base class for Operations and Request Builders""" - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - client: "Client", - name: str, - parameters: ParameterListType, - *, - overloads: Optional[OverloadListType] = None, - ) -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.client = client - self.name = name - self._description: str = yaml_data.get("description", "") - self.parameters = parameters - self.overloads = overloads or cast(OverloadListType, []) - self._summary: str = yaml_data.get("summary", "") - self.want_tracing: bool = yaml_data.get("wantTracing", True) - self.group_name: str = yaml_data["groupName"] # either operationGroup or client I am on - self.is_overload: bool = yaml_data["isOverload"] - self.api_versions: List[str] = yaml_data["apiVersions"] - self.added_on: Optional[str] = yaml_data.get("addedOn") - self.external_docs: Optional[Dict[str, Any]] = yaml_data.get("externalDocs") - - if code_model.options["version_tolerant"] and yaml_data.get("abstract"): - _LOGGER.warning( - 'Not going to generate operation "%s" because we are unable to generate this ' - "type of operation right now. " - 'Please write your own custom operation in the "_patch.py" file ' - "following https://aka.ms/azsdk/python/dpcodegen/python/customize", - name, - ) - self.abstract = True - else: - self.abstract = False - - @property - def summary(self) -> Optional[str]: - if self.abstract: - return None - return self._summary - - @property - def pylint_disable(self) -> str: - return "" - - @abstractmethod - def response_type_annotation(self, **kwargs) -> str: ... - - @abstractmethod - def response_docstring_text(self, **kwargs) -> str: ... - - @abstractmethod - def response_docstring_type(self, **kwargs) -> str: ... - - @property - def description(self) -> str: - if self.abstract: - return ( - f'You need to write a custom operation for "{self.name}". Please refer to ' - "https://aka.ms/azsdk/python/dpcodegen/python/customize to learn how to customize." - ) - return self._description or self.name - - def method_signature(self, async_mode: bool) -> List[str]: - if self.abstract: - return ["*args,", "**kwargs"] - return self.parameters.method_signature(async_mode) diff --git a/packages/autorest.python/generator/pygen/codegen/models/client.py b/packages/autorest.python/generator/pygen/codegen/models/client.py deleted file mode 100644 index 953ad7d63d8..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/client.py +++ /dev/null @@ -1,422 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any, Dict, TYPE_CHECKING, TypeVar, Generic, Union, List, Optional - -from .base import BaseModel -from .parameter_list import ClientGlobalParameterList, ConfigGlobalParameterList -from .imports import FileImport, ImportType, TypingSection, MsrestImportType -from .utils import add_to_pylint_disable, NAME_LENGTH_LIMIT -from .operation_group import OperationGroup -from .request_builder import ( - RequestBuilder, - OverloadedRequestBuilder, - get_request_builder, -) -from .parameter import Parameter, ParameterMethodLocation -from .lro_operation import LROOperation -from .lro_paging_operation import LROPagingOperation - -ParameterListType = TypeVar( - "ParameterListType", - bound=Union[ClientGlobalParameterList, ConfigGlobalParameterList], -) - -if TYPE_CHECKING: - from .code_model import CodeModel - from . import OperationType - - -class _ClientConfigBase(Generic[ParameterListType], BaseModel): - """The service client base. Shared across our Client and Config type""" - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - parameters: ParameterListType, - ): - super().__init__(yaml_data, code_model) - self.parameters = parameters - self.url: str = self.yaml_data["url"] # the base endpoint of the client. Can be parameterized or not - self.legacy_filename: str = self.yaml_data.get("legacyFilename", "client") - - @property - def description(self) -> str: - return self.yaml_data["description"] - - @property - def name(self) -> str: - return self.yaml_data["name"] - - -class Client(_ClientConfigBase[ClientGlobalParameterList]): - """Model representing our service client""" - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - parameters: ClientGlobalParameterList, - *, - is_subclient: bool = False, - ): - super().__init__(yaml_data, code_model, parameters) - self.operation_groups: List[OperationGroup] = [] - self.config = Config.from_yaml(yaml_data, self.code_model) - self.is_subclient = is_subclient - self.request_builders = self._build_request_builders() - if self.code_model.options["show_operations"]: - self.operation_groups = [ - OperationGroup.from_yaml(op_group, code_model, self) - for op_group in self.yaml_data.get("operationGroups", []) - ] - self.link_lro_initial_operations() - self.request_id_header_name = self.yaml_data.get("requestIdHeaderName", None) - self.has_etag: bool = yaml_data.get("hasEtag", False) - - def _build_request_builders( - self, - ) -> List[Union[RequestBuilder, OverloadedRequestBuilder]]: - request_builders: List[Union[RequestBuilder, OverloadedRequestBuilder]] = [] - - def add_og_request_builder(og: Dict[str, Any]): - for operation_yaml in og["operations"]: - request_builder = get_request_builder( - operation_yaml, - code_model=self.code_model, - client=self, - ) - if operation_yaml.get("isLroInitialOperation"): - # we want to change the name - request_builder.name = request_builder.get_name( - request_builder.yaml_data["name"][1 : -len("_initial")], - request_builder.yaml_data, - request_builder.code_model, - request_builder.client, - ) - if request_builder.overloads: - request_builders.extend(request_builder.overloads) - request_builders.append(request_builder) - if operation_yaml.get("nextOperation"): - # i am a paging operation and i have a next operation. - # Make sure to include my next operation - request_builders.append( - get_request_builder( - operation_yaml["nextOperation"], - code_model=self.code_model, - client=self, - ) - ) - - queue = self.yaml_data["operationGroups"].copy() - while queue: - now = queue.pop(0) - add_og_request_builder(now) - if now.get("operationGroups"): - queue.extend(now["operationGroups"]) - - return request_builders - - def pipeline_class(self, async_mode: bool) -> str: - if self.code_model.options["azure_arm"]: - if async_mode: - return "AsyncARMPipelineClient" - return "ARMPipelineClient" - if async_mode: - return "AsyncPipelineClient" - return "PipelineClient" - - @property - def credential(self) -> Optional[Parameter]: - """The credential param, if one exists""" - return self.parameters.credential - - @property - def send_request_name(self) -> str: - """Name of the send request function""" - return "send_request" if self.code_model.options["show_send_request"] else "_send_request" - - @property - def has_parameterized_host(self) -> bool: - """Whether the base url is parameterized or not""" - return not any(p for p in self.parameters if p.is_host) - - @property - def pylint_disable(self) -> str: - retval = add_to_pylint_disable("", "client-accepts-api-version-keyword") - if len(self.operation_groups) > 6: - retval = add_to_pylint_disable(retval, "too-many-instance-attributes") - if len(self.name) > NAME_LENGTH_LIMIT: - retval = add_to_pylint_disable(retval, "name-too-long") - return retval - - @property - def url_pylint_disable(self) -> str: - # if the url is too long - retval = "" - if len(self.url) > 85: - retval = add_to_pylint_disable(retval, "line-too-long") - return retval - - @property - def filename(self) -> str: - """Name of the file for the client""" - if self.code_model.options["version_tolerant"] or self.code_model.options["low_level_client"]: - return "_client" - return f"_{self.legacy_filename}" - - def lookup_request_builder(self, request_builder_id: int) -> Union[RequestBuilder, OverloadedRequestBuilder]: - """Find the request builder based off of id""" - try: - return next(rb for rb in self.request_builders if id(rb.yaml_data) == request_builder_id) - except StopIteration as exc: - raise KeyError(f"No request builder with id {request_builder_id} found.") from exc - - def lookup_operation(self, operation_id: int) -> "OperationType": - try: - return next(o for og in self.operation_groups for o in og.operations if id(o.yaml_data) == operation_id) - except StopIteration as exc: - raise KeyError(f"No operation with id {operation_id} found.") from exc - - def _imports_shared(self, async_mode: bool) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) - if self.code_model.options["azure_arm"]: - file_import.add_submodule_import("azure.mgmt.core", self.pipeline_class(async_mode), ImportType.SDKCORE) - else: - file_import.add_submodule_import( - "" if self.code_model.is_azure_flavor else "runtime", - self.pipeline_class(async_mode), - ImportType.SDKCORE, - ) - - for gp in self.parameters: - if gp.method_location == ParameterMethodLocation.KWARG: - continue - file_import.merge( - gp.imports( - async_mode, - relative_path=".." if async_mode else ".", - operation=True, - ) - ) - file_import.add_submodule_import( - "._configuration", - f"{self.name}Configuration", - ImportType.LOCAL, - ) - file_import.add_msrest_import( - relative_path=".." if async_mode else ".", - msrest_import_type=MsrestImportType.SerializerDeserializer, - typing_section=TypingSection.REGULAR, - ) - file_import.add_submodule_import( - "pipeline" if self.code_model.is_azure_flavor else "runtime", - "policies", - ImportType.SDKCORE, - ) - if self.code_model.options["azure_arm"]: - async_prefix = "Async" if async_mode else "" - file_import.add_submodule_import( - "azure.mgmt.core.policies", - f"{async_prefix}ARMAutoResourceProviderRegistrationPolicy", - ImportType.SDKCORE, - ) - return file_import - - @property - def has_mixin(self) -> bool: - """Do we want a mixin ABC class for typing purposes?""" - return any(og for og in self.operation_groups if og.is_mixin) - - @property - def lro_operations(self) -> List["OperationType"]: - """all LRO operations in this SDK?""" - return [operation for operation_group in self.operation_groups for operation in operation_group.lro_operations] - - @property - def has_public_lro_operations(self) -> bool: - """Are there any public LRO operations in this SDK?""" - return any(not operation.internal for operation in self.lro_operations) - - @property - def has_operations(self) -> bool: - return any(operation_group.has_operations for operation_group in self.operation_groups) - - def link_lro_initial_operations(self) -> None: - """Link each LRO operation to its initial operation""" - for operation_group in self.operation_groups: - for operation in operation_group.operations: - if isinstance(operation, (LROOperation, LROPagingOperation)): - operation.initial_operation = self.lookup_operation(id(operation.yaml_data["initialOperation"])) - - @property - def has_abstract_operations(self) -> bool: - """Whether there is abstract operation in any operation group.""" - return any(og.has_abstract_operations for og in self.operation_groups) - - @property - def has_non_abstract_operations(self) -> bool: - """Whether there is non-abstract operation in any operation group.""" - return any(og.has_non_abstract_operations for og in self.operation_groups) - - def imports(self, async_mode: bool) -> FileImport: - file_import = self._imports_shared(async_mode) - if async_mode: - file_import.add_submodule_import("typing", "Awaitable", ImportType.STDLIB) - file_import.add_submodule_import( - "rest", - "AsyncHttpResponse", - ImportType.SDKCORE, - TypingSection.CONDITIONAL, - ) - else: - file_import.add_submodule_import( - "rest", - "HttpResponse", - ImportType.SDKCORE, - TypingSection.CONDITIONAL, - ) - file_import.add_submodule_import( - "rest", - "HttpRequest", - ImportType.SDKCORE, - TypingSection.CONDITIONAL, - ) - for og in self.operation_groups: - file_import.add_submodule_import( - f".{self.code_model.operations_folder_name}", - og.class_name, - ImportType.LOCAL, - ) - - if self.code_model.model_types and self.code_model.options["models_mode"] == "msrest": - path_to_models = ".." if async_mode else "." - file_import.add_submodule_import(path_to_models, "models", ImportType.LOCAL, alias="_models") - elif self.code_model.options["models_mode"] == "msrest": - # in this case, we have client_models = {} in the service client, which needs a type annotation - # this import will always be commented, so will always add it to the typing section - file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) - file_import.add_submodule_import("copy", "deepcopy", ImportType.STDLIB) - return file_import - - def imports_for_multiapi(self, async_mode: bool) -> FileImport: - file_import = self._imports_shared(async_mode) - file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB, TypingSection.CONDITIONAL) - try: - mixin_operation = next(og for og in self.operation_groups if og.is_mixin) - file_import.add_submodule_import("._operations_mixin", mixin_operation.class_name, ImportType.LOCAL) - except StopIteration: - pass - file_import.add_submodule_import("azure.profiles", "KnownProfiles", import_type=ImportType.SDKCORE) - file_import.add_submodule_import("azure.profiles", "ProfileDefinition", import_type=ImportType.SDKCORE) - file_import.add_submodule_import( - "azure.profiles.multiapiclient", - "MultiApiClientMixin", - import_type=ImportType.SDKCORE, - ) - return file_import - - @classmethod - def from_yaml( - cls, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - *, - is_subclient: bool = False, - ) -> "Client": - return cls( - yaml_data=yaml_data, - code_model=code_model, - parameters=ClientGlobalParameterList.from_yaml(yaml_data, code_model), - is_subclient=is_subclient, - ) - - -class Config(_ClientConfigBase[ConfigGlobalParameterList]): - """Model representing our Config type.""" - - @property - def pylint_disable(self) -> str: - retval = add_to_pylint_disable("", "too-many-instance-attributes") - if len(self.name) + len("Configuration") > NAME_LENGTH_LIMIT: - retval = add_to_pylint_disable(retval, "name-too-long") - return retval - - @property - def description(self) -> str: - return ( - f"Configuration for {self.yaml_data['name']}.\n\n." - "Note that all parameters used to create this instance are saved as instance attributes." - ) - - @property - def sdk_moniker(self) -> str: - package_name = self.code_model.options["package_name"] - if package_name and package_name.startswith("azure-"): - package_name = package_name[len("azure-") :] - return package_name if package_name else self.yaml_data["name"].lower() - - @property - def name(self) -> str: - return f"{super().name}Configuration" - - def _imports_shared(self, async_mode: bool) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_submodule_import( - "pipeline" if self.code_model.is_azure_flavor else "runtime", - "policies", - ImportType.SDKCORE, - ) - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) - if self.code_model.options["package_version"]: - file_import.add_submodule_import(".._version" if async_mode else "._version", "VERSION", ImportType.LOCAL) - if self.code_model.options["azure_arm"]: - policy = "AsyncARMChallengeAuthenticationPolicy" if async_mode else "ARMChallengeAuthenticationPolicy" - file_import.add_submodule_import("azure.mgmt.core.policies", "ARMHttpLoggingPolicy", ImportType.SDKCORE) - file_import.add_submodule_import("azure.mgmt.core.policies", policy, ImportType.SDKCORE) - - return file_import - - def imports(self, async_mode: bool) -> FileImport: - file_import = self._imports_shared(async_mode) - for gp in self.parameters: - if gp.method_location == ParameterMethodLocation.KWARG and gp not in self.parameters.kwargs_to_pop: - continue - file_import.merge( - gp.imports( - async_mode=async_mode, - relative_path=".." if async_mode else ".", - operation=True, - ) - ) - return file_import - - def imports_for_multiapi(self, async_mode: bool) -> FileImport: - file_import = self._imports_shared(async_mode) - for gp in self.parameters: - if ( - gp.method_location == ParameterMethodLocation.KWARG - and gp not in self.parameters.kwargs_to_pop - and gp.client_name == "api_version" - ): - continue - file_import.merge( - gp.imports_for_multiapi( - async_mode=async_mode, - relative_path=".." if async_mode else ".", - operation=True, - ) - ) - return file_import - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "Config": - return cls( - yaml_data=yaml_data, - code_model=code_model, - parameters=ConfigGlobalParameterList.from_yaml(yaml_data, code_model), - ) diff --git a/packages/autorest.python/generator/pygen/codegen/models/code_model.py b/packages/autorest.python/generator/pygen/codegen/models/code_model.py deleted file mode 100644 index c94ec77c2b7..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/code_model.py +++ /dev/null @@ -1,245 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import List, Dict, Any, Set, Union, Literal - -from .base import BaseType -from .enum_type import EnumType -from .model_type import ModelType -from .combined_type import CombinedType -from .client import Client -from .request_builder import RequestBuilder, OverloadedRequestBuilder - - -def _is_legacy(options) -> bool: - return not (options.get("version_tolerant") or options.get("low_level_client")) - - -class CodeModel: # pylint: disable=too-many-public-methods, disable=too-many-instance-attributes - """Top level code model - - :param options: Options of the code model. I.e., whether this is for management generation - :type options: dict[str, bool] - :param str module_name: The module name for the client. Is in snake case. - :param str class_name: The class name for the client. Is in pascal case. - :param str description: The description of the client - :param str namespace: The namespace of our module - :param schemas: The list of schemas we are going to serialize in the models files. Maps their yaml - id to our created ModelType. - :type schemas: dict[int, ~autorest.models.ModelType] - :param sorted_schemas: Our schemas in order by inheritance and alphabet - :type sorted_schemas: list[~autorest.models.ModelType] - :param enums: The enums, if any, we are going to serialize. Maps their yaml id to our created EnumType. - :type enums: Dict[int, ~autorest.models.EnumType] - :param primitives: List of schemas we've created that are not EnumSchemas or ObjectSchemas. Maps their - yaml id to our created schemas. - :type primitives: Dict[int, ~autorest.models.BaseType] - :param package_dependency: All the dependencies needed in setup.py - :type package_dependency: Dict[str, str] - """ - - def __init__( - self, - yaml_data: Dict[str, Any], - options: Dict[str, Any], - *, - is_subnamespace: bool = False, - ) -> None: - self.yaml_data = yaml_data - self.options = options - self.namespace = self.yaml_data["namespace"] - self.types_map: Dict[int, BaseType] = {} # map yaml id to schema - self._model_types: List[ModelType] = [] - from . import build_type - - for type_yaml in yaml_data.get("types", []): - build_type(yaml_data=type_yaml, code_model=self) - self.clients: List[Client] = [ - Client.from_yaml(client_yaml_data, self) for client_yaml_data in yaml_data["clients"] - ] - self.subnamespace_to_clients: Dict[str, List[Client]] = { - subnamespace: [Client.from_yaml(client_yaml, self, is_subclient=True) for client_yaml in client_yamls] - for subnamespace, client_yamls in yaml_data.get("subnamespaceToClients", {}).items() - } - if self.options["models_mode"] and self.model_types: - self.sort_model_types() - self.is_subnamespace = is_subnamespace - self.named_unions: List[CombinedType] = [ - t for t in self.types_map.values() if isinstance(t, CombinedType) and t.name - ] - self.cross_language_package_id = self.yaml_data.get("crossLanguagePackageId") - self.for_test: bool = False - - @property - def has_form_data(self) -> bool: - return any(og.has_form_data_body for client in self.clients for og in client.operation_groups) - - @property - def has_etag(self) -> bool: - return any(client.has_etag for client in self.clients) - - @property - def has_operations(self) -> bool: - if any(c for c in self.clients if c.has_operations): - return True - return any(c for clients in self.subnamespace_to_clients.values() for c in clients if c.has_operations) - - @property - def has_non_abstract_operations(self) -> bool: - return any(c for c in self.clients if c.has_non_abstract_operations) or any( - c for cs in self.subnamespace_to_clients.values() for c in cs if c.has_non_abstract_operations - ) - - def lookup_request_builder(self, request_builder_id: int) -> Union[RequestBuilder, OverloadedRequestBuilder]: - """Find the request builder based off of id""" - for client in self.clients: - try: - return client.lookup_request_builder(request_builder_id) - except KeyError: - pass - raise KeyError(f"No request builder with id {request_builder_id} found.") - - @property - def is_azure_flavor(self) -> bool: - return self.options["flavor"] == "azure" - - @property - def rest_layer_name(self) -> str: - """If we have a separate rest layer, what is its name?""" - return "rest" if self.options["builders_visibility"] == "public" else "_rest" - - @property - def client_filename(self) -> str: - return self.clients[0].filename - - def need_vendored_code(self, async_mode: bool) -> bool: - """Whether we need to vendor code in the _vendor.py file for this SDK""" - if self.has_abstract_operations: - return True - if async_mode: - return self.need_mixin_abc - return self.need_mixin_abc or self.has_etag or self.has_form_data - - @property - def need_mixin_abc(self) -> bool: - return any(c for c in self.clients if c.has_mixin) - - @property - def has_abstract_operations(self) -> bool: - return any(c for c in self.clients if c.has_abstract_operations) - - @property - def operations_folder_name(self) -> str: - """Get the name of the operations folder that holds operations.""" - name = "operations" - if self.options["version_tolerant"] and not any( - og for client in self.clients for og in client.operation_groups if not og.is_mixin - ): - name = f"_{name}" - return name - - @property - def description(self) -> str: - return self.clients[0].description - - def lookup_type(self, schema_id: int) -> BaseType: - """Looks to see if the schema has already been created. - - :param int schema_id: The yaml id of the schema - :return: If created, we return the created schema, otherwise, we throw. - :rtype: ~autorest.models.BaseType - :raises: KeyError if schema is not found - """ - try: - return next(type for id, type in self.types_map.items() if id == schema_id) - except StopIteration as exc: - raise KeyError(f"Couldn't find schema with id {schema_id}") from exc - - @property - def model_types(self) -> List[ModelType]: - """All of the model types in this class""" - if not self._model_types: - self._model_types = [ - t - for t in self.types_map.values() - if isinstance(t, ModelType) and not (self.options["models_mode"] == "dpg" and t.page_result_model) - ] - return self._model_types - - @model_types.setter - def model_types(self, val: List[ModelType]) -> None: - self._model_types = val - - @property - def public_model_types(self) -> List[ModelType]: - return [m for m in self.model_types if not m.internal and not m.base == "json"] - - @property - def enums(self) -> List[EnumType]: - """All of the enums""" - return [t for t in self.types_map.values() if isinstance(t, EnumType)] - - @property - def core_library(self) -> Literal["azure.core", "corehttp"]: - return "azure.core" if self.is_azure_flavor else "corehttp" - - def _sort_model_types_helper( - self, - current: ModelType, - seen_schema_names: Set[str], - seen_schema_yaml_ids: Set[int], - ): - if current.id in seen_schema_yaml_ids: - return [] - if current.name in seen_schema_names: - raise ValueError(f"We have already generated a schema with name {current.name}") - ancestors = [current] - if current.parents: - for parent in current.parents: - if parent.id in seen_schema_yaml_ids: - continue - seen_schema_names.add(current.name) - seen_schema_yaml_ids.add(current.id) - ancestors = self._sort_model_types_helper(parent, seen_schema_names, seen_schema_yaml_ids) + ancestors - seen_schema_names.add(current.name) - seen_schema_yaml_ids.add(current.id) - return ancestors - - def sort_model_types(self) -> None: - """Sorts the final object schemas by inheritance and by alphabetical order. - - :return: None - :rtype: None - """ - seen_schema_names: Set[str] = set() - seen_schema_yaml_ids: Set[int] = set() - sorted_object_schemas: List[ModelType] = [] - for schema in sorted(self.model_types, key=lambda x: x.name.lower()): - sorted_object_schemas.extend(self._sort_model_types_helper(schema, seen_schema_names, seen_schema_yaml_ids)) - self.model_types = sorted_object_schemas - - @property - def models_filename(self) -> str: - """Get the names of the model file(s)""" - if self.is_legacy: - return "_models_py3" - return "_models" - - @property - def enums_filename(self) -> str: - """The name of the enums file""" - if self.is_legacy: - return f"_{self.clients[0].legacy_filename}_enums" - return "_enums" - - @property - def is_legacy(self) -> bool: - return _is_legacy(self.options) - - @property - def need_typing_extensions(self) -> bool: - if self.options["models_mode"] == "dpg": - return True - return False diff --git a/packages/autorest.python/generator/pygen/codegen/models/combined_type.py b/packages/autorest.python/generator/pygen/codegen/models/combined_type.py deleted file mode 100644 index 8b6fc261c26..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/combined_type.py +++ /dev/null @@ -1,153 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any, Dict, List, Optional, TYPE_CHECKING, Type, Tuple, Union -import re -from .imports import FileImport, ImportType, TypingSection -from .base import BaseType -from .model_type import ModelType - -if TYPE_CHECKING: - from .code_model import CodeModel - - -class CombinedType(BaseType): - """A type that consists of multiple different types. - - Used by body parameters that have multiple types, i.e. one that can be - a stream body or a JSON body. - """ - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - types: List[BaseType], - ) -> None: - super().__init__(yaml_data, code_model) - self.types = types # the types that this type is combining - self.name = yaml_data.get("name") - self._is_union_of_literals = all(i.type == "constant" for i in self.types) - - @property - def serialization_type(self) -> str: - """The tag recognized by 'msrest' as a serialization/deserialization. - - 'str', 'int', 'float', 'bool' or - https://github.com/Azure/msrest-for-python/blob/b505e3627b547bd8fdc38327e86c70bdb16df061/msrest/serialization.py#L407-L416 - - or the object schema name (e.g. DotSalmon). - - If list: '[str]' - If dict: '{str}' - """ - if not all(t for t in self.types if t.type == "constant"): - raise ValueError("Shouldn't get serialization type of a combinedtype") - return self.types[0].serialization_type - - @property - def client_default_value(self) -> Any: - return self.yaml_data.get("clientDefaultValue") - - def description(self, *, is_operation_file: bool) -> str: # pylint: disable=unused-argument - if len(self.types) == 2: - return f"Is either a {self.types[0].type_description} type or a {self.types[1].type_description} type." - return f"Is one of the following types: {', '.join([t.type_description for t in self.types])}" - - def docstring_text(self, **kwargs: Any) -> str: - return " or ".join(t.docstring_text(**kwargs) for t in self.types) - - def docstring_type(self, **kwargs: Any) -> str: - return " or ".join(t.docstring_type(**kwargs) for t in self.types) - - def type_annotation(self, **kwargs: Any) -> str: - if self.name: - return f'"_types.{self.name}"' - return self.type_definition(**kwargs) - - def type_definition(self, **kwargs: Any) -> str: - """The python type used for type annotation - - Special case for enum, for instance: Union[str, "EnumName"] - """ - # remove duplicates - inside_types = list(dict.fromkeys([type.type_annotation(**kwargs) for type in self.types])) - if len(inside_types) == 1: - return inside_types[0] - if self._is_union_of_literals: - parsed_values = [] - for entry in inside_types: - match = re.search(r"Literal\[(.*)\]", entry) - if match is not None: - parsed_values.append(match.group(1)) - join_string = ", ".join(parsed_values) - return f"Literal[{join_string}]" - - # If the inside types has been a Union, peel first and then re-union - pattern = re.compile(r"Union\[.*\]") - return f'Union[{", ".join(map(lambda x: x[6: -1] if pattern.match(x) else x, inside_types))}]' - - @property - def is_form_data(self) -> bool: - return any(t.is_form_data for t in self.types) - - def get_json_template_representation( - self, - *, - optional: bool = True, - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - return self.types[0].get_json_template_representation( - optional=optional, - client_default_value_declaration=client_default_value_declaration, - description=description, - ) - - def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: - raise ValueError("You shouldn't get polymorphic subtypes of multiple types") - - @property - def instance_check_template(self) -> str: - """Template of what an instance check of a variable for this type would look like""" - raise ValueError("You shouldn't do instance checks on a multiple type") - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - if self.name and not kwargs.get("is_types_file"): - file_import.add_submodule_import( - kwargs.pop("relative_path"), - "_types", - ImportType.LOCAL, - TypingSection.TYPING, - ) - return file_import - for type in self.types: - file_import.merge(type.imports(**kwargs)) - if not self._is_union_of_literals: - file_import.add_submodule_import("typing", "Union", ImportType.STDLIB) - return file_import - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "BaseType": - from . import build_type - - return cls( - yaml_data, - code_model, - [build_type(t, code_model) for t in yaml_data["types"]], - ) - - def target_model_subtype( - self, - target_types: Union[ - Tuple[Type[ModelType]], - Tuple[Type[ModelType], Type[ModelType]], - ], - ) -> Optional[ModelType]: - for sub_t in self.types: - if isinstance(sub_t, target_types): - return sub_t # type: ignore - return None diff --git a/packages/autorest.python/generator/pygen/codegen/models/constant_type.py b/packages/autorest.python/generator/pygen/codegen/models/constant_type.py deleted file mode 100644 index 16be177de4c..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/constant_type.py +++ /dev/null @@ -1,134 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import logging -from typing import Dict, Any, Optional, TYPE_CHECKING, Union -from .base import BaseType -from .imports import FileImport, ImportType, TypingSection -from .primitive_types import IntegerType, BinaryType, StringType, BooleanType -from .utils import add_to_description - -if TYPE_CHECKING: - from .code_model import CodeModel - -_LOGGER = logging.getLogger(__name__) - - -class ConstantType(BaseType): - """Schema for constants that will be serialized. - - :param yaml_data: the yaml data for this schema - :type yaml_data: dict[str, Any] - :param str value: The actual value of this constant. - :param schema: The schema for the value of this constant. - :type schema: ~autorest.models.PrimitiveType - """ - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - value_type: BaseType, - value: Optional[Union[str, int, float]], - ) -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.value_type = value_type - self.value = value - - def get_declaration(self, value=None): - if value and value != self.value: - _LOGGER.warning( - "Passed in value of %s differs from constant value of %s. Choosing constant value", - str(value), - str(self.value), - ) - if self.value is None: - return "None" - return self.value_type.get_declaration(self.value) - - def description(self, *, is_operation_file: bool) -> str: - if is_operation_file: - return "" - return add_to_description( - self.yaml_data.get("description", ""), - f"Default value is {self.get_declaration()}.", - ) - - @property - def serialization_type(self) -> str: - """Returns the serialization value for msrest. - - :return: The serialization value for msrest - :rtype: str - """ - return self.value_type.serialization_type - - def docstring_text(self, **kwargs: Any) -> str: - return "constant" - - def docstring_type(self, **kwargs: Any) -> str: - """The python type used for RST syntax input and type annotation. - - :param str namespace: Optional. The namespace for the models. - """ - return self.value_type.docstring_type(**kwargs) - - def type_annotation(self, **kwargs: Any) -> str: - return f"Literal[{self.get_declaration()}]" if self._is_literal else self.value_type.type_annotation(**kwargs) - - @property - def _is_literal(self) -> bool: - return isinstance(self.value_type, (IntegerType, BinaryType, StringType, BooleanType)) - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "ConstantType": - """Constructs a ConstantType from yaml data. - - :param yaml_data: the yaml data from which we will construct this schema - :type yaml_data: dict[str, Any] - - :return: A created ConstantType - :rtype: ~autorest.models.ConstantType - """ - from . import build_type - - return cls( - yaml_data=yaml_data, - code_model=code_model, - value_type=build_type(yaml_data["valueType"], code_model), - value=yaml_data["value"], - ) - - def get_json_template_representation( - self, - *, - optional: bool = True, - # pylint: disable=unused-argument - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - return self.value_type.get_json_template_representation( - optional=optional, - client_default_value_declaration=self.get_declaration(), - description=description, - ) - - def _imports_shared(self, **kwargs: Any): - file_import = super().imports(**kwargs) - file_import.merge(self.value_type.imports(**kwargs)) - return file_import - - def imports_for_multiapi(self, **kwargs: Any) -> FileImport: - return self._imports_shared(**kwargs) - - def imports(self, **kwargs: Any) -> FileImport: - file_import = self._imports_shared(**kwargs) - if self._is_literal: - file_import.add_submodule_import("typing", "Literal", ImportType.STDLIB, TypingSection.REGULAR) - return file_import - - @property - def instance_check_template(self) -> str: - return self.value_type.instance_check_template diff --git a/packages/autorest.python/generator/pygen/codegen/models/credential_types.py b/packages/autorest.python/generator/pygen/codegen/models/credential_types.py deleted file mode 100644 index 261cad0e228..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/credential_types.py +++ /dev/null @@ -1,223 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from abc import abstractmethod -from typing import ( - Optional, - Any, - Dict, - TYPE_CHECKING, - List, - Generic, - TypeVar, - Union, - cast, -) - -from .imports import FileImport, ImportType, TypingSection -from .base import BaseType - -if TYPE_CHECKING: - from .code_model import CodeModel - - -class _CredentialPolicyBaseType: - """Base class for our different credential policy types. - - Inherited by our BearerTokenCredentialPolicy and KeyCredentialPolicy types. - """ - - def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - self.yaml_data = yaml_data - self.code_model = code_model - - @abstractmethod - def call(self, async_mode: bool) -> str: - """ - How to call this credential policy. Used to initialize the credential policy in the config file. - """ - - -class BearerTokenCredentialPolicyType(_CredentialPolicyBaseType): - """Credential policy type representing BearerTokenCredentialPolicy""" - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - credential_scopes: List[str], - ) -> None: - super().__init__(yaml_data, code_model) - self.credential_scopes = credential_scopes - - def call(self, async_mode: bool) -> str: - policy_name = f"{'Async' if async_mode else ''}BearerTokenCredentialPolicy" - return f"policies.{policy_name}(self.credential, *self.credential_scopes, **kwargs)" - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "BearerTokenCredentialPolicyType": - return cls(yaml_data, code_model, yaml_data["credentialScopes"]) - - -class ARMChallengeAuthenticationPolicyType(BearerTokenCredentialPolicyType): - """Credential policy type representing ARMChallengeAuthenticationPolicy""" - - def call(self, async_mode: bool) -> str: - policy_name = f"{'Async' if async_mode else ''}ARMChallengeAuthenticationPolicy" - return f"{policy_name}(self.credential, *self.credential_scopes, **kwargs)" - - -class KeyCredentialPolicyType(_CredentialPolicyBaseType): - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - key: str, - scheme: Optional[str] = None, - ) -> None: - super().__init__(yaml_data, code_model) - self.key = key - self.scheme = scheme - - @property - def credential_name(self) -> str: - return "AzureKeyCredential" if self.code_model.is_azure_flavor else "ServiceKeyCredential" - - def call(self, async_mode: bool) -> str: - params = f'"{self.key}", ' - if self.scheme: - params += f'prefix="{self.scheme}", ' - return f"policies.{self.credential_name}Policy(self.credential, {params}**kwargs)" - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "KeyCredentialPolicyType": - return cls(yaml_data, code_model, yaml_data["key"], yaml_data.get("scheme", None)) - - -CredentialPolicyType = TypeVar( - "CredentialPolicyType", - bound=Union[ - BearerTokenCredentialPolicyType, - ARMChallengeAuthenticationPolicyType, - KeyCredentialPolicyType, - ], -) - - -class CredentialType(Generic[CredentialPolicyType], BaseType): # pylint:disable=abstract-method - """Store info about the type of the credential. Can be either an KeyCredential or a TokenCredential""" - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - policy: CredentialPolicyType, - ) -> None: - super().__init__(yaml_data, code_model) - self.policy = policy - - def description(self, *, is_operation_file: bool) -> str: # pylint: disable=unused-argument - return "" - - def get_json_template_representation( - self, - *, - optional: bool = True, - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - raise TypeError("You should not try to get a JSON template representation of a CredentialSchema") - - def docstring_text(self, **kwargs: Any) -> str: - return "credential" - - @property - def serialization_type(self) -> str: - return self.docstring_type() - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "CredentialType": - from . import build_type - - return cls( - yaml_data, - code_model, - policy=cast(CredentialPolicyType, build_type(yaml_data["policy"], code_model)), - ) - - -class TokenCredentialType( - CredentialType[ # pylint: disable=unsubscriptable-object - Union[BearerTokenCredentialPolicyType, ARMChallengeAuthenticationPolicyType] - ] -): - """Type of a token credential. Used by BearerAuth and ARMChallenge policies""" - - def type_annotation(self, **kwargs: Any) -> str: - if kwargs.get("async_mode"): - return '"AsyncTokenCredential"' - return '"TokenCredential"' - - @property - def type_description(self) -> str: - return "TokenCredential" - - @property - def credentials_subfolder(self) -> str: - return "credentials_async" if self.code_model.is_azure_flavor else "credentials" - - def docstring_type(self, **kwargs: Any) -> str: - if kwargs.get("async_mode"): - return f"~{self.code_model.core_library}.{self.credentials_subfolder}.AsyncTokenCredential" - return f"~{self.code_model.core_library}.credentials.TokenCredential" - - def imports(self, **kwargs: Any) -> FileImport: - file_import = super().imports(**kwargs) - if kwargs.get("async_mode"): - file_import.add_submodule_import( - self.credentials_subfolder, - "AsyncTokenCredential", - ImportType.SDKCORE, - typing_section=TypingSection.TYPING, - ) - else: - file_import.add_submodule_import( - "credentials", - "TokenCredential", - ImportType.SDKCORE, - typing_section=TypingSection.TYPING, - ) - return file_import - - @property - def instance_check_template(self) -> str: - return "hasattr({}, 'get_token')" - - -class KeyCredentialType( - # pylint: disable=unsubscriptable-object - CredentialType[KeyCredentialPolicyType] -): - """Type for an KeyCredential""" - - def docstring_type(self, **kwargs: Any) -> str: # pylint: disable=unused-argument - return f"~{self.code_model.core_library}.credentials.{self.policy.credential_name}" - - def type_annotation(self, **kwargs: Any) -> str: # pylint: disable=unused-argument - return self.policy.credential_name - - @property - def instance_check_template(self) -> str: - return "isinstance({}, " + f"{self.policy.credential_name})" - - def imports(self, **kwargs: Any) -> FileImport: # pylint: disable=unused-argument - file_import = super().imports(**kwargs) - file_import.add_submodule_import( - "credentials", - self.policy.credential_name, - ImportType.SDKCORE, - typing_section=TypingSection.CONDITIONAL, - ) - return file_import diff --git a/packages/autorest.python/generator/pygen/codegen/models/dictionary_type.py b/packages/autorest.python/generator/pygen/codegen/models/dictionary_type.py deleted file mode 100644 index bf00b1c2317..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/dictionary_type.py +++ /dev/null @@ -1,131 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any, Dict, Optional, TYPE_CHECKING, List -from .base import BaseType -from .imports import FileImport, ImportType, TypingSection - -if TYPE_CHECKING: - from .code_model import CodeModel - from .model_type import ModelType - - -class DictionaryType(BaseType): - """Schema for dictionaries that will be serialized. - - :param yaml_data: the yaml data for this schema - :type yaml_data: dict[str, Any] - :param element_type: The type of the value for the dictionary - :type element_type: ~autorest.models.BaseType - """ - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - element_type: BaseType, - ) -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.element_type = element_type - - @property - def encode(self) -> Optional[str]: - return self.element_type.encode if hasattr(self.element_type, "encode") else None # type: ignore - - @property - def serialization_type(self) -> str: - """Returns the serialization value for msrest. - - :return: The serialization value for msrest - :rtype: str - """ - return f"{{{self.element_type.serialization_type}}}" - - def type_annotation(self, **kwargs: Any) -> str: - """The python type used for type annotation - - :return: The type annotation for this schema - :rtype: str - """ - return f"Dict[str, {self.element_type.type_annotation(**kwargs)}]" - - def description(self, *, is_operation_file: bool) -> str: - return "" if is_operation_file else self.yaml_data.get("description", "") - - def docstring_text(self, **kwargs: Any) -> str: - return f"dict mapping str to {self.element_type.docstring_text(**kwargs)}" - - @property - def xml_serialization_ctxt(self) -> Optional[str]: - """No serialization ctxt for dictionaries""" - return None - - def docstring_type(self, **kwargs: Any) -> str: - """The python type used for RST syntax input and type annotation. - - :param str namespace: Optional. The namespace for the models. - """ - return f"dict[str, {self.element_type.docstring_type(**kwargs)}]" - - def get_json_template_representation( - self, - *, - optional: bool = True, - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - return { - '"str"': self.element_type.get_json_template_representation( - optional=optional, - client_default_value_declaration=client_default_value_declaration, - description=description, - ) - } - - def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: - from .model_type import ModelType - - if isinstance(self.element_type, ModelType): - is_polymorphic_subtype = ( - self.element_type.discriminator_value and not self.element_type.discriminated_subtypes - ) - if self.element_type.name not in (m.name for m in polymorphic_subtypes) and is_polymorphic_subtype: - polymorphic_subtypes.append(self.element_type) - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "DictionaryType": - """Constructs a DictionaryType from yaml data. - - :param yaml_data: the yaml data from which we will construct this schema - :type yaml_data: dict[str, Any] - - :return: A created DictionaryType - :rtype: ~autorest.models.DictionaryType - """ - element_schema: Dict[str, Any] = yaml_data["elementType"] - - from . import build_type # pylint: disable=import-outside-toplevel - - element_type = build_type(yaml_data=element_schema, code_model=code_model) - - return cls( - yaml_data=yaml_data, - code_model=code_model, - element_type=element_type, - ) - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB, TypingSection.CONDITIONAL) - file_import.merge(self.element_type.imports(**kwargs)) - return file_import - - @property - def instance_check_template(self) -> str: - return "isinstance({}, dict)" - - @property - def type_description(self) -> str: - return f"{{str: {self.element_type.type_description}}}" diff --git a/packages/autorest.python/generator/pygen/codegen/models/enum_type.py b/packages/autorest.python/generator/pygen/codegen/models/enum_type.py deleted file mode 100644 index 6e9c74531c7..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/enum_type.py +++ /dev/null @@ -1,246 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any, Dict, List, TYPE_CHECKING, Optional, cast - -from .base import BaseType -from .imports import FileImport, ImportType, TypingSection - -if TYPE_CHECKING: - from .code_model import CodeModel - - -class EnumValue(BaseType): - """Model containing necessary information for a single value of an enum. - - :param str name: The name of this enum value - :param str value: The value of this enum value - :param str description: Optional. The description for this enum value - """ - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - enum_type: "EnumType", - value_type: BaseType, - ) -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.name: str = self.yaml_data["name"] - self.value: str = self.yaml_data["value"] - self.enum_type = enum_type - self.value_type = value_type - - def description(self, *, is_operation_file: bool) -> str: - return self.yaml_data.get("description", "") - - def type_annotation(self, **kwargs: Any) -> str: - """The python type used for type annotation""" - return f"Literal[{self.enum_type.name}.{self.name}]" - - def get_declaration(self, value=None): - return self.enum_type.name + "." + self.name - - def docstring_text(self, **kwargs: Any) -> str: - return self.enum_type.name + "." + self.name - - def docstring_type(self, **kwargs: Any) -> str: - """The python type used for RST syntax input and type annotation.""" - - type_annotation = self.value_type.type_annotation(**kwargs) - enum_type_annotation = f"{self.code_model.namespace}.models.{self.name}" - return f"{type_annotation} or ~{enum_type_annotation}" - - def get_json_template_representation( - self, - *, - optional: bool = True, - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - # for better display effect, use the only value instead of var type - return self.value_type.get_json_template_representation( - optional=optional, - client_default_value_declaration=client_default_value_declaration, - description=description, - ) - - @property - def serialization_type(self) -> str: - return self.value_type.serialization_type - - @property - def instance_check_template(self) -> str: - return self.value_type.instance_check_template - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.merge(self.value_type.imports(**kwargs)) - file_import.add_submodule_import("typing", "Literal", ImportType.STDLIB, TypingSection.REGULAR) - file_import.add_submodule_import("._enums", self.enum_type.name, ImportType.LOCAL, TypingSection.REGULAR) - - return file_import - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "EnumValue": - """Constructs an EnumValue from yaml data. - - :param yaml_data: the yaml data from which we will construct this object - :type yaml_data: dict[str, Any] - - :return: A created EnumValue - :rtype: ~autorest.models.EnumValue - """ - from . import build_type - - return cls( - yaml_data=yaml_data, - code_model=code_model, - enum_type=cast(EnumType, build_type(yaml_data["enumType"], code_model)), - value_type=build_type(yaml_data["valueType"], code_model), - ) - - -class EnumType(BaseType): - """Schema for enums that will be serialized. - - :param yaml_data: the yaml data for this schema - :type yaml_data: dict[str, Any] - :param str description: The description of this enum - :param str name: The name of the enum. - :type element_type: ~autorest.models.PrimitiveType - :param values: List of the values for this enum - :type values: list[~autorest.models.EnumValue] - """ - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - values: List["EnumValue"], - value_type: BaseType, - ) -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.name: str = yaml_data["name"][0].upper() + yaml_data["name"][1:] - self.values = values - self.value_type = value_type - self.internal: bool = self.yaml_data.get("internal", False) - self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") - - def __lt__(self, other): - return self.name.lower() < other.name.lower() - - @property - def serialization_type(self) -> str: - """Returns the serialization value for msrest. - - :return: The serialization value for msrest - :rtype: str - """ - return self.value_type.serialization_type - - def description(self, *, is_operation_file: bool) -> str: # pylint: disable=unused-argument - possible_values = [self.get_declaration(v.value) for v in self.values] - if not possible_values: - return "" - if len(possible_values) == 1: - return possible_values[0] - if len(possible_values) == 2: - possible_values_str = " and ".join(possible_values) - else: - possible_values_str = ( - ", ".join(possible_values[: len(possible_values) - 1]) + f", and {possible_values[-1]}" - ) - - enum_description = f"Known values are: {possible_values_str}." - return enum_description - - def type_annotation(self, **kwargs: Any) -> str: - """The python type used for type annotation - - :return: The type annotation for this schema - :rtype: str - """ - if self.code_model.options["models_mode"]: - module_name = "_models." if kwargs.get("need_module_name", True) else "" - file_name = f"{self.code_model.enums_filename}." if self.internal else "" - model_name = module_name + file_name + self.name - # we don't need quoted annotation in operation files, and need it in model folder files. - if not kwargs.get("is_operation_file", False): - model_name = f'"{model_name}"' - - return f"Union[{self.value_type.type_annotation(**kwargs)}, {model_name}]" - return self.value_type.type_annotation(**kwargs) - - def get_declaration(self, value: Any) -> str: - return self.value_type.get_declaration(value) - - def docstring_text(self, **kwargs: Any) -> str: - if self.code_model.options["models_mode"]: - return self.name - return self.value_type.type_annotation(**kwargs) - - def docstring_type(self, **kwargs: Any) -> str: - """The python type used for RST syntax input and type annotation.""" - if self.code_model.options["models_mode"]: - type_annotation = self.value_type.type_annotation(**kwargs) - enum_type_annotation = f"{self.code_model.namespace}.models.{self.name}" - return f"{type_annotation} or ~{enum_type_annotation}" - return self.value_type.type_annotation(**kwargs) - - def get_json_template_representation( - self, - *, - optional: bool = True, - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - # for better display effect, use the only value instead of var type - return self.value_type.get_json_template_representation( - optional=optional, - client_default_value_declaration=client_default_value_declaration, - description=description, - ) - - @property - def instance_check_template(self) -> str: - return self.value_type.instance_check_template - - def fill_instance_from_yaml(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - for value in yaml_data["values"]: - self.values.append(EnumValue.from_yaml(value, code_model)) - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "EnumType": - raise ValueError( - "You shouldn't call from_yaml for EnumType to avoid recursion. " - "Please initial a blank EnumType, then call .fill_instance_from_yaml on the created type." - ) - - def imports(self, **kwargs: Any) -> FileImport: - operation = kwargs.pop("operation", False) - file_import = FileImport(self.code_model) - if self.code_model.options["models_mode"]: - file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) - if not operation: - file_import.add_submodule_import( - "..", - "models", - ImportType.LOCAL, - TypingSection.TYPING, - alias="_models", - ) - file_import.merge(self.value_type.imports(operation=operation, **kwargs)) - relative_path = kwargs.pop("relative_path", None) - if self.code_model.options["models_mode"] and relative_path: - # add import for enums in operations file - file_import.add_submodule_import( - relative_path, - "models", - ImportType.LOCAL, - alias="_models", - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), - ) - return file_import diff --git a/packages/autorest.python/generator/pygen/codegen/models/imports.py b/packages/autorest.python/generator/pygen/codegen/models/imports.py deleted file mode 100644 index 8e21e4b9c1b..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/imports.py +++ /dev/null @@ -1,291 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from enum import Enum, auto -from typing import Dict, List, Optional, Tuple, Union, Set, TYPE_CHECKING - -if TYPE_CHECKING: - from .code_model import CodeModel - - -class ImportType(str, Enum): - """ - Ordering of these enum matters. We order import groupings in a file based off of this ordering. - """ - - STDLIB = "stdlib" - THIRDPARTY = "thirdparty" - SDKCORE = "sdkcore" - LOCAL = "local" - BYVERSION = "by_version" - - -class TypingSection(str, Enum): - REGULAR = "regular" # this import is always a typing import - CONDITIONAL = "conditional" # is a typing import when we're dealing with files that py2 will use, else regular - TYPING = "typing" # never a typing import - - -class MsrestImportType(Enum): - Module = auto() # import _serialization.py or msrest.serialization as Module - Serializer = auto() # from _serialization.py or msrest.serialization import Serializer - SerializerDeserializer = auto() # from _serialization.py or msrest.serialization import Serializer and Deserializer - - -class ImportModel: - def __init__( - self, - typing_section: TypingSection, - import_type: ImportType, - module_name: str, - *, - submodule_name: Optional[str] = None, - alias: Optional[str] = None, - version_modules: Optional[Tuple[Tuple[Tuple[int, int], str, Optional[str]]]] = None, - ): - self.typing_section = typing_section - self.import_type = import_type - self.module_name = module_name - self.submodule_name = submodule_name - self.alias = alias - # version_modules: this field is for imports submodule from specified module by python version. - # It's a list of "python version, module_name, comments". - # The python version is in form of (major, minor), for instance (3, 9) stands for py3.9. - self.version_modules = version_modules - - def __eq__(self, other): - try: - return ( - self.typing_section == other.typing_section - and self.import_type == other.import_type - and self.module_name == other.module_name - and self.submodule_name == other.submodule_name - and self.alias == other.alias - ) - except AttributeError: - return False - - def __hash__(self) -> int: - retval: int = 0 - for attr in dir(self): - if attr[0] != "_": - retval += hash(getattr(self, attr)) - return retval - - -class TypeDefinition: - def __init__( - self, - sync_definition: str, - async_definition: str, - ): - self.sync_definition = sync_definition - self.async_definition = async_definition - - -class FileImport: - def __init__(self, code_model: "CodeModel") -> None: - self.imports: List[ImportModel] = [] - self.code_model = code_model - # has sync and async type definitions - self.type_definitions: Dict[str, TypeDefinition] = {} - self.core_library = self.code_model.core_library - - def _append_import(self, import_model: ImportModel) -> None: - if import_model.import_type == ImportType.SDKCORE: - mod_name = import_model.module_name - core_libraries = [ - self.code_model.core_library, - "azure", - "msrest", - ] - if all(l not in mod_name for l in core_libraries): - # this is to make sure we don't tack on core libraries when we don't need to - import_model.module_name = f"{self.code_model.core_library}{'.' if mod_name else ''}{mod_name}" - if not any( - i - for i in self.imports - if all(getattr(i, attr) == getattr(import_model, attr) for attr in dir(i) if attr[0] != "_") - ): - self.imports.append(import_model) - - def get_imports_from_section(self, typing_section: TypingSection) -> List[ImportModel]: - return [i for i in self.imports if i.typing_section == typing_section] - - def add_submodule_import( - self, - module_name: str, - submodule_name: str, - import_type: ImportType, - typing_section: TypingSection = TypingSection.REGULAR, - alias: Optional[str] = None, - version_modules: Optional[Tuple[Tuple[Tuple[int, int], str, Optional[str]]]] = None, - ) -> None: - """Add an import to this import block.""" - self._append_import( - ImportModel( - typing_section=typing_section, - import_type=import_type, - module_name=module_name, - submodule_name=submodule_name, - alias=alias, - version_modules=version_modules, - ) - ) - - def add_import( - self, - module_name: str, - import_type: ImportType, - typing_section: TypingSection = TypingSection.REGULAR, - alias: Optional[str] = None, - ) -> None: - # Implementation detail: a regular import is just a "from" with no from - self._append_import( - ImportModel( - typing_section=typing_section, - import_type=import_type, - module_name=module_name, - alias=alias, - ) - ) - - def define_mypy_type( - self, - type_name: str, - type_value: str, - async_type_value: Optional[str] = None, - ): - self.type_definitions[type_name] = TypeDefinition(type_value, async_type_value or type_value) - - def merge(self, file_import: "FileImport") -> None: - """Merge the given file import format.""" - for i in file_import.imports: - self._append_import(i) - self.type_definitions.update(file_import.type_definitions) - - def add_mutable_mapping_import(self) -> None: - self.add_import("sys", ImportType.STDLIB) - self.add_submodule_import( - "typing", - "MutableMapping", - ImportType.BYVERSION, - TypingSection.REGULAR, - None, - (((3, 9), "collections.abc", None),), - ) - - def define_mutable_mapping_type(self) -> None: - """Helper function for defining the mutable mapping type""" - self.add_mutable_mapping_import() - self.define_mypy_type( - "JSON", - "MutableMapping[str, Any] # pylint: disable=unsubscriptable-object", - ) - self.add_submodule_import("typing", "Any", ImportType.STDLIB) - - def to_dict( - self, - ) -> Dict[ - TypingSection, - Dict[ - ImportType, - Dict[ - str, - Set[ - Optional[ - Union[ - str, - Tuple[str, str], - Tuple[ - str, - Optional[str], - Tuple[Tuple[Tuple[int, int], str, Optional[str]]], - ], - ] - ] - ], - ], - ], - ]: - retval: Dict[ - TypingSection, - Dict[ - ImportType, - Dict[ - str, - Set[ - Optional[ - Union[ - str, - Tuple[str, str], - Tuple[ - str, - Optional[str], - Tuple[Tuple[Tuple[int, int], str, Optional[str]]], - ], - ] - ] - ], - ], - ], - ] = {} - for i in self.imports: - name_import: Optional[ - Union[ - str, - Tuple[str, str], - Tuple[ - str, - Optional[str], - Tuple[Tuple[Tuple[int, int], str, Optional[str]]], - ], - ] - ] = None - if i.submodule_name: - if i.version_modules: - name_import = (i.submodule_name, i.alias, i.version_modules) - elif i.alias: - name_import = (i.submodule_name, i.alias) - else: - name_import = i.submodule_name - retval.setdefault(i.typing_section, {}).setdefault(i.import_type, {}).setdefault(i.module_name, set()).add( - name_import - ) - return retval - - def add_msrest_import( - self, - *, - relative_path: str, - msrest_import_type: MsrestImportType, - typing_section: TypingSection, - ): - if self.code_model.options["client_side_validation"]: - if msrest_import_type == MsrestImportType.Module: - self.add_import("msrest.serialization", ImportType.SDKCORE, typing_section) - else: - self.add_submodule_import("msrest", "Serializer", ImportType.THIRDPARTY, typing_section) - if msrest_import_type == MsrestImportType.SerializerDeserializer: - self.add_submodule_import("msrest", "Deserializer", ImportType.THIRDPARTY, typing_section) - else: - if self.code_model.options["multiapi"]: - relative_path += "." - if msrest_import_type == MsrestImportType.Module: - self.add_submodule_import(relative_path, "_serialization", ImportType.LOCAL, typing_section) - else: - self.add_submodule_import( - f"{relative_path}_serialization", - "Serializer", - ImportType.LOCAL, - typing_section, - ) - if msrest_import_type == MsrestImportType.SerializerDeserializer: - self.add_submodule_import( - f"{relative_path}_serialization", - "Deserializer", - ImportType.LOCAL, - typing_section, - ) diff --git a/packages/autorest.python/generator/pygen/codegen/models/list_type.py b/packages/autorest.python/generator/pygen/codegen/models/list_type.py deleted file mode 100644 index 79a2745086f..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/list_type.py +++ /dev/null @@ -1,147 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any, Dict, Optional, Union, TYPE_CHECKING, List -from .base import BaseType -from .imports import FileImport, ImportType, TypingSection - -if TYPE_CHECKING: - from .code_model import CodeModel - from .model_type import ModelType - - -class ListType(BaseType): - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - element_type: BaseType, - ) -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.element_type = element_type - self.max_items: Optional[int] = yaml_data.get("maxItems") - self.min_items: Optional[int] = yaml_data.get("minItems") - self.unique_items: bool = yaml_data.get("uniqueItems", False) - - @property - def encode(self) -> Optional[str]: - return self.element_type.encode if hasattr(self.element_type, "encode") else None # type: ignore - - @property - def serialization_type(self) -> str: - return f"[{self.element_type.serialization_type}]" - - def type_annotation(self, **kwargs: Any) -> str: - if ( - self.code_model.options["version_tolerant"] - and self.element_type.is_xml - and not self.code_model.options["models_mode"] - ): - # this means we're version tolerant XML, we just return the XML element - return self.element_type.type_annotation(**kwargs) - return f"List[{self.element_type.type_annotation(**kwargs)}]" - - def description(self, *, is_operation_file: bool) -> str: - return "" if is_operation_file else self.yaml_data.get("description", "") - - @property - def xml_serialization_ctxt(self) -> Optional[str]: - attrs_list = [] - base_xml_map = super().xml_serialization_ctxt - if base_xml_map: - attrs_list.append(base_xml_map) - - # Attribute at the list level - if self.xml_metadata.get("wrapped", False): - attrs_list.append("'wrapped': True") - - # Attributes of the items - item_xml_metadata = self.element_type.xml_metadata - if item_xml_metadata.get("name"): - attrs_list.append(f"'itemsName': '{item_xml_metadata['name']}'") - if item_xml_metadata.get("prefix", False): - attrs_list.append(f"'itemsPrefix': '{item_xml_metadata['prefix']}'") - if item_xml_metadata.get("namespace", False): - attrs_list.append(f"'itemsNs': '{item_xml_metadata['namespace']}'") - - return ", ".join(attrs_list) - - def docstring_type(self, **kwargs: Any) -> str: - if self.code_model.options["version_tolerant"] and self.element_type.xml_metadata: - # this means we're version tolerant XML, we just return the XML element - return self.element_type.docstring_type(**kwargs) - return f"list[{self.element_type.docstring_type(**kwargs)}]" - - def docstring_text(self, **kwargs: Any) -> str: - if self.code_model.options["version_tolerant"] and self.element_type.xml_metadata: - # this means we're version tolerant XML, we just return the XML element - return self.element_type.docstring_text(**kwargs) - return f"list of {self.element_type.docstring_text(**kwargs)}" - - @property - def validation(self) -> Optional[Dict[str, Union[bool, int, str]]]: - validation: Dict[str, Union[bool, int, str]] = {} - if self.max_items: - validation["max_items"] = self.max_items - validation["min_items"] = self.min_items or 0 - if self.min_items: - validation["min_items"] = self.min_items - if self.unique_items: - validation["unique"] = True - return validation or None - - def get_json_template_representation( - self, - *, - optional: bool = True, - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - return [ - self.element_type.get_json_template_representation( - optional=optional, - client_default_value_declaration=client_default_value_declaration, - description=description, - ) - ] - - def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: - from .model_type import ModelType - - if isinstance(self.element_type, ModelType): - is_polymorphic_subtype = ( - self.element_type.discriminator_value and not self.element_type.discriminated_subtypes - ) - if self.element_type.name not in (m.name for m in polymorphic_subtypes) and is_polymorphic_subtype: - polymorphic_subtypes.append(self.element_type) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, list)" - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "ListType": - from . import build_type - - return cls( - yaml_data=yaml_data, - code_model=code_model, - element_type=build_type(yaml_data=yaml_data["elementType"], code_model=code_model), - ) - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - if not ( - self.code_model.options["version_tolerant"] - and self.element_type.is_xml - and not self.code_model.options["models_mode"] - ): - file_import.add_submodule_import("typing", "List", ImportType.STDLIB, TypingSection.CONDITIONAL) - file_import.merge(self.element_type.imports(**kwargs)) - return file_import - - @property - def type_description(self) -> str: - return f"[{self.element_type.type_description}]" diff --git a/packages/autorest.python/generator/pygen/codegen/models/lro_operation.py b/packages/autorest.python/generator/pygen/codegen/models/lro_operation.py deleted file mode 100644 index 2e127df9d3b..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/lro_operation.py +++ /dev/null @@ -1,143 +0,0 @@ -# pylint: disable=multiple-statements -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any, Dict, Optional, List, TYPE_CHECKING, TypeVar, Union -from .imports import FileImport -from .operation import OperationBase, Operation -from .response import LROPagingResponse, LROResponse, Response -from .imports import ImportType, TypingSection -from .request_builder import RequestBuilder -from .parameter_list import ParameterList - -if TYPE_CHECKING: - from .code_model import CodeModel - from .client import Client - from . import OperationType - -LROResponseType = TypeVar("LROResponseType", bound=Union[LROResponse, LROPagingResponse]) - - -class LROOperationBase(OperationBase[LROResponseType]): - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - client: "Client", - name: str, - request_builder: RequestBuilder, - parameters: ParameterList, - responses: List[LROResponseType], - exceptions: List[Response], - *, - overloads: Optional[List[Operation]] = None, - ) -> None: - super().__init__( - code_model=code_model, - client=client, - yaml_data=yaml_data, - name=name, - request_builder=request_builder, - parameters=parameters, - responses=responses, - exceptions=exceptions, - overloads=overloads, - ) - if not self.name.lstrip("_").startswith("begin"): - self.name = ("_begin" if self.internal else "begin_") + self.name - self.lro_options: Dict[str, Any] = self.yaml_data.get("lroOptions", {}) - self._initial_operation: Optional["OperationType"] = None - - @property - def initial_operation(self) -> "OperationType": - if not self._initial_operation: - raise ValueError("You need to first call client.link_lro_initial_operations before accessing") - return self._initial_operation - - @initial_operation.setter - def initial_operation(self, val: "OperationType") -> None: - self._initial_operation = val - - @property - def operation_type(self) -> str: - return "lro" - - @property - def has_optional_return_type(self) -> bool: - return False - - @property - def lro_response(self) -> Optional[LROResponseType]: - responses_with_bodies = [r for r in self.responses if r.type] - num_response_schemas = {id(r.type.yaml_data) for r in responses_with_bodies if r.type} - response = None - if len(num_response_schemas) > 1: - # choose the response that has a status code of 200 - try: - response = next(r for r in responses_with_bodies if 200 in r.status_codes) - except StopIteration as exc: - raise ValueError( - "Your swagger is invalid because you have multiple response schemas for LRO" - + f" method {self.name} and none of them have a 200 status code." - ) from exc - - elif num_response_schemas: - response = responses_with_bodies[0] - return response - - def response_type_annotation(self, **kwargs) -> str: - lro_response = self.lro_response or next(iter(self.responses), None) - if lro_response: - return lro_response.type_annotation(**kwargs) - return "None" - - def cls_type_annotation(self, *, async_mode: bool) -> str: - """We don't want the poller to show up in ClsType, so we call super() on resposne type annotation""" - return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode)}]" - - def get_poller_with_response_type(self, async_mode: bool) -> str: - return self.response_type_annotation(async_mode=async_mode) - - def get_poller(self, async_mode: bool) -> str: - return self.responses[0].get_poller(async_mode) - - def get_polling_method(self, async_mode: bool) -> str: - return self.responses[0].get_polling_method(async_mode) - - def get_base_polling_method(self, async_mode: bool) -> str: - return self.responses[0].get_base_polling_method(async_mode) - - def get_base_polling_method_path(self, async_mode: bool) -> str: - return self.responses[0].get_base_polling_method_path(async_mode) - - def get_no_polling_method(self, async_mode: bool) -> str: - return self.responses[0].get_no_polling_method(async_mode) - - def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: - file_import = super().imports(async_mode, **kwargs) - if self.abstract: - return file_import - if async_mode and self.code_model.options["tracing"] and self.want_tracing: - file_import.add_submodule_import( - "azure.core.tracing.decorator_async", - "distributed_trace_async", - ImportType.SDKCORE, - ) - if ( - self.code_model.options["models_mode"] == "dpg" - and self.lro_response - and self.lro_response.type - and self.lro_response.type.type == "model" - ): - # used in the case if initial operation returns none - # but final call returns a model - relative_path = "..." if async_mode else ".." - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) - file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) - file_import.add_submodule_import("typing", "cast", ImportType.STDLIB) - return file_import - - -class LROOperation(LROOperationBase[LROResponse]): ... diff --git a/packages/autorest.python/generator/pygen/codegen/models/lro_paging_operation.py b/packages/autorest.python/generator/pygen/codegen/models/lro_paging_operation.py deleted file mode 100644 index 42c885f308f..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/lro_paging_operation.py +++ /dev/null @@ -1,32 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any -from .imports import FileImport -from .lro_operation import LROOperationBase -from .paging_operation import PagingOperationBase -from .response import LROPagingResponse, Response - - -class LROPagingOperation(LROOperationBase[LROPagingResponse], PagingOperationBase[LROPagingResponse]): - @property - def success_status_codes(self): - """The list of all successfull status code.""" - return [200] - - @property - def operation_type(self) -> str: - return "lropaging" - - def cls_type_annotation(self, *, async_mode: bool) -> str: - return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode)}]" # pylint: disable=no-member - - def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: - lro_imports = LROOperationBase.imports(self, async_mode, **kwargs) - paging_imports = PagingOperationBase.imports(self, async_mode, **kwargs) - - file_import = lro_imports - file_import.merge(paging_imports) - return file_import diff --git a/packages/autorest.python/generator/pygen/codegen/models/model_type.py b/packages/autorest.python/generator/pygen/codegen/models/model_type.py deleted file mode 100644 index 84bae17609a..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/model_type.py +++ /dev/null @@ -1,350 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from collections import OrderedDict -from typing import Any, Dict, List, Optional, TYPE_CHECKING, cast -import sys -from .utils import ( - add_to_pylint_disable, - NAME_LENGTH_LIMIT, -) -from .base import BaseType -from .constant_type import ConstantType -from .property import Property -from .imports import FileImport, ImportType, TypingSection - -if sys.version_info >= (3, 8): - from typing import Literal # pylint: disable=no-name-in-module, ungrouped-imports -else: - from typing_extensions import Literal # type: ignore # pylint: disable=ungrouped-imports - -if TYPE_CHECKING: - from .code_model import CodeModel - - -def _get_properties(type: "ModelType", properties: List[Property]) -> List[Property]: - for parent in type.parents: - # here we're adding the properties from our parents - - # need to make sure that the properties we choose from our parent also don't contain - # any of our own properties - property_names = set([p.client_name for p in properties] + [p.client_name for p in type.properties]) - chosen_parent_properties = [p for p in parent.properties if p.client_name not in property_names] - properties = _get_properties(parent, chosen_parent_properties) + properties - return properties - - -class ModelType( # pylint: disable=abstract-method - BaseType -): # pylint: disable=too-many-instance-attributes, too-many-public-methods - """Represents a class ready to be serialized in Python. - - :param str name: The name of the class. - :param str description: The description of the class. - :param properties: the optional properties of the class. - :type properties: dict(str, str) - """ - - base: Literal["msrest", "dpg", "json"] - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - *, - properties: Optional[List[Property]] = None, - parents: Optional[List["ModelType"]] = None, - discriminated_subtypes: Optional[Dict[str, "ModelType"]] = None, - ) -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.name: str = self.yaml_data["name"] - self.max_properties: Optional[int] = self.yaml_data.get("maxProperties") - self.min_properties: Optional[int] = self.yaml_data.get("minProperties") - self.properties = properties or [] - self.parents = parents or [] - self.discriminated_subtypes = discriminated_subtypes or {} - self.discriminator_value: Optional[str] = self.yaml_data.get("discriminatorValue") - self._created_json_template_representation = False - self._got_polymorphic_subtypes = False - self.internal: bool = self.yaml_data.get("internal", False) - self.snake_case_name: str = self.yaml_data["snakeCaseName"] - self.page_result_model: bool = self.yaml_data.get("pageResultModel", False) - self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") - - @property - def flattened_property(self) -> Optional[Property]: - try: - return next(p for p in self.properties if p.flatten) - except StopIteration: - return None - - @property - def flattened_items(self) -> List[str]: - return [ - item.client_name - for prop in self.properties - if isinstance(prop.type, ModelType) and prop.flatten - for item in prop.type.properties - ] - - @property - def is_form_data(self) -> bool: - return any(p.is_multipart_file_input for p in self.properties) - - @property - def is_xml(self) -> bool: - return self.yaml_data.get("isXml", False) - - @property - def msrest_deserialization_key(self) -> str: - return self.name - - @property - def is_polymorphic(self) -> bool: - return any(p.is_polymorphic for p in self.properties) - - def description(self, *, is_operation_file: bool = False) -> str: - return "" if is_operation_file else self.yaml_data.get("description", self.name) - - def get_declaration(self, value: Any) -> str: - return f"{self.name}()" - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} {self.name}>" - - @property - def xml_serialization_ctxt(self) -> Optional[str]: - # object schema contains _xml_map, they don't need serialization context - return "" - - @property - def xml_map_content(self) -> Optional[str]: - # This is NOT an error on the super call, we use the serialization context for "xml_map", - # but we don't want to write a serialization context for an object. - return super().xml_serialization_ctxt - - @property - def discriminated_subtypes_name_mapping(self) -> Dict[str, str]: - return {k: v.name for k, v in self.discriminated_subtypes.items()} - - def get_json_template_representation( - self, - *, - optional: bool = True, - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - if self._created_json_template_representation: - return "..." # do this to avoid loop - self._created_json_template_representation = True - if self.discriminated_subtypes: - # we will instead print the discriminated subtypes - self._created_json_template_representation = False - return f'"{self.snake_case_name}"' if self.code_model.for_test else self.snake_case_name - - # don't add additional properties, because there's not really a concept of - # additional properties in the template - representation = { - f'"{prop.wire_name}"': prop.get_json_template_representation( - optional=optional, - client_default_value_declaration=client_default_value_declaration, - description=description, - ) - for prop in [ - p for p in self.properties if not (p.is_discriminator or p.client_name == "additional_properties") - ] - } - if self.discriminator and self.discriminator_value: - representation[f'"{self.discriminator.wire_name}"'] = f'"{self.discriminator_value}"' - - # once we've finished, we want to reset created_json_template_representation to false - # so we can call it again - self._created_json_template_representation = False - optional_keys = [f'"{p.wire_name}"' for p in self.properties if getattr(p, "optional", False)] - return OrderedDict( - sorted( - representation.items(), - key=lambda item: f"{1 if item[0] in optional_keys else 0}{item[0]}", - ) - ) - - def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: - is_polymorphic_subtype = self.discriminator_value and not self.discriminated_subtypes - if self._got_polymorphic_subtypes: - return - self._got_polymorphic_subtypes = True - if self.name not in (m.name for m in polymorphic_subtypes) and is_polymorphic_subtype: - polymorphic_subtypes.append(self) - for discriminated_subtype in self.discriminated_subtypes.values(): - discriminated_subtype.get_polymorphic_subtypes(polymorphic_subtypes) - for property in self.properties: - property.get_polymorphic_subtypes(polymorphic_subtypes) - self._got_polymorphic_subtypes = False - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "ModelType": - raise ValueError( - "You shouldn't call from_yaml for ModelType to avoid recursion. " - "Please initial a blank ModelType, then call .fill_instance_from_yaml on the created type." - ) - - def fill_instance_from_yaml(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - from . import build_type - - self.parents = [cast(ModelType, build_type(bm, code_model)) for bm in yaml_data.get("parents", [])] - properties = [Property.from_yaml(p, code_model) for p in yaml_data["properties"]] - self.properties = _get_properties(self, properties) - # checking to see if this is a polymorphic class - self.discriminated_subtypes = { - k: cast(ModelType, build_type(v, code_model)) - for k, v in self.yaml_data.get("discriminatedSubtypes", {}).items() - } - - @property - def has_readonly_or_constant_property(self) -> bool: - return any(x.readonly or x.constant or x.visibility == ["read"] for x in self.properties) - - @property - def discriminator(self) -> Optional[Property]: - try: - return next(p for p in self.properties if p.is_discriminator) - except StopIteration: - return None - - @property - def discriminator_property(self) -> Optional[Property]: - try: - return next( - p - for p in self.properties - if p.is_discriminator and isinstance(p.type, ConstantType) and p.type.value == self.discriminator_value - ) - except StopIteration: - return None - - @property - def pylint_disable(self) -> str: - retval: str = "" - if len(self.properties) > 10: - retval = add_to_pylint_disable(retval, "too-many-instance-attributes") - if len(self.name) > NAME_LENGTH_LIMIT: - retval = add_to_pylint_disable(retval, "name-too-long") - return retval - - @property - def init_pylint_disable(self) -> str: - retval: str = "" - if len(self.properties) > 23: - retval = add_to_pylint_disable(retval, "too-many-locals") - return retval - - -class JSONModelType(ModelType): - base = "json" - - def type_annotation(self, **kwargs: Any) -> str: - return "ET.Element" if self.is_xml else "JSON" - - @property - def serialization_type(self) -> str: - return "object" - - def docstring_type(self, **kwargs: Any) -> str: - return "ET.Element" if self.is_xml else "JSON" - - def docstring_text(self, **kwargs: Any) -> str: - return "XML Element" if self.is_xml else "JSON object" - - @property - def instance_check_template(self) -> str: - return "isinstance({}, MutableMapping)" - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) - file_import.define_mutable_mapping_type() - if self.is_xml: - file_import.add_submodule_import("xml.etree", "ElementTree", ImportType.STDLIB, alias="ET") - return file_import - - -class GeneratedModelType(ModelType): # pylint: disable=abstract-method - def type_annotation(self, **kwargs: Any) -> str: - is_operation_file = kwargs.pop("is_operation_file", False) - skip_quote = kwargs.get("skip_quote", False) - module_name = "_models." if kwargs.get("need_module_name", True) else "" - file_name = f"{self.code_model.models_filename}." if self.internal else "" - retval = module_name + file_name + self.name - return retval if is_operation_file or skip_quote else f'"{retval}"' - - def docstring_type(self, **kwargs: Any) -> str: - return f"~{self.code_model.namespace}.models.{self.type_annotation(need_module_name=False, skip_quote=True)}" - - def docstring_text(self, **kwargs: Any) -> str: - return self.name - - @property - def type_description(self) -> str: - return self.name - - def imports(self, **kwargs: Any) -> FileImport: - file_import = super().imports(**kwargs) - relative_path = kwargs.pop("relative_path", None) - if relative_path: - # add import for models in operations or _types file - file_import.add_submodule_import( - relative_path, - "models", - ImportType.LOCAL, - alias="_models", - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), - ) - if self.is_form_data: - file_import.add_submodule_import( - relative_path, - "_model_base", - ImportType.LOCAL, - typing_section=(TypingSection.TYPING if kwargs.get("model_typing") else TypingSection.REGULAR), - ) - return file_import - - -class MsrestModelType(GeneratedModelType): - base = "msrest" - - @property - def serialization_type(self) -> str: - return self.type_annotation(skip_quote=True) if self.internal else self.name - - @property - def instance_check_template(self) -> str: - return "isinstance({}, msrest.Model)" - - def imports(self, **kwargs: Any) -> FileImport: - file_import = super().imports(**kwargs) - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) - return file_import - - -class DPGModelType(GeneratedModelType): - base = "dpg" - - @property - def serialization_type(self) -> str: - return ( - self.type_annotation(skip_quote=True) - if self.internal - else self.type_annotation(need_module_name=False, skip_quote=True) - ) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, _model_base.Model)" - - def imports(self, **kwargs: Any) -> FileImport: - file_import = super().imports(**kwargs) - if self.flattened_property: - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) - return file_import diff --git a/packages/autorest.python/generator/pygen/codegen/models/operation.py b/packages/autorest.python/generator/pygen/codegen/models/operation.py deleted file mode 100644 index c121649ca88..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/operation.py +++ /dev/null @@ -1,510 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from itertools import chain -from typing import ( - Dict, - List, - Any, - Optional, - Union, - TYPE_CHECKING, - Generic, - TypeVar, - cast, - Sequence, -) - -from .request_builder_parameter import RequestBuilderParameter - -from .utils import OrderedSet, add_to_pylint_disable, NAME_LENGTH_LIMIT -from .base_builder import BaseBuilder -from .imports import FileImport, ImportType, TypingSection -from .response import ( - Response, - PagingResponse, - LROResponse, - LROPagingResponse, - get_response, -) -from .parameter import ( - BodyParameter, - Parameter, - ParameterLocation, -) -from .parameter_list import ParameterList -from .model_type import ModelType -from .base import BaseType -from .request_builder import OverloadedRequestBuilder, RequestBuilder - -if TYPE_CHECKING: - from .code_model import CodeModel - from .client import Client - from . import OperationType - -ResponseType = TypeVar( - "ResponseType", - bound=Union[Response, PagingResponse, LROResponse, LROPagingResponse], -) - - -def is_internal(target: Optional[BaseType]) -> bool: - return isinstance(target, ModelType) and target.base == "dpg" and target.internal - - -class OperationBase( # pylint: disable=too-many-public-methods,too-many-instance-attributes - Generic[ResponseType], BaseBuilder[ParameterList, List["Operation"]] -): - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - client: "Client", - name: str, - request_builder: Union[RequestBuilder, OverloadedRequestBuilder], - parameters: ParameterList, - responses: List[ResponseType], - exceptions: List[Response], - *, - overloads: Optional[List["Operation"]] = None, - ) -> None: - super().__init__( - code_model=code_model, - client=client, - yaml_data=yaml_data, - name=name, - parameters=parameters, - overloads=overloads, - ) - self.overloads: List["Operation"] = overloads or [] - self.responses = responses - self.request_builder = request_builder - self.deprecated = False - self.exceptions = exceptions - self.is_lro_initial_operation: bool = self.yaml_data.get("isLroInitialOperation", False) - self.include_documentation: bool = not self.is_lro_initial_operation - self.internal: bool = self.yaml_data.get("internal", False) - if self.internal: - self.name = "_" + self.name - self.has_etag: bool = self.yaml_data.get("hasEtag", False) - self.cross_language_definition_id: Optional[str] = self.yaml_data.get("crossLanguageDefinitionId") - - @property - def has_form_data_body(self): - return self.parameters.has_form_data_body - - @property - def expose_stream_keyword(self) -> bool: - return self.yaml_data.get("exposeStreamKeyword", False) - - @property - def operation_type(self) -> str: - return "operation" - - @property - def has_optional_return_type(self) -> bool: - """Has optional return type if there are multiple successful response types where some have - bodies and some are None - """ - # means if we have at least one successful response with a body and one without - successful_response_with_body = any(r for r in self.responses if r.type) - successful_response_without_body = any(r for r in self.responses if not r.type) - return successful_response_with_body and successful_response_without_body - - def response_type_annotation(self, **kwargs) -> str: - if self.code_model.options["head_as_boolean"] and self.request_builder.method.lower() == "head": - return "bool" - response_type_annotations: OrderedSet[str] = { - response.type_annotation(**kwargs): None for response in self.responses if response.type - } - response_str = ", ".join(response_type_annotations.keys()) - if len(response_type_annotations) > 1: - return f"Union[{response_str}]" - if self.has_optional_return_type: - return f"Optional[{response_str}]" - if self.responses: - return self.responses[0].type_annotation(**kwargs) - return "None" - - @property - def pylint_disable(self) -> str: - retval: str = "" - if self.response_type_annotation(async_mode=False) == "None": - # doesn't matter if it's async or not - retval = add_to_pylint_disable(retval, "inconsistent-return-statements") - try: - if any(is_internal(r.type) for r in self.responses) or is_internal(self.parameters.body_parameter.type): - retval = add_to_pylint_disable(retval, "protected-access") - except ValueError: - pass - if len(self.name) > NAME_LENGTH_LIMIT: - retval = add_to_pylint_disable(retval, "name-too-long") - return retval - - def cls_type_annotation(self, *, async_mode: bool) -> str: - if self.request_builder.method.lower() == "head" and self.code_model.options["head_as_boolean"]: - return "ClsType[None]" - return f"ClsType[{self.response_type_annotation(async_mode=async_mode)}]" - - def _response_docstring_helper(self, attr_name: str, **kwargs: Any) -> str: - responses_with_body = [r for r in self.responses if r.type] - if self.request_builder.method.lower() == "head" and self.code_model.options["head_as_boolean"]: - return "bool" - if responses_with_body: - response_docstring_values: OrderedSet[str] = { - getattr(response, attr_name)(**kwargs): None for response in responses_with_body - } - retval = " or ".join(response_docstring_values.keys()) - if self.has_optional_return_type: - retval += " or None" - return retval - if self.responses: - return getattr(self.responses[0], attr_name)(**kwargs) - return "None" - - def response_docstring_text(self, **kwargs) -> str: - retval = self._response_docstring_helper("docstring_text", **kwargs) - if not self.code_model.options["version_tolerant"]: - retval += " or the result of cls(response)" - if self.code_model.options["models_mode"] == "dpg" and any( - isinstance(r.type, ModelType) for r in self.responses - ): - r = next(r for r in self.responses if isinstance(r.type, ModelType)) - item_type = getattr(r, "item_type", getattr(r, "type")) - if item_type: - type_name = item_type.docstring_text(**kwargs) - retval += f". The {type_name} is compatible with MutableMapping" - return retval - - def response_docstring_type(self, **kwargs) -> str: - return self._response_docstring_helper("docstring_type", **kwargs) - - @property - def has_response_body(self) -> bool: - """Tell if at least one response has a body.""" - return any(response.type for response in self.responses) - - @property - def any_response_has_headers(self) -> bool: - return any(response.headers for response in self.responses) - - @property - def default_error_deserialization(self) -> Optional[str]: - default_exceptions = [e for e in self.exceptions if "default" in e.status_codes and e.type] - if not default_exceptions: - return None - excep_schema = default_exceptions[0].type - if isinstance(excep_schema, ModelType): - return excep_schema.type_annotation(skip_quote=True) - # in this case, it's just an AnyType - return "'object'" - - @property - def non_default_errors(self) -> List[Response]: - return [e for e in self.exceptions if "default" not in e.status_codes] - - @property - def non_default_error_status_codes(self) -> List[Union[str, int]]: - """Actually returns all of the status codes from exceptions (besides default)""" - return list(chain.from_iterable([error.status_codes for error in self.non_default_errors])) - - def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: # pylint: disable=unused-argument - file_import = FileImport(self.code_model) - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) - - response_types = [r.type_annotation(async_mode=async_mode, operation=self) for r in self.responses if r.type] - if len(set(response_types)) > 1: - file_import.add_submodule_import("typing", "Union", ImportType.STDLIB, TypingSection.CONDITIONAL) - if self.added_on: - file_import.add_submodule_import( - f"{'.' if async_mode else ''}.._validation", - "api_version_validation", - ImportType.LOCAL, - ) - return file_import - - def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport: - if self.abstract: - return FileImport(self.code_model) - file_import = self._imports_shared(async_mode, **kwargs) - for param in self.parameters.method: - file_import.merge( - param.imports_for_multiapi( - async_mode, - operation=self, - **kwargs, - ) - ) - for response in self.responses: - file_import.merge(response.imports_for_multiapi(async_mode=async_mode, operation=self, **kwargs)) - if self.code_model.options["models_mode"]: - for exception in self.exceptions: - file_import.merge(exception.imports_for_multiapi(async_mode=async_mode, operation=self, **kwargs)) - return file_import - - @staticmethod - def has_kwargs_to_pop_with_default( - kwargs_to_pop: List[ - Union[ - Parameter, - RequestBuilderParameter, - BodyParameter, - ] - ], - location: ParameterLocation, - ) -> bool: - return any( - (kwarg.client_default_value or kwarg.optional) and kwarg.location == location for kwarg in kwargs_to_pop - ) - - @property - def need_validation(self) -> bool: - """Whether we need parameter / operation validation. For API version.""" - return bool(self.added_on) or any(p for p in self.parameters if p.added_on) - - def get_request_builder_import( - self, - request_builder: Union[RequestBuilder, OverloadedRequestBuilder], - async_mode: bool, - ) -> FileImport: - """Helper method to get a request builder import.""" - file_import = FileImport(self.code_model) - if self.code_model.options["builders_visibility"] != "embedded": - group_name = request_builder.group_name - rest_import_path = "..." if async_mode else ".." - if group_name: - file_import.add_submodule_import( - f"{rest_import_path}{self.code_model.rest_layer_name}", - group_name, - import_type=ImportType.LOCAL, - alias=f"rest_{group_name}", - ) - else: - file_import.add_submodule_import( - rest_import_path, - self.code_model.rest_layer_name, - import_type=ImportType.LOCAL, - alias="rest", - ) - if self.code_model.options["builders_visibility"] == "embedded" and async_mode: - file_import.add_submodule_import( - f"...{self.code_model.operations_folder_name}.{self.filename}", - request_builder.name, - import_type=ImportType.LOCAL, - ) - return file_import - - def imports( # pylint: disable=too-many-branches, disable=too-many-statements - self, async_mode: bool, **kwargs: Any - ) -> FileImport: - if self.abstract: - return FileImport(self.code_model) - file_import = self._imports_shared(async_mode, **kwargs) - - for param in self.parameters.method: - file_import.merge( - param.imports( - async_mode, - operation=self, - **kwargs, - ) - ) - for response in self.responses: - file_import.merge(response.imports(async_mode=async_mode, operation=self, **kwargs)) - if self.code_model.options["models_mode"]: - for exception in self.exceptions: - file_import.merge(exception.imports(async_mode=async_mode, **kwargs)) - - if self.parameters.has_body and self.parameters.body_parameter.flattened: - file_import.merge(self.parameters.body_parameter.type.imports(operation=self, **kwargs)) - if not async_mode: - for param in self.parameters.headers: - if param.wire_name.lower() == "repeatability-request-id": - file_import.add_import("uuid", ImportType.STDLIB) - elif param.wire_name.lower() == "repeatability-first-sent": - file_import.add_import("datetime", ImportType.STDLIB) - - # Exceptions - errors = [ - "map_error", - "HttpResponseError", - "ClientAuthenticationError", - "ResourceNotFoundError", - "ResourceExistsError", - "ResourceNotModifiedError", - ] - for error in errors: - file_import.add_submodule_import("exceptions", error, ImportType.SDKCORE) - if self.code_model.options["azure_arm"]: - file_import.add_submodule_import("azure.mgmt.core.exceptions", "ARMErrorFormat", ImportType.SDKCORE) - file_import.add_submodule_import( - "typing", - "Type", - ImportType.STDLIB, - ) - file_import.add_mutable_mapping_import() - if self.non_default_error_status_codes: - file_import.add_submodule_import( - "typing", - "cast", - ImportType.STDLIB, - ) - - if self.has_kwargs_to_pop_with_default( - self.parameters.kwargs_to_pop, ParameterLocation.HEADER # type: ignore - ) or self.has_kwargs_to_pop_with_default( - self.parameters.kwargs_to_pop, ParameterLocation.QUERY # type: ignore - ): - file_import.add_submodule_import( - "utils", - "case_insensitive_dict", - ImportType.SDKCORE, - ) - if self.deprecated: - file_import.add_import("warnings", ImportType.STDLIB) - - relative_path = "..." if async_mode else ".." - if self.has_etag: - file_import.add_submodule_import( - "exceptions", - "ResourceModifiedError", - ImportType.SDKCORE, - ) - if not async_mode: - file_import.add_submodule_import(f"{relative_path}_vendor", "prep_if_match", ImportType.LOCAL) - file_import.add_submodule_import(f"{relative_path}_vendor", "prep_if_none_match", ImportType.LOCAL) - if async_mode: - file_import.add_submodule_import( - "rest", - "AsyncHttpResponse", - ImportType.SDKCORE, - ) - else: - file_import.add_submodule_import( - "rest", - "HttpResponse", - ImportType.SDKCORE, - ) - if self.code_model.options["builders_visibility"] == "embedded" and not async_mode: - file_import.merge(self.request_builder.imports()) - file_import.add_submodule_import( - f"{'' if self.code_model.is_azure_flavor else 'runtime.'}pipeline", - "PipelineResponse", - ImportType.SDKCORE, - ) - file_import.add_submodule_import("rest", "HttpRequest", ImportType.SDKCORE) - file_import.add_submodule_import("typing", "Callable", ImportType.STDLIB, TypingSection.CONDITIONAL) - file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB, TypingSection.CONDITIONAL) - file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB, TypingSection.CONDITIONAL) - file_import.add_submodule_import("typing", "TypeVar", ImportType.STDLIB, TypingSection.CONDITIONAL) - if self.code_model.options["tracing"] and self.want_tracing and not async_mode: - file_import.add_submodule_import( - "azure.core.tracing.decorator", - "distributed_trace", - ImportType.SDKCORE, - ) - file_import.merge(self.get_request_builder_import(self.request_builder, async_mode)) - if self.overloads: - file_import.add_submodule_import("typing", "overload", ImportType.STDLIB) - if self.non_default_errors and self.code_model.options["models_mode"] == "dpg": - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) - return file_import - - def get_response_from_status(self, status_code: Optional[Union[str, int]]) -> ResponseType: - try: - return next(r for r in self.responses if status_code in r.status_codes) - except StopIteration as exc: - raise ValueError(f"Incorrect status code {status_code}, operation {self.name}") from exc - - @property - def success_status_codes(self) -> Sequence[Union[str, int]]: - """The list of all successfull status code.""" - return sorted([code for response in self.responses for code in response.status_codes]) - - @property - def filename(self) -> str: - basename = self.group_name - if basename == "": - # in a mixin - basename = self.code_model.clients[0].legacy_filename - - if basename == "operations" or self.code_model.options["combine_operation_files"]: - return "_operations" - return f"_{basename}_operations" - - @property - def has_stream_response(self) -> bool: - return any(r.is_stream_response for r in self.responses) - - @classmethod - def get_request_builder(cls, yaml_data: Dict[str, Any], client: "Client"): - return client.lookup_request_builder(id(yaml_data)) - - @classmethod - def from_yaml( - cls, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - client: "Client", - ): - name = yaml_data["name"] - request_builder = cls.get_request_builder(yaml_data, client) - responses = [cast(ResponseType, get_response(r, code_model)) for r in yaml_data["responses"]] - exceptions = [Response.from_yaml(e, code_model) for e in yaml_data["exceptions"]] - parameter_list = ParameterList.from_yaml(yaml_data, code_model) - overloads = [cls.from_yaml(overload, code_model, client) for overload in yaml_data.get("overloads", [])] - - return cls( - yaml_data=yaml_data, - code_model=code_model, - client=client, - request_builder=request_builder, - name=name, - parameters=parameter_list, - overloads=overloads, - responses=responses, - exceptions=exceptions, - ) - - -class Operation(OperationBase[Response]): - def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: - file_import = super().imports(async_mode, **kwargs) - if self.abstract: - return file_import - if async_mode and self.code_model.options["tracing"] and self.want_tracing: - file_import.add_submodule_import( - "azure.core.tracing.decorator_async", - "distributed_trace_async", - ImportType.SDKCORE, - ) - if self.has_response_body and not self.has_optional_return_type and not self.code_model.options["models_mode"]: - file_import.add_submodule_import("typing", "cast", ImportType.STDLIB) - relative_path = "..." if async_mode else ".." - if self.code_model.options["models_mode"] == "dpg": - if self.parameters.has_body: - if not self.has_form_data_body: - file_import.add_submodule_import( - f"{relative_path}_model_base", - "SdkJSONEncoder", - ImportType.LOCAL, - ) - file_import.add_import("json", ImportType.STDLIB) - if self.default_error_deserialization or any(r.type for r in self.responses): - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) - - return file_import - - -def get_operation(yaml_data: Dict[str, Any], code_model: "CodeModel", client: "Client") -> "OperationType": - if yaml_data["discriminator"] == "lropaging": - from .lro_paging_operation import LROPagingOperation as OperationCls - elif yaml_data["discriminator"] == "lro": - from .lro_operation import LROOperation as OperationCls # type: ignore - elif yaml_data["discriminator"] == "paging": - from .paging_operation import PagingOperation as OperationCls # type: ignore - else: - from . import Operation as OperationCls # type: ignore - return OperationCls.from_yaml(yaml_data, code_model, client) diff --git a/packages/autorest.python/generator/pygen/codegen/models/operation_group.py b/packages/autorest.python/generator/pygen/codegen/models/operation_group.py deleted file mode 100644 index 23b6f4daaba..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/operation_group.py +++ /dev/null @@ -1,184 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Dict, List, Any, TYPE_CHECKING - -from .utils import OrderedSet - -from .base import BaseModel -from .operation import get_operation -from .imports import FileImport, ImportType, TypingSection -from .utils import add_to_pylint_disable, NAME_LENGTH_LIMIT -from .lro_operation import LROOperation -from .lro_paging_operation import LROPagingOperation - -if TYPE_CHECKING: - from .code_model import CodeModel - from .client import Client - from . import OperationType - - -class OperationGroup(BaseModel): - """Represent an operation group.""" - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - client: "Client", - operations: List["OperationType"], - api_versions: List[str], - ) -> None: - super().__init__(yaml_data, code_model) - self.client = client - self.class_name: str = yaml_data["className"] - self.identify_name: str = yaml_data["identifyName"] - self.property_name: str = yaml_data["propertyName"] - self.operations = operations - self.api_versions = api_versions - self.operation_groups: List[OperationGroup] = [] - if self.code_model.options["show_operations"]: - self.operation_groups = [ - OperationGroup.from_yaml(op_group, code_model, client) - for op_group in self.yaml_data.get("operationGroups", []) - ] - self.link_lro_initial_operations() - - @property - def has_abstract_operations(self) -> bool: - return any(o for o in self.operations if o.abstract) or any( - operation_group.has_abstract_operations for operation_group in self.operation_groups - ) - - @property - def has_non_abstract_operations(self) -> bool: - return any(o for o in self.operations if not o.abstract) or any( - operation_group.has_non_abstract_operations for operation_group in self.operation_groups - ) - - @property - def base_class(self) -> str: - base_classes: List[str] = [] - if self.is_mixin: - base_classes.append(f"{self.client.name}MixinABC") - return ", ".join(base_classes) - - def imports_for_multiapi(self, async_mode: bool) -> FileImport: - file_import = FileImport(self.code_model) - relative_path = ".." if async_mode else "." - for operation in self.operations: - file_import.merge(operation.imports_for_multiapi(async_mode, relative_path=relative_path)) - if (self.code_model.model_types or self.code_model.enums) and self.code_model.options[ - "models_mode" - ] == "msrest": - file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") - return file_import - - @property - def pylint_disable(self) -> str: - retval: str = "" - if self.has_abstract_operations: - retval = add_to_pylint_disable(retval, "abstract-class-instantiated") - if len(self.operations) > 20: - retval = add_to_pylint_disable(retval, "too-many-public-methods") - if len(self.class_name) > NAME_LENGTH_LIMIT: - retval = add_to_pylint_disable(retval, "name-too-long") - if len(self.operation_groups) > 6: - retval = add_to_pylint_disable(retval, "too-many-instance-attributes") - return retval - - @property - def need_validation(self) -> bool: - """Whether any of its operations need validation""" - return any(o for o in self.operations if o.need_validation) - - def imports(self, async_mode: bool) -> FileImport: - file_import = FileImport(self.code_model) - - relative_path = ("..." if async_mode else "..") + ("." if self.client.is_subclient else "") - for operation in self.operations: - file_import.merge(operation.imports(async_mode, relative_path=relative_path)) - if not self.code_model.options["combine_operation_files"]: - for og in self.operation_groups: - file_import.add_submodule_import( - ".", - og.class_name, - ImportType.LOCAL, - ) - # for multiapi - if ( - (self.code_model.public_model_types) - and self.code_model.options["models_mode"] == "msrest" - and not self.is_mixin - ): - file_import.add_submodule_import(relative_path, "models", ImportType.LOCAL, alias="_models") - if self.code_model.need_mixin_abc: - file_import.add_submodule_import(".._vendor", f"{self.client.name}MixinABC", ImportType.LOCAL) - if self.has_abstract_operations: - file_import.add_submodule_import(".._vendor", "raise_if_not_implemented", ImportType.LOCAL) - if all(o.abstract for o in self.operations): - return file_import - file_import.add_submodule_import("typing", "TypeVar", ImportType.STDLIB, TypingSection.CONDITIONAL) - file_import.define_mypy_type("T", "TypeVar('T')") - type_value = "Optional[Callable[[PipelineResponse[HttpRequest, {}HttpResponse], T, Dict[str, Any]], Any]]" - file_import.define_mypy_type("ClsType", type_value.format(""), type_value.format("Async")) - return file_import - - @property - def filename(self) -> str: - return self.operations[0].filename - - @property - def is_mixin(self) -> bool: - """The operation group with no name is the direct client methods.""" - return self.identify_name == "" - - def link_lro_initial_operations(self) -> None: - """Link each LRO operation to its initial operation""" - for operation_group in self.operation_groups: - for operation in operation_group.operations: - if isinstance(operation, (LROOperation, LROPagingOperation)): - operation.initial_operation = self.lookup_operation(id(operation.yaml_data["initialOperation"])) - - def lookup_operation(self, operation_id: int) -> "OperationType": - try: - return next(o for og in self.operation_groups for o in og.operations if id(o.yaml_data) == operation_id) - except StopIteration as exc: - raise KeyError(f"No operation with id {operation_id} found.") from exc - - @property - def lro_operations(self) -> List["OperationType"]: - return [operation for operation in self.operations if operation.operation_type in ("lro", "lropaging")] + [ - operation for operation_group in self.operation_groups for operation in operation_group.lro_operations - ] - - @property - def has_operations(self) -> bool: - return any(operation_group.has_operations for operation_group in self.operation_groups) or bool(self.operations) - - @property - def has_form_data_body(self) -> bool: - operations = self.operations + [o for og in self.operation_groups for o in og.operations] - return any(operation.has_form_data_body for operation in operations) - - @classmethod - def from_yaml( - cls, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - client: "Client", - ) -> "OperationGroup": - operations = [get_operation(o, code_model, client) for o in yaml_data["operations"]] - api_versions: OrderedSet[str] = {} - for operation in operations: - for api_version in operation.api_versions: - api_versions[api_version] = None - return cls( - yaml_data=yaml_data, - code_model=code_model, - client=client, - operations=operations, - api_versions=list(api_versions.keys()), - ) diff --git a/packages/autorest.python/generator/pygen/codegen/models/paging_operation.py b/packages/autorest.python/generator/pygen/codegen/models/paging_operation.py deleted file mode 100644 index 554b72d4431..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/paging_operation.py +++ /dev/null @@ -1,156 +0,0 @@ -# pylint: disable=multiple-statements -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Dict, List, Any, Optional, Union, TYPE_CHECKING, cast, TypeVar - -from .operation import Operation, OperationBase -from .response import PagingResponse, LROPagingResponse, Response -from .request_builder import ( - OverloadedRequestBuilder, - RequestBuilder, - get_request_builder, -) -from .imports import ImportType, FileImport, TypingSection -from .parameter_list import ParameterList -from .model_type import ModelType -from .list_type import ListType - -if TYPE_CHECKING: - from .code_model import CodeModel - from .client import Client - -PagingResponseType = TypeVar("PagingResponseType", bound=Union[PagingResponse, LROPagingResponse]) - - -class PagingOperationBase(OperationBase[PagingResponseType]): - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - client: "Client", - name: str, - request_builder: RequestBuilder, - parameters: ParameterList, - responses: List[PagingResponseType], - exceptions: List[Response], - *, - overloads: Optional[List[Operation]] = None, - override_success_response_to_200: bool = False, - ) -> None: - super().__init__( - code_model=code_model, - client=client, - yaml_data=yaml_data, - name=name, - request_builder=request_builder, - parameters=parameters, - responses=responses, - exceptions=exceptions, - overloads=overloads, - ) - self.next_request_builder: Optional[Union[RequestBuilder, OverloadedRequestBuilder]] = ( - get_request_builder(self.yaml_data["nextOperation"], code_model, client) - if self.yaml_data.get("nextOperation") - else None - ) - self.override_success_response_to_200 = override_success_response_to_200 - self.pager_sync: str = yaml_data.get("pagerSync") or f"{self.code_model.core_library}.paging.ItemPaged" - self.pager_async: str = yaml_data.get("pagerAsync") or f"{self.code_model.core_library}.paging.AsyncItemPaged" - - def _get_attr_name(self, wire_name: str) -> str: - response_type = self.responses[0].type - if not response_type: - raise ValueError(f"Can't find a matching property in response for {wire_name}") - if response_type.type == "list": - response_type = cast(ListType, response_type).element_type - try: - return next(p.client_name for p in cast(ModelType, response_type).properties if p.wire_name == wire_name) - except StopIteration as exc: - raise ValueError(f"Can't find a matching property in response for {wire_name}") from exc - - def get_pager(self, async_mode: bool) -> str: - return self.responses[0].get_pager(async_mode) - - @property - def continuation_token_name(self) -> Optional[str]: - wire_name = self.yaml_data.get("continuationTokenName") - if not wire_name: - # That's an ok scenario, it just means no next page possible - return None - if self.code_model.options["models_mode"] == "msrest": - return self._get_attr_name(wire_name) - return wire_name - - @property - def item_name(self) -> str: - wire_name = self.yaml_data["itemName"] - if self.code_model.options["models_mode"] == "msrest": - # we don't use the paging model for dpg - return self._get_attr_name(wire_name) - return wire_name - - @property - def item_type(self) -> ModelType: - try: - item_type_yaml = self.yaml_data["itemType"] - except KeyError as e: - raise ValueError("Only call this for DPG paging model deserialization") from e - return cast(ModelType, self.code_model.types_map[id(item_type_yaml)]) - - @property - def operation_type(self) -> str: - return "paging" - - def cls_type_annotation(self, *, async_mode: bool) -> str: - return f"ClsType[{Response.type_annotation(self.responses[0], async_mode=async_mode)}]" - - def _imports_shared(self, async_mode: bool, **kwargs: Any) -> FileImport: - file_import = super()._imports_shared(async_mode, **kwargs) - if async_mode: - file_import.add_submodule_import("typing", "AsyncIterable", ImportType.STDLIB, TypingSection.CONDITIONAL) - else: - file_import.add_submodule_import("typing", "Iterable", ImportType.STDLIB, TypingSection.CONDITIONAL) - if ( - self.next_request_builder - and self.code_model.options["builders_visibility"] == "embedded" - and not async_mode - ): - file_import.merge(self.next_request_builder.imports()) - return file_import - - @property - def has_optional_return_type(self) -> bool: - return False - - def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: - if self.abstract: - return FileImport(self.code_model) - file_import = self._imports_shared(async_mode, **kwargs) - file_import.merge(super().imports(async_mode, **kwargs)) - if self.code_model.options["tracing"] and self.want_tracing: - file_import.add_submodule_import( - "azure.core.tracing.decorator", - "distributed_trace", - ImportType.SDKCORE, - ) - if self.next_request_builder: - file_import.merge(self.get_request_builder_import(self.next_request_builder, async_mode)) - elif any(p.is_api_version for p in self.client.parameters): - file_import.add_import("urllib.parse", ImportType.STDLIB) - file_import.add_submodule_import( - "utils", - "case_insensitive_dict", - ImportType.SDKCORE, - ) - if self.code_model.options["models_mode"] == "dpg": - relative_path = "..." if async_mode else ".." - file_import.merge(self.item_type.imports(**kwargs)) - if self.default_error_deserialization or any(r.type for r in self.responses): - file_import.add_submodule_import(f"{relative_path}_model_base", "_deserialize", ImportType.LOCAL) - return file_import - - -class PagingOperation(PagingOperationBase[PagingResponse]): ... diff --git a/packages/autorest.python/generator/pygen/codegen/models/parameter.py b/packages/autorest.python/generator/pygen/codegen/models/parameter.py deleted file mode 100644 index 296c8386008..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/parameter.py +++ /dev/null @@ -1,402 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import abc -from enum import Enum - -from typing import ( - Dict, - Any, - TYPE_CHECKING, - List, - Optional, - TypeVar, - Union, -) - -from .imports import FileImport, ImportType, TypingSection -from .base import BaseModel -from .base import BaseType -from .constant_type import ConstantType -from .utils import add_to_description -from .combined_type import CombinedType -from .model_type import JSONModelType - -if TYPE_CHECKING: - from .code_model import CodeModel - from .request_builder_parameter import RequestBuilderBodyParameter - - -class ParameterLocation(str, Enum): - HEADER = "header" - PATH = "path" - ENDPOINT_PATH = "endpointPath" - QUERY = "query" - BODY = "body" - OTHER = "other" - - -class ParameterMethodLocation(str, Enum): - POSITIONAL = "positional" - KEYWORD_ONLY = "keywordOnly" - KWARG = "kwarg" - - -class ParameterDelimeter(str, Enum): - SPACE = "space" - PIPE = "pipe" - TAB = "tab" - COMMA = "comma" - - -class _ParameterBase(BaseModel, abc.ABC): # pylint: disable=too-many-instance-attributes - """Base class for all parameters""" - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - type: BaseType, - ) -> None: - super().__init__(yaml_data, code_model) - self.wire_name: str = yaml_data.get("wireName", "") - self.client_name: str = self.yaml_data["clientName"] - self.optional: bool = self.yaml_data["optional"] - self.implementation: str = yaml_data.get("implementation", None) - self.location: ParameterLocation = self.yaml_data["location"] - self.client_default_value = self.yaml_data.get("clientDefaultValue", None) - self.in_docstring = self.yaml_data.get("inDocstring", True) - self.type = type - if self.client_default_value is None: - self.client_default_value = self.type.client_default_value - # name of grouper if it is grouped by another parameter - self.grouped_by: Optional[str] = self.yaml_data.get("groupedBy") - # property matching property name to parameter name for grouping params - # and flattened body params - self.property_to_parameter_name: Optional[Dict[str, str]] = self.yaml_data.get("propertyToParameterName") - self.flattened: bool = self.yaml_data.get("flattened", False) - self.in_flattened_body: bool = self.yaml_data.get("inFlattenedBody", False) - self.grouper: bool = self.yaml_data.get("grouper", False) - self.check_client_input: bool = self.yaml_data.get("checkClientInput", False) - self.added_on: Optional[str] = self.yaml_data.get("addedOn") - self.is_api_version: bool = self.yaml_data.get("isApiVersion", False) - self.in_overload: bool = self.yaml_data.get("inOverload", False) - self.default_to_unset_sentinel: bool = self.yaml_data.get("defaultToUnsetSentinel", False) - self.hide_in_method: bool = self.yaml_data.get("hideInMethod", False) - - def get_declaration(self, value: Any = None) -> Any: - return self.type.get_declaration(value) - - @property - def hide_in_operation_signature(self) -> bool: - return False - - @property - def constant(self) -> bool: - """Returns whether a parameter is a constant or not. - Checking to see if it's required, because if not, we don't consider it - a constant because it can have a value of None. - """ - return (not self.optional or self.is_api_version) and isinstance(self.type, ConstantType) - - @property - def description(self) -> str: - base_description = self.yaml_data["description"] - type_description = self.type.description(is_operation_file=True) - if type_description: - base_description = add_to_description(base_description, type_description) - if self.optional and isinstance(self.type, ConstantType): - base_description = add_to_description( - base_description, - f"Known values are {self.get_declaration()} and None.", - ) - if not (self.optional or self.client_default_value): - base_description = add_to_description(base_description, "Required.") - if self.client_default_value is not None: - base_description = add_to_description( - base_description, - f"Default value is {self.client_default_value_declaration}.", - ) - if self.optional and self.client_default_value is None: - base_description = add_to_description( - base_description, - f"Default value is {self.client_default_value_declaration}.", - ) - if self.constant: - base_description = add_to_description( - base_description, - "Note that overriding this default value may result in unsupported behavior.", - ) - return base_description - - @property - def client_default_value_declaration(self): - """Declaration of parameter's client default value""" - if self.client_default_value is None: - return None - return self.get_declaration(self.client_default_value) - - def type_annotation(self, **kwargs: Any) -> str: - kwargs["is_operation_file"] = True - # special logic for api-version parameter - if self.is_api_version: - type_annot = "str" - else: - type_annot = self.type.type_annotation(**kwargs) - if self.optional and self.client_default_value is None: - return f"Optional[{type_annot}]" - return type_annot - - def docstring_text(self, **kwargs: Any) -> str: - return self.type.docstring_text(**kwargs) - - def docstring_type(self, **kwargs: Any) -> str: - return self.type.docstring_type(**kwargs) - - @property - def serialization_type(self) -> str: - return self.type.serialization_type - - def _imports_shared(self, async_mode: bool, **_: Any) -> FileImport: - file_import = FileImport(self.code_model) - if self.optional and self.client_default_value is None: - file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) - if self.added_on and self.implementation != "Client": - file_import.add_submodule_import( - f"{'.' if async_mode else ''}.._validation", - "api_version_validation", - ImportType.LOCAL, - ) - if isinstance(self.type, CombinedType) and self.type.name: - file_import.add_submodule_import( - "..." if async_mode else "..", - "_types", - ImportType.LOCAL, - TypingSection.TYPING, - ) - return file_import - - def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: - file_import = self._imports_shared(async_mode, **kwargs) - # special logic for api-version parameter - if not self.is_api_version: - file_import.merge(self.type.imports(async_mode=async_mode, **kwargs)) - if self.default_to_unset_sentinel: - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) - file_import.define_mypy_type( - "_Unset: Any", - "object()", - ) - return file_import - - def imports_for_multiapi(self, async_mode: bool, **kwargs: Any) -> FileImport: - file_import = self._imports_shared(async_mode, **kwargs) - file_import.merge(self.type.imports_for_multiapi(async_mode=async_mode, **kwargs)) - return file_import - - @property - def method_location(self) -> ParameterMethodLocation: - raise NotImplementedError("Please implement in children") - - @property - def description_keyword(self) -> str: - return "param" if self.method_location == ParameterMethodLocation.POSITIONAL else "keyword" - - @property - def docstring_type_keyword(self) -> str: - return "type" if self.method_location == ParameterMethodLocation.POSITIONAL else "paramtype" - - @property - @abc.abstractmethod - def in_method_signature(self) -> bool: ... - - def method_signature(self, async_mode: bool) -> str: - type_annot = self.type_annotation(async_mode=async_mode) - if self.client_default_value is not None or self.optional: - return f"{self.client_name}: {type_annot} = {self.client_default_value_declaration}," - if self.default_to_unset_sentinel: - return f"{self.client_name}: {type_annot} = _Unset," - return f"{self.client_name}: {type_annot}," - - -class BodyParameter(_ParameterBase): - """Body parameter.""" - - @property - def entries(self) -> List["BodyParameter"]: - return [BodyParameter.from_yaml(e, self.code_model) for e in self.yaml_data.get("entries", [])] - - @property - def is_form_data(self) -> bool: - # hacky, but rn in legacy, there is no formdata model type, it's just a dict - # with all of the entries splatted out - return self.type.is_form_data or bool(self.entries) - - @property - def is_partial_body(self) -> bool: - """Whether it's part of a bigger body parameter, i.e. a MultipartBodyParameter""" - return self.yaml_data.get("isPartialBody", False) - - @property - def method_location(self) -> ParameterMethodLocation: - return ParameterMethodLocation.KWARG if self.constant else ParameterMethodLocation.POSITIONAL - - @property - def in_method_signature(self) -> bool: - if self.yaml_data.get("entries"): - # Right now, only legacy generates with multipart bodies and entries - # and legacy generates with the multipart body arguments splatted out - return False - return not (self.flattened or self.grouped_by) - - @property - def content_types(self) -> List[str]: - return self.yaml_data["contentTypes"] - - @property - def default_content_type(self) -> str: - return self.yaml_data["defaultContentType"] - - @property - def has_json_model_type(self) -> bool: - if isinstance(self.type, CombinedType): - return self.type.target_model_subtype((JSONModelType,)) is not None - return isinstance(self.type, JSONModelType) - - def imports(self, async_mode: bool, **kwargs: Any) -> FileImport: - file_import = super().imports(async_mode, **kwargs) - if self.is_form_data: - relative_path = "..." if async_mode else ".." - file_import.add_submodule_import( - f"{relative_path}_vendor", - "prepare_multipart_form_data", - ImportType.LOCAL, - ) - file_import.add_submodule_import("typing", "List", ImportType.STDLIB) - return file_import - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "BodyParameter": - return cls( - yaml_data=yaml_data, - code_model=code_model, - type=code_model.lookup_type(id(yaml_data["type"])), - ) - - -EntryBodyParameterType = TypeVar("EntryBodyParameterType", bound=Union[BodyParameter, "RequestBuilderBodyParameter"]) - - -class Parameter(_ParameterBase): - """Basic Parameter class""" - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - type: BaseType, - ) -> None: - super().__init__(yaml_data, code_model, type=type) - - self.skip_url_encoding: bool = self.yaml_data.get("skipUrlEncoding", False) - self.explode: bool = self.yaml_data.get("explode", False) - self.in_overriden: bool = self.yaml_data.get("inOverriden", False) - self.delimiter: Optional[ParameterDelimeter] = self.yaml_data.get("delimiter") - self._default_to_unset_sentinel: bool = False - - @property - def hide_in_operation_signature(self) -> bool: - if self.code_model.options["version_tolerant"] and self.client_name == "maxpagesize": - return True - return False - - @property - def in_method_signature(self) -> bool: - return not (self.wire_name == "Accept" or self.grouped_by or self.flattened) - - @property - def full_client_name(self) -> str: - if self.implementation == "Client": - return f"self._config.{self.client_name}" - return self.client_name - - @property - def xml_serialization_ctxt(self) -> str: - return self.type.xml_serialization_ctxt or "" - - @property - def is_content_type(self) -> bool: - return bool(self.wire_name) and self.wire_name.lower() == "content-type" - - @property - def method_location( # pylint: disable=too-many-return-statements - self, - ) -> ParameterMethodLocation: - if not self.in_method_signature: - raise ValueError(f"Parameter '{self.client_name}' is not in the method.") - if self.code_model.options["models_mode"] == "dpg" and self.in_flattened_body: - return ParameterMethodLocation.KEYWORD_ONLY - if self.grouper: - return ParameterMethodLocation.POSITIONAL - if self.constant and self.wire_name != "Content-Type": - return ParameterMethodLocation.KWARG - if self.is_content_type: - if self.in_overload: - return ParameterMethodLocation.KEYWORD_ONLY - return ParameterMethodLocation.KWARG - query_or_header = self.location in ( - ParameterLocation.HEADER, - ParameterLocation.QUERY, - ) - if self.code_model.options["only_path_and_body_params_positional"] and query_or_header: - return ParameterMethodLocation.KEYWORD_ONLY - return ParameterMethodLocation.POSITIONAL - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel"): - return cls( - yaml_data=yaml_data, - code_model=code_model, - type=code_model.lookup_type(id(yaml_data["type"])), - ) - - -class ClientParameter(Parameter): - """Client parameter""" - - @property - def is_host(self) -> bool: - return self.wire_name == "$host" - - @property - def method_location(self) -> ParameterMethodLocation: - if self.constant: - return ParameterMethodLocation.KWARG - if ( - self.is_host - and (self.code_model.options["version_tolerant"] or self.code_model.options["low_level_client"]) - and not self.code_model.options["azure_arm"] - ): - # this means i am the base url - return ParameterMethodLocation.KEYWORD_ONLY - return ParameterMethodLocation.POSITIONAL - - -class ConfigParameter(Parameter): - """Config Parameter""" - - @property - def in_method_signature(self) -> bool: - return not self.is_host - - @property - def is_host(self) -> bool: - return self.wire_name == "$host" - - @property - def method_location(self) -> ParameterMethodLocation: - if self.constant: - return ParameterMethodLocation.KWARG - return ParameterMethodLocation.POSITIONAL diff --git a/packages/autorest.python/generator/pygen/codegen/models/parameter_list.py b/packages/autorest.python/generator/pygen/codegen/models/parameter_list.py deleted file mode 100644 index 73278584a34..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/parameter_list.py +++ /dev/null @@ -1,390 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import logging -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - TYPE_CHECKING, - Union, - Generic, - TypeVar, - cast, -) -from abc import abstractmethod -from collections.abc import MutableSequence -from enum import Enum - -from .request_builder_parameter import ( - RequestBuilderBodyParameter, - RequestBuilderParameter, -) -from .parameter import ( - ParameterLocation, - BodyParameter, - Parameter, - ParameterMethodLocation, - ClientParameter, - ConfigParameter, -) - -ParameterType = TypeVar("ParameterType", bound=Union[Parameter, RequestBuilderParameter]) -BodyParameterType = TypeVar("BodyParameterType", bound=Union[BodyParameter, RequestBuilderBodyParameter]) - -if TYPE_CHECKING: - from .code_model import CodeModel - - -class ParameterImplementation(Enum): - METHOD = "method" - CLIENT = "client" - - -_LOGGER = logging.getLogger(__name__) - - -def method_signature_helper(positional: List[str], keyword_only: Optional[List[str]], kwarg_params: List[str]): - keyword_only = keyword_only or [] - return positional + keyword_only + kwarg_params - - -def _sort(params): - return sorted(params, key=lambda x: not (x.client_default_value or x.optional), reverse=True) - - -class _ParameterListBase( - MutableSequence, Generic[ParameterType, BodyParameterType] -): # pylint: disable=too-many-public-methods - """Base class for all of our different ParameterList classes""" - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - parameters: List[ParameterType], - body_parameter: Optional[BodyParameterType] = None, - ) -> None: - self.yaml_data = yaml_data - self.code_model = code_model - self.parameters = parameters or [] - self._body_parameter = body_parameter - - # MutableSequence - - def __getitem__(self, index): - if isinstance(index, str): - raise TypeError(f"{index} is invalid type") - return self.parameters[index] - - def __len__(self) -> int: - return len(self.parameters) - - def __setitem__(self, index, parameter): - self.parameters[index] = parameter - - def __delitem__(self, index): - del self.parameters[index] - - def insert(self, index: int, value: ParameterType) -> None: - self.parameters.insert(index, value) - - # Parameter helpers - - @staticmethod - @abstractmethod - def parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], ParameterType]: - """Callable for creating parameters""" - - @staticmethod - @abstractmethod - def body_parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], BodyParameterType]: - """Callable for creating body parameters""" - - @property - def grouped(self) -> List[Union[ParameterType, BodyParameterType]]: - """All parameters that are inside a parameter group""" - params: List[Union[ParameterType, BodyParameterType]] = [p for p in self.parameters if p.grouped_by] - if self.has_body and self.body_parameter.grouped_by: - params.append(self.body_parameter) - return params - - @property - def has_form_data_body(self): - return self.has_body and self.body_parameter.is_form_data - - @property - def has_body(self) -> bool: - """Whether there is a body parameter in the parameter list""" - return bool(self._body_parameter) - - @property - def path(self) -> List[ParameterType]: - """All path parameters""" - return [p for p in self.parameters if p.location in (ParameterLocation.PATH, ParameterLocation.ENDPOINT_PATH)] - - @property - def query(self) -> List[ParameterType]: - """All query parameters""" - return [p for p in self.parameters if p.location == ParameterLocation.QUERY] - - @property - def headers(self) -> List[ParameterType]: - """All header parameters""" - return [p for p in self.parameters if p.location == ParameterLocation.HEADER] - - @property - def constant(self) -> List[Union[ParameterType, BodyParameterType]]: - """All constant parameters""" - return [p for p in self.parameters if p.constant] - - @property - def positional(self) -> List[Union[ParameterType, BodyParameterType]]: - """All positional parameters""" - return _sort( - [p for p in self.unsorted_method_params if p.method_location == ParameterMethodLocation.POSITIONAL] - ) - - @property - def keyword_only(self) -> List[Union[ParameterType, BodyParameterType]]: - """All keyword only parameters""" - return _sort( - [p for p in self.unsorted_method_params if p.method_location == ParameterMethodLocation.KEYWORD_ONLY] - ) - - @property - def kwarg(self) -> List[Union[ParameterType, BodyParameterType]]: - """All kwargs""" - return _sort([p for p in self.unsorted_method_params if p.method_location == ParameterMethodLocation.KWARG]) - - @property - def body_parameter(self) -> BodyParameterType: - """The body parameter of the parameter list. Will only ever be at most one.""" - if not self._body_parameter: - raise ValueError("There is no body parameter") - return self._body_parameter - - @property - @abstractmethod - def implementation(self) -> str: - """Whether this is a client or a method parameter""" - - @property - def unsorted_method_params(self) -> List[Union[ParameterType, BodyParameterType]]: - """Method params before sorting""" - method_params: List[Union[ParameterType, BodyParameterType]] = [ - p - for p in self.parameters - if p.in_method_signature - and p.implementation == self.implementation - and (self.code_model.is_legacy or not p.hide_in_method) - ] - if self._body_parameter: - if self._body_parameter.in_method_signature: - method_params.append(self._body_parameter) - try: - # i am a multipart body parameter - # Only legacy generates operations with me, so I will follow the legacy rules - # I will splat out my entries as individual entries - method_params.extend(self._body_parameter.entries) # type: ignore - except AttributeError: - pass - return method_params - - @property - def method(self) -> List[Union[ParameterType, BodyParameterType]]: - """Sorted method params. First positional, then keyword only, then kwarg""" - return self.positional + self.keyword_only + self.kwarg - - def method_signature(self, async_mode: bool) -> List[str]: - """Method signature for this parameter list.""" - return method_signature_helper( - positional=self.method_signature_positional(async_mode), - keyword_only=self.method_signature_keyword_only(async_mode), - kwarg_params=self.method_signature_kwargs, - ) - - def method_signature_positional(self, async_mode: bool) -> List[str]: - """Signature for positional parameters""" - return [parameter.method_signature(async_mode) for parameter in self.positional] - - def method_signature_keyword_only(self, async_mode: bool) -> List[str]: - """Signature for keyword only parameters""" - result = [ - parameter.method_signature(async_mode) - for parameter in self.keyword_only - if not parameter.hide_in_operation_signature - ] - return ["*,"] + result if result else [] - - @property - def method_signature_kwargs(self) -> List[str]: - """Signature for kwargs""" - return ["**kwargs: Any"] - - @property - def kwargs_to_pop(self) -> List[Union[ParameterType, BodyParameterType]]: - """Method kwargs we want to pop""" - # don't want to pop bodies unless it's a constant - kwargs_to_pop = self.kwarg - return [k for k in kwargs_to_pop if k.location != ParameterLocation.BODY or k.constant] - - @property - def call(self) -> List[str]: - """How to pass in parameters to call the operation""" - retval = [p.client_name for p in self.method if p.method_location == ParameterMethodLocation.POSITIONAL] - retval.extend( - [ - f"{p.client_name}={p.client_name}" - for p in self.method - if p.method_location == ParameterMethodLocation.KEYWORD_ONLY - ] - ) - retval.append("**kwargs") - return retval - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel"): - parameters = [cls.parameter_creator()(parameter, code_model) for parameter in yaml_data["parameters"]] - body_parameter = None - if yaml_data.get("bodyParameter"): - body_parameter = cls.body_parameter_creator()(yaml_data["bodyParameter"], code_model) - return cls( - yaml_data, - code_model, - parameters=parameters, - body_parameter=body_parameter, - ) - - -class _ParameterList(_ParameterListBase[Parameter, BodyParameter]): # pylint: disable=unsubscriptable-object - """Base Parameter class for the two operation ParameterLists""" - - @staticmethod - def parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], Parameter]: - return Parameter.from_yaml - - @staticmethod - def body_parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], BodyParameter]: - return BodyParameter.from_yaml - - @property - def implementation(self) -> str: - return "Method" - - @property - def path(self) -> List[Parameter]: - return [k for k in super().path if k.location == ParameterLocation.ENDPOINT_PATH] - - -class ParameterList(_ParameterList): - """ParameterList is the parameter list for Operation classes""" - - -class _RequestBuilderParameterList( - _ParameterListBase[RequestBuilderParameter, RequestBuilderBodyParameter] # pylint: disable=unsubscriptable-object -): - """_RequestBuilderParameterList is base parameter list for RequestBuilder classes""" - - @staticmethod - def parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], RequestBuilderParameter]: - return RequestBuilderParameter.from_yaml - - @staticmethod - def body_parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], RequestBuilderBodyParameter]: - return RequestBuilderBodyParameter.from_yaml - - @property - def implementation(self) -> str: - return "Method" - - @property - def unsorted_method_params( - self, - ) -> List[Union[RequestBuilderParameter, RequestBuilderBodyParameter]]: - # don't have access to client params in request builder - retval = [ - p - for p in super().unsorted_method_params - if not (p.location == ParameterLocation.BODY and cast(RequestBuilderBodyParameter, p).is_partial_body) - ] - retval.extend([p for p in self.parameters if p.implementation == "Client" and p.in_method_signature]) - return retval - - @property - def path(self) -> List[RequestBuilderParameter]: - return [p for p in super().path if p.location != ParameterLocation.ENDPOINT_PATH] - - @property - def constant( - self, - ) -> List[Union[RequestBuilderParameter, RequestBuilderBodyParameter]]: - """All constant parameters""" - return [p for p in super().constant if p.location != ParameterLocation.ENDPOINT_PATH] - - -class RequestBuilderParameterList(_RequestBuilderParameterList): - """Parameter list for Request Builder""" - - -class OverloadedRequestBuilderParameterList(_RequestBuilderParameterList): - """Parameter list for OverloadedRequestBuilder""" - - -class _ClientGlobalParameterList(_ParameterListBase[ParameterType, BodyParameter]): # pylint: disable=abstract-method - """Base parameter list for client and config classes""" - - @staticmethod - def body_parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], BodyParameter]: - return BodyParameter.from_yaml - - @property - def implementation(self) -> str: - return "Client" - - @property - def credential(self) -> Optional[ParameterType]: - try: - return next(p for p in self.parameters if p.client_name == "credential") - except StopIteration: - return None - - @property - def path(self) -> List[ParameterType]: - return [p for p in super().path if p.location == ParameterLocation.ENDPOINT_PATH] - - -class ClientGlobalParameterList(_ClientGlobalParameterList[ClientParameter]): - """Parameter list for Client class""" - - @staticmethod - def parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], ClientParameter]: - return ClientParameter.from_yaml - - @property - def path(self) -> List[ClientParameter]: - return [p for p in super().path if not p.is_host] - - @property - def host(self) -> Optional[ClientParameter]: - """Get the host parameter""" - try: - return next(p for p in self.parameters if p.is_host) - except StopIteration: - return None - - -class ConfigGlobalParameterList(_ClientGlobalParameterList[ConfigParameter]): - """Parameter list for config""" - - @staticmethod - def parameter_creator() -> Callable[[Dict[str, Any], "CodeModel"], ConfigParameter]: - return ConfigParameter.from_yaml - - @property - def implementation(self) -> str: - return "Client" diff --git a/packages/autorest.python/generator/pygen/codegen/models/primitive_types.py b/packages/autorest.python/generator/pygen/codegen/models/primitive_types.py deleted file mode 100644 index a4fc0392a6d..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/primitive_types.py +++ /dev/null @@ -1,640 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import datetime -import decimal -from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING - -from .base import BaseType -from .imports import FileImport, ImportType, TypingSection -from .utils import add_to_description - -if TYPE_CHECKING: - from .code_model import CodeModel - - -class RawString(object): - def __init__(self, string: str) -> None: - self.string = string - - def __repr__(self) -> str: - return "r'{}'".format(self.string.replace("'", "\\'")) - - -class PrimitiveType(BaseType): # pylint: disable=abstract-method - def description(self, *, is_operation_file: bool) -> str: # pylint: disable=unused-argument - return "" - - def type_annotation(self, **kwargs: Any) -> str: - return self.docstring_type(**kwargs) - - def docstring_text(self, **kwargs: Any) -> str: - return self.docstring_type(**kwargs) - - def get_json_template_representation( - self, - *, - optional: bool = True, - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - comment = "" - if optional: - comment = add_to_description(comment, "Optional.") - if self.client_default_value is not None: - client_default_value_declaration = client_default_value_declaration or self.get_declaration( - self.client_default_value - ) - if client_default_value_declaration: - comment = add_to_description(comment, f"Default value is {client_default_value_declaration}.") - else: - client_default_value_declaration = self.default_template_representation_declaration - if description: - comment = add_to_description(comment, description) - if comment: - comment = f"# {comment}" - return client_default_value_declaration + ("" if self.code_model.for_test else comment) - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration(self.docstring_type()) - - -class BooleanType(PrimitiveType): - @property - def serialization_type(self) -> str: - return "bool" - - def docstring_type(self, **kwargs: Any) -> str: - return "bool" - - @property - def instance_check_template(self) -> str: - return "isinstance({}, bool)" - - -class BinaryType(PrimitiveType): - def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.type = "IO" - - @property - def serialization_type(self) -> str: - return self.type - - def docstring_type(self, **kwargs: Any) -> str: - return f"{self.type}[bytes]" - - def type_annotation(self, **kwargs: Any) -> str: - return f"{self.type}[bytes]" - - def docstring_text(self, **kwargs: Any) -> str: - return f"{self.type}[bytes]" - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration(b"bytes") - - def imports(self, **kwargs: Any) -> FileImport: - from .combined_type import CombinedType - from .operation import OperationBase - - file_import = FileImport(self.code_model) - file_import.add_submodule_import("typing", "IO", ImportType.STDLIB) - operation = kwargs.get("operation") - if ( - isinstance(operation, OperationBase) - and operation.parameters.has_body - and isinstance(operation.parameters.body_parameter.type, CombinedType) - ): - file_import.add_submodule_import("io", "IOBase", ImportType.STDLIB) - return file_import - - @property - def instance_check_template(self) -> str: - return "isinstance({}, (IOBase, bytes))" - - -class BinaryIteratorType(PrimitiveType): - def _iterator_name(self, **kwargs: Any) -> str: - return "AsyncIterator" if kwargs.pop("async_mode") else "Iterator" - - @property - def serialization_type(self) -> str: - return "IO" - - def docstring_type(self, **kwargs: Any) -> str: - return f"{self._iterator_name(**kwargs)}[bytes]" - - def type_annotation(self, **kwargs: Any) -> str: - return f"{self._iterator_name(**kwargs)}[bytes]" - - def docstring_text(self, **kwargs: Any) -> str: - return f"{self._iterator_name(**kwargs)}[bytes]" - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration(b"bytes") - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_submodule_import("typing", self._iterator_name(**kwargs), ImportType.STDLIB) - return file_import - - @property - def instance_check_template(self) -> str: - return "getattr({}, '__aiter__', None) is not None or getattr({}, '__iter__', None) is not None" - - -class AnyType(PrimitiveType): - @property - def serialization_type(self) -> str: - return "object" - - def docstring_type(self, **kwargs: Any) -> str: - return "any" - - def type_annotation(self, **kwargs: Any) -> str: - return "Any" - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration({}) - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, TypingSection.CONDITIONAL) - return file_import - - @property - def instance_check_template(self) -> str: - raise ValueError("Shouldn't do instance check on an anytype, it can be anything") - - -class AnyObjectType(PrimitiveType): - @property - def serialization_type(self) -> str: - return "object" - - def docstring_type(self, **kwargs: Any) -> str: - return "JSON" - - def type_annotation(self, **kwargs: Any) -> str: - return "JSON" - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration({}) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, MutableMapping)" - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.define_mutable_mapping_type() - return file_import - - @property - def type_description(self) -> str: - return "JSON" - - -class NumberType(PrimitiveType): # pylint: disable=abstract-method - def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.precision: Optional[int] = yaml_data.get("precision") - self.multiple: Optional[int] = yaml_data.get("multipleOf") - self.maximum: Optional[int] = yaml_data.get("maximum") - self.minimum: Optional[int] = yaml_data.get("minimum") - self.exclusive_maximum: Optional[int] = yaml_data.get("exclusiveMaximum") - self.exclusive_minimum: Optional[int] = yaml_data.get("exclusiveMinimum") - - @property - def serialization_constraints(self) -> List[str]: - validation_constraints = [ - (f"maximum_ex={self.maximum}" if self.maximum is not None and self.exclusive_maximum else None), - (f"maximum={self.maximum}" if self.maximum is not None and not self.exclusive_maximum else None), - (f"minimum_ex={self.minimum}" if self.minimum is not None and self.exclusive_minimum else None), - (f"minimum={self.minimum}" if self.minimum is not None and not self.exclusive_minimum else None), - f"multiple={self.multiple}" if self.multiple else None, - ] - return [x for x in validation_constraints if x is not None] - - @property - def validation(self) -> Optional[Dict[str, Union[bool, int, str]]]: - validation: Dict[str, Union[bool, int, str]] = {} - if self.maximum is not None: - if self.exclusive_maximum: - validation["maximum_ex"] = self.maximum - else: - validation["maximum"] = self.maximum - if self.minimum is not None: - if self.exclusive_minimum: - validation["minimum_ex"] = self.minimum - else: - validation["minimum"] = self.minimum - if self.multiple: - validation["multiple"] = self.multiple - return validation or None - - @property - def default_template_representation_declaration(self) -> str: - default_value = 0 if self.docstring_type() == "int" else 0.0 - return self.get_declaration(default_value) - - -class IntegerType(NumberType): - @property - def serialization_type(self) -> str: - return "int" - - def docstring_type(self, **kwargs: Any) -> str: - return "int" - - def type_annotation(self, **kwargs: Any) -> str: - return "int" - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration(0) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, int)" - - -class FloatType(NumberType): - @property - def serialization_type(self) -> str: - return "float" - - def docstring_type(self, **kwargs: Any) -> str: - return "float" - - def type_annotation(self, **kwargs: Any) -> str: - return "float" - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration(0.0) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, float)" - - -class DecimalType(NumberType): - @property - def serialization_type(self) -> str: - return "decimal" - - def docstring_type(self, **kwargs: Any) -> str: - return "~" + self.type_annotation() - - def type_annotation(self, **kwargs: Any) -> str: - return "decimal.Decimal" - - def docstring_text(self, **kwargs: Any) -> str: - return self.type_annotation() - - def get_declaration(self, value: decimal.Decimal) -> str: - return str(value) - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_import("decimal", ImportType.STDLIB) - return file_import - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration(decimal.Decimal("0.0")) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, decimal.Decimal)" - - -class StringType(PrimitiveType): - def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.max_length: Optional[int] = yaml_data.get("maxLength") - self.min_length: Optional[int] = ( - yaml_data.get("minLength", 0) if yaml_data.get("maxLength") else yaml_data.get("minLength") - ) - self.pattern: Optional[str] = yaml_data.get("pattern") - - @property - def serialization_constraints(self) -> List[str]: - validation_constraints = [ - f"max_length={self.max_length}" if self.max_length is not None else None, - f"min_length={self.min_length}" if self.min_length is not None else None, - f"pattern={RawString(self.pattern)}" if self.pattern else None, - ] - return [x for x in validation_constraints if x is not None] - - @property - def validation(self) -> Optional[Dict[str, Union[bool, int, str]]]: - validation: Dict[str, Union[bool, int, str]] = {} - if self.max_length is not None: - validation["max_length"] = self.max_length - if self.min_length is not None: - validation["min_length"] = self.min_length - if self.pattern: - # https://github.com/Azure/autorest.python/issues/407 - validation["pattern"] = RawString(self.pattern) # type: ignore - return validation or None - - def get_declaration(self, value) -> str: - return f'"{value}"' - - @property - def serialization_type(self) -> str: - return "str" - - def docstring_type(self, **kwargs: Any) -> str: - return "str" - - @property - def instance_check_template(self) -> str: - return "isinstance({}, str)" - - -class DatetimeType(PrimitiveType): - def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.encode = ( - "rfc3339" - if yaml_data.get("encode", "date-time") == "date-time" or yaml_data.get("encode", "date-time") == "rfc3339" - else "rfc7231" - ) - - @property - def serialization_type(self) -> str: - formats_to_attribute_type = { - "rfc3339": "iso-8601", - "rfc7231": "rfc-1123", - } - return formats_to_attribute_type[self.encode] - - def docstring_type(self, **kwargs: Any) -> str: - return "~" + self.type_annotation() - - def type_annotation(self, **kwargs: Any) -> str: - return "datetime.datetime" - - def docstring_text(self, **kwargs: Any) -> str: - return "datetime" - - def get_declaration(self, value: datetime.datetime) -> str: - """Could be discussed, since technically I should return a datetime object, - but msrest will do fine. - """ - return f'"{value}"' - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_import("datetime", ImportType.STDLIB) - return file_import - - @property - def default_template_representation_declaration(self): - return self.get_declaration(datetime.datetime(2020, 2, 20)) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, datetime.datetime)" - - def imports_for_sample(self) -> FileImport: - file_import = super().imports_for_sample() - file_import.add_import("isodate", ImportType.STDLIB) - return file_import - - @staticmethod - def serialize_sample_value(value: Any) -> str: - return f"isodate.parse_datetime({repr(value)})" - - -class TimeType(PrimitiveType): - @property - def serialization_type(self) -> str: - return "time" - - def docstring_type(self, **kwargs: Any) -> str: - return "~" + self.type_annotation() - - def type_annotation(self, **kwargs: Any) -> str: - return "datetime.time" - - def docstring_text(self, **kwargs: Any) -> str: - return "time" - - def get_declaration(self, value: datetime.time) -> str: - """Could be discussed, since technically I should return a time object, - but msrest will do fine. - """ - return f'"{value}"' - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_import("datetime", ImportType.STDLIB) - return file_import - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration(datetime.time(12, 30, 0)) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, datetime.time)" - - def imports_for_sample(self) -> FileImport: - file_import = super().imports_for_sample() - file_import.add_import("isodate", ImportType.STDLIB) - return file_import - - @staticmethod - def serialize_sample_value(value: Any) -> str: - return f"isodate.parse_time({repr(value)})" - - -class UnixTimeType(PrimitiveType): - @property - def encode(self) -> str: - return "unix-timestamp" - - @property - def serialization_type(self) -> str: - return "unix-time" - - def docstring_type(self, **kwargs: Any) -> str: - return "~" + self.type_annotation() - - def type_annotation(self, **kwargs: Any) -> str: - return "datetime.datetime" - - def docstring_text(self, **kwargs: Any) -> str: - return "datetime" - - def get_declaration(self, value: datetime.datetime) -> str: - """Could be discussed, since technically I should return a datetime object, - but msrest will do fine. - """ - return f'"{value}"' - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_import("datetime", ImportType.STDLIB) - return file_import - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration(datetime.datetime(2020, 2, 20)) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, datetime.time)" - - def imports_for_sample(self) -> FileImport: - file_import = super().imports_for_sample() - file_import.add_import("datetime", ImportType.STDLIB) - return file_import - - @staticmethod - def serialize_sample_value(value: Any) -> str: - return f"datetime.datetime.fromtimestamp({repr(value)}, datetime.timezone.utc)" - - -class DateType(PrimitiveType): - @property - def serialization_type(self) -> str: - return "date" - - def docstring_type(self, **kwargs: Any) -> str: - return "~" + self.type_annotation() - - def type_annotation(self, **kwargs: Any) -> str: - return "datetime.date" - - def docstring_text(self, **kwargs: Any) -> str: - return "date" - - def get_declaration(self, value: datetime.date) -> str: - """Could be discussed, since technically I should return a datetime object, - but msrest will do fine. - """ - return f'"{value}"' - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_import("datetime", ImportType.STDLIB) - return file_import - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration(datetime.date(2020, 2, 20)) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, datetime.date)" - - def imports_for_sample(self) -> FileImport: - file_import = super().imports_for_sample() - file_import.add_import("isodate", ImportType.STDLIB) - return file_import - - @staticmethod - def serialize_sample_value(value: Any) -> str: - return f"isodate.parse_date({repr(value)})" - - -class DurationType(PrimitiveType): - @property - def serialization_type(self) -> str: - return "duration" - - def docstring_type(self, **kwargs: Any) -> str: - return "~" + self.type_annotation() - - def type_annotation(self, **kwargs: Any) -> str: - return "datetime.timedelta" - - def docstring_text(self, **kwargs: Any) -> str: - return "timedelta" - - def get_declaration(self, value: datetime.timedelta) -> str: - """Could be discussed, since technically I should return a datetime object, - but msrest will do fine. - """ - return f'"{value}"' - - def imports(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_import("datetime", ImportType.STDLIB) - return file_import - - @property - def default_template_representation_declaration(self) -> str: - return self.get_declaration(datetime.timedelta(1)) - - @property - def instance_check_template(self) -> str: - return "isinstance({}, datetime.timedelta)" - - def imports_for_sample(self) -> FileImport: - file_import = super().imports_for_sample() - file_import.add_import("isodate", ImportType.STDLIB) - return file_import - - @staticmethod - def serialize_sample_value(value: Any) -> str: - return f"isodate.parse_duration({repr(value)})" - - -class ByteArraySchema(PrimitiveType): - def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.encode = yaml_data.get("encode", "base64") - - @property - def serialization_type(self) -> str: - if self.encode == "base64url": - return "base64" - return "bytearray" - - def docstring_type(self, **kwargs: Any) -> str: - return "bytes" - - def get_declaration(self, value: str) -> str: - return f'bytes("{value}", encoding="utf-8")' - - @property - def instance_check_template(self) -> str: - return "isinstance({}, bytes)" - - -class SdkCoreType(PrimitiveType): - def __init__(self, yaml_data: Dict[str, Any], code_model: "CodeModel") -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.name = yaml_data.get("name", "") - - def docstring_type(self, **kwargs: Any) -> str: - return f"~{self.code_model.core_library}.{self.type_annotation(**kwargs)}" - - def type_annotation(self, **kwargs: Any) -> str: - return self.name - - def imports(self, **kwargs: Any) -> FileImport: - file_import = super().imports(**kwargs) - file_import.add_submodule_import("", self.name, ImportType.SDKCORE) - return file_import - - @property - def instance_check_template(self) -> str: - return f"isinstance({{}}, {self.name})" - - @property - def serialization_type(self) -> str: - return self.name diff --git a/packages/autorest.python/generator/pygen/codegen/models/property.py b/packages/autorest.python/generator/pygen/codegen/models/property.py deleted file mode 100644 index bdda254c248..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/property.py +++ /dev/null @@ -1,182 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any, Dict, Optional, TYPE_CHECKING, List, cast - -from .base import BaseModel -from .constant_type import ConstantType -from .enum_type import EnumType -from .base import BaseType -from .imports import FileImport, ImportType -from .utils import add_to_description, add_to_pylint_disable - -if TYPE_CHECKING: - from .code_model import CodeModel - from .model_type import ModelType - - -class Property(BaseModel): # pylint: disable=too-many-instance-attributes - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - type: BaseType, - ) -> None: - super().__init__(yaml_data, code_model) - self.wire_name: str = self.yaml_data["wireName"] - self.client_name: str = self.yaml_data["clientName"] - self.type = type - self.optional: bool = self.yaml_data["optional"] - self.readonly: bool = self.yaml_data.get("readonly", False) - self.visibility: List[str] = self.yaml_data.get("visibility", []) - self.is_polymorphic: bool = self.yaml_data.get("isPolymorphic", False) - self.is_discriminator: bool = yaml_data.get("isDiscriminator", False) - self.client_default_value = yaml_data.get("clientDefaultValue", None) - if self.client_default_value is None: - self.client_default_value = self.type.client_default_value - self.flattened_names: List[str] = yaml_data.get("flattenedNames", []) - self.is_multipart_file_input: bool = yaml_data.get("isMultipartFileInput", False) - self.flatten = self.yaml_data.get("flatten", False) and not getattr(self.type, "flattened_property", False) - - @property - def pylint_disable(self) -> str: - retval: str = "" - if self.yaml_data.get("pylintDisable"): - retval = add_to_pylint_disable(retval, self.yaml_data["pylintDisable"]) - return retval - - def description(self, *, is_operation_file: bool) -> str: - from .model_type import ModelType - - description = self.yaml_data.get("description", "") - if not (self.optional or self.client_default_value): - description = add_to_description(description, "Required.") - # don't want model type documentation as part of property doc - type_description = ( - "" if isinstance(self.type, ModelType) else self.type.description(is_operation_file=is_operation_file) - ) - return add_to_description(description, type_description) - - @property - def client_default_value_declaration(self) -> str: - if self.client_default_value is not None: - return self.get_declaration(self.client_default_value) - if self.type.client_default_value is not None: - return self.get_declaration(self.type.client_default_value) - return "None" - - @property - def constant(self) -> bool: - # this bool doesn't consider you to be constant if you are a discriminator - # you also have to be required to be considered a constant - return isinstance(self.type, ConstantType) and not self.optional and not self.is_discriminator - - @property - def is_input(self): - return not (self.constant or self.readonly or self.is_discriminator) - - @property - def serialization_type(self) -> str: - return self.type.serialization_type - - @property - def msrest_deserialization_key(self) -> str: - return self.type.msrest_deserialization_key - - @property - def is_enum_discriminator(self) -> bool: - return self.is_discriminator and self.type.type == "enum" - - @property - def is_base_discriminator(self) -> bool: - """If this discriminator is on the base model for polymorphic inheritance""" - if self.is_enum_discriminator: - return self.is_polymorphic and self.client_default_value is None - return self.is_discriminator and self.is_polymorphic and cast(ConstantType, self.type).value is None - - def type_annotation(self, *, is_operation_file: bool = False) -> str: - types_type_annotation = self.type.type_annotation(is_operation_file=is_operation_file) - if self.is_multipart_file_input: - # we only support FileType or list of FileType - types_type_annotation = types_type_annotation.replace("bytes", "FileType") - if self.is_base_discriminator: - return "str" - if self.optional and self.client_default_value is None: - return f"Optional[{types_type_annotation}]" - return types_type_annotation - - def get_declaration(self, value: Any = None) -> Any: - return self.type.get_declaration(value) - - def get_json_template_representation( - self, - *, - optional: bool = True, # pylint: disable=unused-argument - client_default_value_declaration: Optional[str] = None, - description: Optional[str] = None, - ) -> Any: - if self.is_multipart_file_input: - file_type_str = '"filetype"' if self.code_model.for_test else "filetype" - return f"[{file_type_str}]" if self.type.type == "list" else file_type_str - if self.client_default_value: - client_default_value_declaration = self.get_declaration(self.client_default_value) - if self.description(is_operation_file=True): - description = self.description(is_operation_file=True) - # make sure there is no \n otherwise the json template will be invalid - description = (description or "").replace("\n", " ") - return self.type.get_json_template_representation( - optional=self.optional, - client_default_value_declaration=client_default_value_declaration, - description=description, - ) - - def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: - from .model_type import ModelType - - if isinstance(self.type, ModelType): - self.type.get_polymorphic_subtypes(polymorphic_subtypes) - - @property - def validation(self) -> Optional[Dict[str, Any]]: - retval: Dict[str, Any] = {} - if not self.optional: - retval["required"] = True - if self.readonly: - retval["readonly"] = True - if self.constant: - retval["constant"] = True - retval.update(self.type.validation or {}) - return retval or None - - def imports(self, **kwargs) -> FileImport: - file_import = FileImport(self.code_model) - if self.is_discriminator and isinstance(self.type, EnumType): - return file_import - file_import.merge(self.type.imports(**kwargs, relative_path="..", model_typing=True)) - if self.optional and self.client_default_value is None: - file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) - if self.code_model.options["models_mode"] == "dpg": - file_import.add_submodule_import( - ".._model_base", - "rest_discriminator" if self.is_discriminator else "rest_field", - ImportType.LOCAL, - ) - if self.is_multipart_file_input: - file_import.add_submodule_import(".._vendor", "FileType", ImportType.LOCAL) - return file_import - - @classmethod - def from_yaml( - cls, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - ) -> "Property": - from . import build_type # pylint: disable=import-outside-toplevel - - return cls( - yaml_data=yaml_data, - code_model=code_model, - type=build_type(yaml_data["type"], code_model), - ) diff --git a/packages/autorest.python/generator/pygen/codegen/models/request_builder.py b/packages/autorest.python/generator/pygen/codegen/models/request_builder.py deleted file mode 100644 index b8786adba6f..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/request_builder.py +++ /dev/null @@ -1,189 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import ( - Any, - Callable, - Dict, - List, - TypeVar, - TYPE_CHECKING, - Union, - Optional, -) -from abc import abstractmethod - -from .base_builder import BaseBuilder -from .utils import add_to_pylint_disable, NAME_LENGTH_LIMIT -from .parameter_list import ( - RequestBuilderParameterList, - OverloadedRequestBuilderParameterList, -) -from .imports import FileImport, ImportType, TypingSection, MsrestImportType - -if TYPE_CHECKING: - from .code_model import CodeModel - from .client import Client - -ParameterListType = TypeVar( - "ParameterListType", - bound=Union[RequestBuilderParameterList, OverloadedRequestBuilderParameterList], -) - - -class RequestBuilderBase(BaseBuilder[ParameterListType, List["RequestBuilder"]]): - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - client: "Client", - name: str, - parameters: ParameterListType, - *, - overloads: Optional[List["RequestBuilder"]] = None, - ) -> None: - super().__init__( - code_model=code_model, - client=client, - yaml_data=yaml_data, - name=name, - parameters=parameters, - overloads=overloads, - ) - self.overloads: List["RequestBuilder"] = overloads or [] - self.url: str = yaml_data["url"] - self.method: str = yaml_data["method"] - self.want_tracing = False - - @property - def has_form_data_body(self): - return self.parameters.has_form_data_body - - @property - def is_lro(self) -> bool: - return self.yaml_data.get("discriminator") in ("lro", "lropaging") - - @property - def pylint_disable(self) -> str: - if len(self.name) > NAME_LENGTH_LIMIT: - return add_to_pylint_disable("", "name-too-long") - return "" - - def response_type_annotation(self, **kwargs) -> str: - return "HttpRequest" - - def response_docstring_text(self, **kwargs) -> str: - return ( - f"Returns an :class:`~{self.response_docstring_type()}` that you will pass to the client's " - + "`send_request` method. See https://aka.ms/azsdk/dpcodegen/python/send_request for how to " - + "incorporate this response into your code flow." - ) - - def response_docstring_type(self, **kwargs) -> str: - return f"~{self.code_model.core_library}.rest.HttpRequest" - - def imports(self) -> FileImport: - file_import = FileImport(self.code_model) - relative_path = ".." - if not self.code_model.options["builders_visibility"] == "embedded" and self.group_name: - relative_path = "..." if self.group_name else ".." - if self.abstract: - return file_import - for parameter in self.parameters.method: - file_import.merge(parameter.imports(async_mode=False, relative_path=relative_path, operation=self)) - - file_import.add_submodule_import( - "rest", - "HttpRequest", - ImportType.SDKCORE, - ) - - if self.parameters.headers or self.parameters.query: - file_import.add_submodule_import( - "utils", - "case_insensitive_dict", - ImportType.SDKCORE, - ) - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB, typing_section=TypingSection.CONDITIONAL) - file_import.add_msrest_import( - relative_path=( - "..." - if (not self.code_model.options["builders_visibility"] == "embedded" and self.group_name) - else ".." - ), - msrest_import_type=MsrestImportType.Serializer, - typing_section=TypingSection.REGULAR, - ) - if self.overloads and self.code_model.options["builders_visibility"] != "embedded": - file_import.add_submodule_import("typing", "overload", ImportType.STDLIB) - return file_import - - @staticmethod - @abstractmethod - def parameter_list_type() -> Callable[[Dict[str, Any], "CodeModel"], ParameterListType]: ... - - @classmethod - def get_name( - cls, - name: str, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - client: "Client", - ) -> str: - additional_mark = "" - if code_model.options["combine_operation_files"] and code_model.options["builders_visibility"] == "embedded": - additional_mark = yaml_data["groupName"] or client.yaml_data["builderPadName"] - names = [ - "build", - additional_mark, - name, - "request", - ] - return "_".join([n for n in names if n]) - - @classmethod - def from_yaml( - cls, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - client: "Client", - ): - # when combine embedded builders into one operation file, we need to avoid duplicated build function name. - # So add operation group name is effective method - - overloads = [ - RequestBuilder.from_yaml(rb_yaml_data, code_model, client) - for rb_yaml_data in yaml_data.get("overloads", []) - ] - parameter_list = cls.parameter_list_type()(yaml_data, code_model) - - return cls( - yaml_data=yaml_data, - code_model=code_model, - client=client, - name=cls.get_name(yaml_data["name"], yaml_data, code_model, client), - parameters=parameter_list, - overloads=overloads, - ) - - -class RequestBuilder(RequestBuilderBase[RequestBuilderParameterList]): - @staticmethod - def parameter_list_type() -> Callable[[Dict[str, Any], "CodeModel"], RequestBuilderParameterList]: - return RequestBuilderParameterList.from_yaml - - -class OverloadedRequestBuilder(RequestBuilderBase[OverloadedRequestBuilderParameterList]): - @staticmethod - def parameter_list_type() -> Callable[[Dict[str, Any], "CodeModel"], OverloadedRequestBuilderParameterList]: - return OverloadedRequestBuilderParameterList.from_yaml - - -def get_request_builder( - yaml_data: Dict[str, Any], code_model: "CodeModel", client: "Client" -) -> Union[RequestBuilder, OverloadedRequestBuilder]: - if yaml_data.get("overloads"): - return OverloadedRequestBuilder.from_yaml(yaml_data, code_model, client) - return RequestBuilder.from_yaml(yaml_data, code_model, client) diff --git a/packages/autorest.python/generator/pygen/codegen/models/request_builder_parameter.py b/packages/autorest.python/generator/pygen/codegen/models/request_builder_parameter.py deleted file mode 100644 index 9ed4e9e590b..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/request_builder_parameter.py +++ /dev/null @@ -1,115 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import TYPE_CHECKING, Any, Dict -from .parameter import ( - ParameterLocation, - ParameterMethodLocation, - Parameter, - BodyParameter, -) -from .base import BaseType -from .primitive_types import BinaryType, StringType -from .combined_type import CombinedType - -if TYPE_CHECKING: - from .code_model import CodeModel - - -class RequestBuilderBodyParameter(BodyParameter): - """BOdy parmaeter for RequestBuilders""" - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - if ( - isinstance(self.type, (BinaryType, StringType)) - or any("xml" in ct for ct in self.content_types) - or self.code_model.options["models_mode"] == "dpg" - ): - self.client_name = "content" - else: - self.client_name = "json" - - def type_annotation(self, **kwargs: Any) -> str: - if self.type.is_xml: - return "Any" # xml type technically not in type signature for HttpRequest content param - return super().type_annotation(**kwargs) - - @property - def in_method_signature(self) -> bool: - return ( - super().in_method_signature and not self.is_partial_body and self.code_model.options["models_mode"] != "dpg" - ) - - @property - def method_location(self) -> ParameterMethodLocation: - return ( - ParameterMethodLocation.KWARG - if (self.constant or isinstance(self.type, CombinedType)) - else ParameterMethodLocation.KEYWORD_ONLY - ) - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "RequestBuilderBodyParameter": - return super().from_yaml(yaml_data, code_model) # type: ignore - - @property - def name_in_high_level_operation(self) -> str: - if self.client_name == "json": - return "_json" - return "_content" - - -class RequestBuilderParameter(Parameter): - """Basic RequestBuilder Parameter.""" - - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - type: BaseType, - ) -> None: - super().__init__(yaml_data, code_model, type) - # we don't want any default content type behavior in request builder - if self.is_content_type: - self.client_default_value = None - if self.grouped_by and self.client_name[0] == "_": - # we don't want hidden parameters for grouped by in request builders - self.client_name = self.client_name[1:] - - @property - def hide_in_operation_signature(self) -> bool: - return False - - @property - def in_method_signature(self) -> bool: - if self.grouped_by and not self.in_flattened_body: - return True - return super().in_method_signature and not ( - self.location == ParameterLocation.ENDPOINT_PATH or self.in_flattened_body or self.grouper - ) - - @property - def full_client_name(self) -> str: - return self.client_name - - @property - def method_location(self) -> ParameterMethodLocation: - super_method_location = super().method_location - if super_method_location == ParameterMethodLocation.KWARG: - return super_method_location - if self.in_overriden and super_method_location == ParameterMethodLocation.KEYWORD_ONLY: - return ParameterMethodLocation.KWARG - if self.location != ParameterLocation.PATH: - return ParameterMethodLocation.KEYWORD_ONLY - return super_method_location - - @property - def name_in_high_level_operation(self) -> str: - if self.grouped_by: - return f"_{self.client_name}" - if self.implementation == "Client": - return f"self._config.{self.client_name}" - return self.client_name diff --git a/packages/autorest.python/generator/pygen/codegen/models/response.py b/packages/autorest.python/generator/pygen/codegen/models/response.py deleted file mode 100644 index bd78c440f12..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/response.py +++ /dev/null @@ -1,348 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Dict, Optional, List, Any, TYPE_CHECKING, Union - -from .base import BaseModel -from .base import BaseType -from .imports import FileImport, ImportType, TypingSection -from .primitive_types import BinaryType, BinaryIteratorType, ByteArraySchema -from .dictionary_type import DictionaryType -from .list_type import ListType -from .model_type import ModelType -from .combined_type import CombinedType - -if TYPE_CHECKING: - from .code_model import CodeModel - - -class ResponseHeader(BaseModel): - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - type: BaseType, - ) -> None: - super().__init__(yaml_data, code_model) - self.wire_name: str = yaml_data["wireName"] - self.type = type - - @property - def serialization_type(self) -> str: - return self.type.serialization_type - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "ResponseHeader": - from . import build_type - - return cls( - yaml_data=yaml_data, - code_model=code_model, - type=build_type(yaml_data["type"], code_model), - ) - - -class Response(BaseModel): - def __init__( - self, - yaml_data: Dict[str, Any], - code_model: "CodeModel", - *, - headers: Optional[List[ResponseHeader]] = None, - type: Optional[BaseType] = None, - ) -> None: - super().__init__(yaml_data=yaml_data, code_model=code_model) - self.status_codes: List[Union[int, str]] = yaml_data["statusCodes"] - self.headers = headers or [] - self.type = type - self.nullable = yaml_data.get("nullable") - self.default_content_type = yaml_data.get("defaultContentType") - - @property - def result_property(self) -> str: - field = self.yaml_data.get("resultProperty") - if field: - return f'.get("{field}")' - return "" - - def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: - if self.type: - self.type.get_polymorphic_subtypes(polymorphic_subtypes) - - def get_json_template_representation(self) -> Any: - if not self.type: - return None - if not isinstance(self.type, (DictionaryType, ListType, ModelType)): - return None - return self.type.get_json_template_representation() - - @property - def is_stream_response(self) -> bool: - """Is the response expected to be streamable, like a download.""" - retval = isinstance(self.type, BinaryIteratorType) or ( - isinstance(self.type, ByteArraySchema) - and bool(self.default_content_type) - and self.default_content_type != "application/json" - ) - return retval - - @property - def serialization_type(self) -> str: - if self.type: - return self.type.serialization_type - return "None" - - def type_annotation(self, **kwargs: Any) -> str: - if self.type: - kwargs["is_operation_file"] = True - type_annot = self.type.type_annotation(**kwargs) - if self.nullable: - return f"Optional[{type_annot}]" - return type_annot - return "None" - - def docstring_text(self, **kwargs: Any) -> str: - if self.nullable and self.type: - return f"{self.type.docstring_text(**kwargs)} or None" - return self.type.docstring_text(**kwargs) if self.type else "None" - - def docstring_type(self, **kwargs: Any) -> str: - if self.nullable and self.type: - return f"{self.type.docstring_type(**kwargs)} or None" - return self.type.docstring_type(**kwargs) if self.type else "None" - - def _imports_shared(self, **kwargs: Any) -> FileImport: - file_import = FileImport(self.code_model) - if self.type: - file_import.merge(self.type.imports(**kwargs)) - if self.nullable: - file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) - if isinstance(self.type, CombinedType) and self.type.name: - async_mode = kwargs.get("async_mode", False) - file_import.add_submodule_import( - "..." if async_mode else "..", - "_types", - ImportType.LOCAL, - TypingSection.TYPING, - ) - return file_import - - def imports(self, **kwargs: Any) -> FileImport: - return self._imports_shared(**kwargs) - - def imports_for_multiapi(self, **kwargs: Any) -> FileImport: - return self._imports_shared(**kwargs) - - def _get_import_type(self, input_path: str) -> ImportType: - # helper function to return imports for responses based off - # of whether we're importing from the core library, or users - # are customizing responses - return ImportType.SDKCORE if self.code_model.core_library.split(".")[0] in input_path else ImportType.THIRDPARTY - - @classmethod - def from_yaml(cls, yaml_data: Dict[str, Any], code_model: "CodeModel") -> "Response": - type = code_model.lookup_type(id(yaml_data["type"])) if yaml_data.get("type") else None - # use ByteIteratorType if we are returning a binary type - default_content_type = yaml_data.get("defaultContentType", "application/json") - if isinstance(type, BinaryType) or ( - isinstance(type, ByteArraySchema) and default_content_type != "application/json" - ): - type = BinaryIteratorType(type.yaml_data, type.code_model) - return cls( - yaml_data=yaml_data, - code_model=code_model, - headers=[ResponseHeader.from_yaml(header, code_model) for header in yaml_data["headers"]], - type=type, - ) - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} {self.status_codes}>" - - -class PagingResponse(Response): - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.item_type = self.code_model.lookup_type(id(self.yaml_data["itemType"])) - self.pager_sync: str = self.yaml_data.get("pagerSync") or f"{self.code_model.core_library}.paging.ItemPaged" - default_paging_submodule = f"{'async_' if self.code_model.is_azure_flavor else ''}paging" - self.pager_async: str = ( - self.yaml_data.get("pagerAsync") - or f"{self.code_model.core_library}.{default_paging_submodule}.AsyncItemPaged" - ) - - def get_polymorphic_subtypes(self, polymorphic_subtypes: List["ModelType"]) -> None: - return self.item_type.get_polymorphic_subtypes(polymorphic_subtypes) - - def get_json_template_representation(self) -> Any: - return self.item_type.get_json_template_representation() - - def get_pager_import_path(self, async_mode: bool) -> str: - return ".".join(self.get_pager_path(async_mode).split(".")[:-1]) - - def get_pager_path(self, async_mode: bool) -> str: - return self.pager_async if async_mode else self.pager_sync - - def get_pager(self, async_mode: bool) -> str: - return self.get_pager_path(async_mode).split(".")[-1] - - def type_annotation(self, **kwargs: Any) -> str: - iterable = "AsyncIterable" if kwargs["async_mode"] else "Iterable" - return f"{iterable}[{self.item_type.type_annotation(**kwargs)}]" - - def docstring_text(self, **kwargs: Any) -> str: - base_description = "An iterator like instance of " - if not self.code_model.options["version_tolerant"]: - base_description += "either " - return base_description + self.item_type.docstring_text(**kwargs) - - def docstring_type(self, **kwargs: Any) -> str: - return f"~{self.get_pager_path(kwargs['async_mode'])}[{self.item_type.docstring_type(**kwargs)}]" - - def _imports_shared(self, **kwargs: Any) -> FileImport: - file_import = super()._imports_shared(**kwargs) - async_mode = kwargs.get("async_mode", False) - pager = self.get_pager(async_mode) - pager_path = self.get_pager_import_path(async_mode) - - file_import.add_submodule_import(pager_path, pager, self._get_import_type(pager_path)) - return file_import - - def imports(self, **kwargs: Any) -> FileImport: - file_import = self._imports_shared(**kwargs) - async_mode = kwargs.get("async_mode") - if async_mode: - file_import.add_submodule_import( - f"{'async_' if self.code_model.is_azure_flavor else ''}paging", - "AsyncList", - ImportType.SDKCORE, - ) - - return file_import - - def imports_for_multiapi(self, **kwargs: Any) -> FileImport: - return self._imports_shared(**kwargs) - - -class LROResponse(Response): - def get_poller_path(self, async_mode: bool) -> str: - return self.yaml_data["pollerAsync"] if async_mode else self.yaml_data["pollerSync"] - - def get_poller(self, async_mode: bool) -> str: - """Get the name of the poller. Default is LROPoller / AsyncLROPoller""" - return self.get_poller_path(async_mode).split(".")[-1] - - def get_polling_method_path(self, async_mode: bool) -> str: - """Get the full name of the poller path. Default are the azure core pollers""" - return self.yaml_data["pollingMethodAsync"] if async_mode else self.yaml_data["pollingMethodSync"] - - def get_polling_method(self, async_mode: bool) -> str: - """Get the default pollint method""" - return self.get_polling_method_path(async_mode).split(".")[-1] - - @staticmethod - def get_no_polling_method_path(async_mode: bool) -> str: - """Get the path of the default of no polling method""" - return f"azure.core.polling.{'Async' if async_mode else ''}NoPolling" - - def get_no_polling_method(self, async_mode: bool) -> str: - """Get the default no polling method""" - return self.get_no_polling_method_path(async_mode).split(".")[-1] - - @staticmethod - def get_base_polling_method_path(async_mode: bool) -> str: - """Get the base polling method path. Used in docstrings and type annotations.""" - return f"azure.core.polling.{'Async' if async_mode else ''}PollingMethod" - - def get_base_polling_method(self, async_mode: bool) -> str: - """Get the base polling method.""" - return self.get_base_polling_method_path(async_mode).split(".")[-1] - - def type_annotation(self, **kwargs: Any) -> str: - return f"{self.get_poller(kwargs.get('async_mode', False))}[{super().type_annotation(**kwargs)}]" - - def docstring_type(self, **kwargs: Any) -> str: - return f"~{self.get_poller_path(kwargs.get('async_mode', False))}[{super().docstring_type(**kwargs)}]" - - def docstring_text(self, **kwargs) -> str: - super_text = super().docstring_text(**kwargs) - base_description = f"An instance of {self.get_poller(kwargs.get('async_mode', False))} that returns " - if not self.code_model.options["version_tolerant"]: - base_description += "either " - return base_description + super_text - - def _imports_shared(self, **kwargs: Any) -> FileImport: - file_import = super()._imports_shared(**kwargs) - async_mode = kwargs["async_mode"] - poller_import_path = ".".join(self.get_poller_path(async_mode).split(".")[:-1]) - poller = self.get_poller(async_mode) - file_import.add_submodule_import(poller_import_path, poller, self._get_import_type(poller_import_path)) - return file_import - - def imports(self, **kwargs: Any) -> FileImport: - file_import = self._imports_shared(**kwargs) - async_mode = kwargs["async_mode"] - - default_polling_method_import_path = ".".join(self.get_polling_method_path(async_mode).split(".")[:-1]) - default_polling_method = self.get_polling_method(async_mode) - file_import.add_submodule_import( - default_polling_method_import_path, - default_polling_method, - self._get_import_type(default_polling_method_import_path), - ) - default_no_polling_method_import_path = ".".join(self.get_no_polling_method_path(async_mode).split(".")[:-1]) - default_no_polling_method = self.get_no_polling_method(async_mode) - file_import.add_submodule_import( - default_no_polling_method_import_path, - default_no_polling_method, - self._get_import_type(default_no_polling_method_import_path), - ) - - base_polling_method_import_path = ".".join(self.get_base_polling_method_path(async_mode).split(".")[:-1]) - base_polling_method = self.get_base_polling_method(async_mode) - file_import.add_submodule_import( - base_polling_method_import_path, - base_polling_method, - self._get_import_type(base_polling_method_import_path), - ) - return file_import - - def imports_for_multiapi(self, **kwargs: Any) -> FileImport: - return self._imports_shared(**kwargs) - - -class LROPagingResponse(LROResponse, PagingResponse): - def type_annotation(self, **kwargs: Any) -> str: - paging_type_annotation = PagingResponse.type_annotation(self, **kwargs) - return f"{self.get_poller(kwargs.get('async_mode', False))}[{paging_type_annotation}]" - - def docstring_type(self, **kwargs: Any) -> str: - paging_docstring_type = PagingResponse.docstring_type(self, **kwargs) - return f"~{self.get_poller_path(kwargs.get('async_mode', False))}[{paging_docstring_type}]" - - def docstring_text(self, **kwargs) -> str: - base_description = "An instance of LROPoller that returns an iterator like instance of " - if not self.code_model.options["version_tolerant"]: - base_description += "either " - return base_description + Response.docstring_text(self) - - def imports_for_multiapi(self, **kwargs: Any) -> FileImport: - file_import = LROResponse.imports_for_multiapi(self, **kwargs) - file_import.merge(PagingResponse.imports_for_multiapi(self, **kwargs)) - return file_import - - def imports(self, **kwargs: Any) -> FileImport: - file_import = LROResponse.imports(self, **kwargs) - file_import.merge(PagingResponse.imports(self, **kwargs)) - return file_import - - -def get_response(yaml_data: Dict[str, Any], code_model: "CodeModel") -> Response: - if yaml_data["discriminator"] == "lropaging": - return LROPagingResponse.from_yaml(yaml_data, code_model) - if yaml_data["discriminator"] == "lro": - return LROResponse.from_yaml(yaml_data, code_model) - if yaml_data["discriminator"] == "paging": - return PagingResponse.from_yaml(yaml_data, code_model) - return Response.from_yaml(yaml_data, code_model) diff --git a/packages/autorest.python/generator/pygen/codegen/models/utils.py b/packages/autorest.python/generator/pygen/codegen/models/utils.py deleted file mode 100644 index 374d659135a..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/models/utils.py +++ /dev/null @@ -1,23 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import TypeVar, Dict - -T = TypeVar("T") -OrderedSet = Dict[T, None] - -NAME_LENGTH_LIMIT = 40 - - -def add_to_description(description: str, entry: str) -> str: - if description: - return f"{description} {entry}" - return entry - - -def add_to_pylint_disable(curr_str: str, entry: str) -> str: - if curr_str: - return f"{curr_str},{entry}" - return f" # pylint: disable={entry}" diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/__init__.py b/packages/autorest.python/generator/pygen/codegen/serializers/__init__.py deleted file mode 100644 index ee3d27ff8ad..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/__init__.py +++ /dev/null @@ -1,570 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import logging -from typing import List, Optional, Any, Union -from pathlib import Path -from jinja2 import PackageLoader, Environment, FileSystemLoader, StrictUndefined - -from ... import ReaderAndWriter -from ..models import ( - OperationGroup, - RequestBuilder, - OverloadedRequestBuilder, - CodeModel, - Client, -) -from .enum_serializer import EnumSerializer -from .general_serializer import GeneralSerializer -from .model_init_serializer import ModelInitSerializer -from .model_serializer import DpgModelSerializer, MsrestModelSerializer -from .operations_init_serializer import OperationsInitSerializer -from .operation_groups_serializer import OperationGroupsSerializer -from .metadata_serializer import MetadataSerializer -from .request_builders_serializer import RequestBuildersSerializer -from .patch_serializer import PatchSerializer -from .sample_serializer import SampleSerializer -from .test_serializer import TestSerializer, TestGeneralSerializer -from .types_serializer import TypesSerializer -from ...utils import to_snake_case -from .._utils import VALID_PACKAGE_MODE -from .utils import ( - extract_sample_name, - get_namespace_from_package_name, - get_namespace_config, - get_all_operation_groups_recursively, -) - -_LOGGER = logging.getLogger(__name__) - -__all__ = [ - "JinjaSerializer", -] - -_PACKAGE_FILES = [ - "CHANGELOG.md.jinja2", - "dev_requirements.txt.jinja2", - "LICENSE.jinja2", - "MANIFEST.in.jinja2", - "README.md.jinja2", - "setup.py.jinja2", -] - -_REGENERATE_FILES = {"setup.py", "MANIFEST.in"} - - -# extract sub folders. For example, source_file_path is like: -# "xxx/resource-manager/Microsoft.XX/stable/2023-04-01/examples/Compute/createOrUpdate/AKSCompute.json", -# and we want to extract the sub folders after "examples/", which is "compute/create_or_update" -def _sample_output_path(source_file_path: str) -> Path: - posix_path = Path(source_file_path).as_posix() - if "examples/" in posix_path: - after_examples = Path(posix_path.split("examples/", maxsplit=1)[-1]).parent - return Path("/".join([to_snake_case(i) for i in after_examples.parts])) - return Path("") - - -class JinjaSerializer(ReaderAndWriter): # pylint: disable=abstract-method - def __init__( - self, - code_model: CodeModel, - *, - output_folder: Union[str, Path], - **kwargs: Any, - ) -> None: - super().__init__(output_folder=output_folder, **kwargs) - self.code_model = code_model - - @property - def has_aio_folder(self) -> bool: - return not self.code_model.options["no_async"] and bool(self.code_model.has_operations) - - @property - def has_operations_folder(self) -> bool: - return self.code_model.options["show_operations"] and bool(self.code_model.has_operations) - - def _serialize_namespace_level(self, env: Environment, namespace_path: Path, clients: List[Client]) -> None: - # if there was a patch file before, we keep it - self._keep_patch_file(namespace_path / Path("_patch.py"), env) - if self.has_aio_folder: - self._keep_patch_file(namespace_path / Path("aio") / Path("_patch.py"), env) - - if self.has_operations_folder: - self._keep_patch_file( - namespace_path / Path(self.code_model.operations_folder_name) / Path("_patch.py"), - env, - ) - if self.has_aio_folder: - self._keep_patch_file( - namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path("_patch.py"), - env, - ) - self._serialize_and_write_top_level_folder(env=env, namespace_path=namespace_path, clients=clients) - - if any(c for c in self.code_model.clients if c.operation_groups): - if self.code_model.options["builders_visibility"] != "embedded": - self._serialize_and_write_rest_layer(env=env, namespace_path=namespace_path) - if self.has_aio_folder: - self._serialize_and_write_aio_top_level_folder( - env=env, - namespace_path=namespace_path, - clients=clients, - ) - - if self.has_operations_folder: - self._serialize_and_write_operations_folder(clients, env=env, namespace_path=namespace_path) - if self.code_model.options["multiapi"]: - self._serialize_and_write_metadata(env=env, namespace_path=namespace_path) - if self.code_model.options["package_mode"]: - self._serialize_and_write_package_files(namespace_path=namespace_path) - - if ( - self.code_model.options["show_operations"] - and self.code_model.has_operations - and self.code_model.options["generate_sample"] - ): - self._serialize_and_write_sample(env, namespace_path) - - if ( - self.code_model.options["show_operations"] - and self.code_model.has_operations - and self.code_model.options["generate_test"] - and not self.code_model.options["azure_arm"] - ): - self._serialize_and_write_test(env, namespace_path) - - def serialize(self) -> None: - env = Environment( - loader=PackageLoader("pygen.codegen", "templates"), - keep_trailing_newline=True, - line_statement_prefix="##", - line_comment_prefix="###", - trim_blocks=True, - lstrip_blocks=True, - ) - - namespace_path = ( - Path(".") if self.code_model.options["no_namespace_folders"] else Path(*self._name_space().split(".")) - ) - - p = namespace_path.parent - general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) - while p != Path("."): - # write pkgutil init file - self.write_file( - p / Path("__init__.py"), - general_serializer.serialize_pkgutil_init_file(), - ) - p = p.parent - - # serialize main module - self._serialize_namespace_level( - env, - namespace_path, - [c for c in self.code_model.clients if c.has_operations], - ) - # serialize sub modules - for ( - subnamespace, - clients, - ) in self.code_model.subnamespace_to_clients.items(): - subnamespace_path = namespace_path / Path(subnamespace) - self._serialize_namespace_level(env, subnamespace_path, [c for c in clients if c.has_operations]) - - if self.code_model.options["models_mode"] and (self.code_model.model_types or self.code_model.enums): - self._keep_patch_file(namespace_path / Path("models") / Path("_patch.py"), env) - - if self.code_model.options["models_mode"] and (self.code_model.model_types or self.code_model.enums): - self._serialize_and_write_models_folder(env=env, namespace_path=namespace_path) - if not self.code_model.options["models_mode"]: - # keep models file if users ended up just writing a models file - if self.read_file(namespace_path / Path("models.py")): - self.write_file( - namespace_path / Path("models.py"), - self.read_file(namespace_path / Path("models.py")), - ) - if self.code_model.named_unions: - self.write_file( - namespace_path / Path("_types.py"), - TypesSerializer(code_model=self.code_model, env=env).serialize(), - ) - - def _serialize_and_write_package_files(self, namespace_path: Path) -> None: - root_of_sdk = self._package_root_folder(namespace_path) - if self.code_model.options["package_mode"] in VALID_PACKAGE_MODE: - env = Environment( - loader=PackageLoader("pygen.codegen", "templates/packaging_templates"), - undefined=StrictUndefined, - ) - - package_files = _PACKAGE_FILES - elif Path(self.code_model.options["package_mode"]).exists(): - env = Environment( - loader=FileSystemLoader(str(Path(self.code_model.options["package_mode"]))), - keep_trailing_newline=True, - undefined=StrictUndefined, - ) - package_files = env.list_templates() - else: - return - serializer = GeneralSerializer(self.code_model, env, async_mode=False) - params = self.code_model.options["packaging_files_config"] or {} - for template_name in package_files: - if not self.code_model.is_azure_flavor and template_name == "dev_requirements.txt.jinja2": - continue - file = template_name.replace(".jinja2", "") - output_name = root_of_sdk / file - if not self.read_file(output_name) or file in _REGENERATE_FILES: - self.write_file( - output_name, - serializer.serialize_package_file(template_name, **params), - ) - - def _keep_patch_file(self, path_file: Path, env: Environment): - if self.read_file(path_file): - self.write_file(path_file, self.read_file(path_file)) - else: - self.write_file( - path_file, - PatchSerializer(env=env, code_model=self.code_model).serialize(), - ) - - def _serialize_and_write_models_folder(self, env: Environment, namespace_path: Path) -> None: - # Write the models folder - models_path = namespace_path / Path("models") - serializer = DpgModelSerializer if self.code_model.options["models_mode"] == "dpg" else MsrestModelSerializer - if self.code_model.model_types: - self.write_file( - models_path / Path(f"{self.code_model.models_filename}.py"), - serializer(code_model=self.code_model, env=env).serialize(), - ) - if self.code_model.enums: - self.write_file( - models_path / Path(f"{self.code_model.enums_filename}.py"), - EnumSerializer(code_model=self.code_model, env=env).serialize(), - ) - self.write_file( - models_path / Path("__init__.py"), - ModelInitSerializer(code_model=self.code_model, env=env).serialize(), - ) - - def _serialize_and_write_rest_layer(self, env: Environment, namespace_path: Path) -> None: - rest_path = namespace_path / Path(self.code_model.rest_layer_name) - group_names = {rb.group_name for c in self.code_model.clients for rb in c.request_builders} - - for group_name in group_names: - request_builders = [ - r for c in self.code_model.clients for r in c.request_builders if r.group_name == group_name - ] - self._serialize_and_write_single_rest_layer(env, rest_path, request_builders) - if not "" in group_names: - self.write_file( - rest_path / Path("__init__.py"), - self.code_model.options["license_header"], - ) - - def _serialize_and_write_single_rest_layer( - self, - env: Environment, - rest_path: Path, - request_builders: List[Union[RequestBuilder, OverloadedRequestBuilder]], - ) -> None: - group_name = request_builders[0].group_name - output_path = rest_path / Path(group_name) if group_name else rest_path - # write generic request builders file - self.write_file( - output_path / Path("_request_builders.py"), - RequestBuildersSerializer( - code_model=self.code_model, - env=env, - request_builders=request_builders, - ).serialize_request_builders(), - ) - - # write rest init file - self.write_file( - output_path / Path("__init__.py"), - RequestBuildersSerializer( - code_model=self.code_model, - env=env, - request_builders=request_builders, - ).serialize_init(), - ) - - def _serialize_and_write_operations_file( - self, - env: Environment, - clients: List[Client], - namespace_path: Path, - operation_group: Optional[OperationGroup] = None, - ) -> None: - filename = operation_group.filename if operation_group else "_operations" - # write first sync file - operation_group_serializer = OperationGroupsSerializer( - code_model=self.code_model, - clients=clients, - env=env, - async_mode=False, - operation_group=operation_group, - ) - self.write_file( - namespace_path / Path(self.code_model.operations_folder_name) / Path(f"{filename}.py"), - operation_group_serializer.serialize(), - ) - - if self.has_aio_folder: - # write async operation group and operation files - operation_group_async_serializer = OperationGroupsSerializer( - code_model=self.code_model, - clients=clients, - env=env, - async_mode=True, - operation_group=operation_group, - ) - self.write_file( - (namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path(f"{filename}.py")), - operation_group_async_serializer.serialize(), - ) - - def _serialize_and_write_operations_folder( - self, clients: List[Client], env: Environment, namespace_path: Path - ) -> None: - # write sync operations init file - operations_init_serializer = OperationsInitSerializer( - code_model=self.code_model, clients=clients, env=env, async_mode=False - ) - self.write_file( - namespace_path / Path(self.code_model.operations_folder_name) / Path("__init__.py"), - operations_init_serializer.serialize(), - ) - - # write async operations init file - if self.has_aio_folder: - operations_async_init_serializer = OperationsInitSerializer( - code_model=self.code_model, clients=clients, env=env, async_mode=True - ) - self.write_file( - namespace_path / Path("aio") / Path(self.code_model.operations_folder_name) / Path("__init__.py"), - operations_async_init_serializer.serialize(), - ) - - if self.code_model.options["combine_operation_files"]: - self._serialize_and_write_operations_file( - env=env, - namespace_path=namespace_path, - clients=clients, - ) - else: - for operation_group in get_all_operation_groups_recursively(self.code_model.clients): - self._serialize_and_write_operations_file( - env=env, - namespace_path=namespace_path, - operation_group=operation_group, - clients=clients, - ) - - def _serialize_and_write_version_file( - self, - namespace_path: Path, - general_serializer: GeneralSerializer, - ): - def _read_version_file(original_version_file_name: str) -> str: - return self.read_file(namespace_path / original_version_file_name) - - def _write_version_file(original_version_file_name: str) -> None: - self.write_file( - namespace_path / Path("_version.py"), - _read_version_file(original_version_file_name), - ) - - keep_version_file = self.code_model.options["keep_version_file"] - if keep_version_file and _read_version_file("_version.py"): - _write_version_file(original_version_file_name="_version.py") - elif keep_version_file and _read_version_file("version.py"): - _write_version_file(original_version_file_name="version.py") - elif self.code_model.options["package_version"]: - self.write_file( - namespace_path / Path("_version.py"), - general_serializer.serialize_version_file(), - ) - - def _serialize_client_and_config_files( - self, - namespace_path: Path, - general_serializer: GeneralSerializer, - async_mode: bool, - clients: List[Client], - ) -> None: - if self.code_model.has_operations: - namespace_path = namespace_path / Path("aio") if async_mode else namespace_path - self.write_file( - namespace_path / Path(f"{self.code_model.client_filename}.py"), - general_serializer.serialize_service_client_file(clients), - ) - self.write_file( - namespace_path / Path("_configuration.py"), - general_serializer.serialize_config_file(clients), - ) - - def _serialize_and_write_top_level_folder( - self, env: Environment, namespace_path: Path, clients: List[Client] - ) -> None: - general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=False) - - self.write_file( - namespace_path / Path("__init__.py"), - general_serializer.serialize_init_file(clients), - ) - - # Write the service client - self._serialize_client_and_config_files(namespace_path, general_serializer, async_mode=False, clients=clients) - if self.code_model.need_vendored_code(async_mode=False): - self.write_file( - namespace_path / Path("_vendor.py"), - general_serializer.serialize_vendor_file(clients), - ) - - self._serialize_and_write_version_file(namespace_path, general_serializer) - - # write the empty py.typed file - self.write_file(namespace_path / Path("py.typed"), "# Marker file for PEP 561.") - - if not self.code_model.options["client_side_validation"] and not self.code_model.options["multiapi"]: - self.write_file( - namespace_path / Path("_serialization.py"), - general_serializer.serialize_serialization_file(), - ) - if self.code_model.options["models_mode"] == "dpg": - self.write_file( - namespace_path / Path("_model_base.py"), - general_serializer.serialize_model_base_file(), - ) - - if any(og for client in self.code_model.clients for og in client.operation_groups if og.need_validation): - self.write_file( - namespace_path / Path("_validation.py"), - general_serializer.serialize_validation_file(), - ) - if self.code_model.options.get("emit_cross_language_definition_file"): - self.write_file( - Path("./apiview_mapping_python.json"), - general_serializer.serialize_cross_language_definition_file(), - ) - - # Write the setup file - if self.code_model.options["basic_setup_py"]: - self.write_file(Path("setup.py"), general_serializer.serialize_setup_file()) - - def _serialize_and_write_aio_top_level_folder( - self, env: Environment, namespace_path: Path, clients: List[Client] - ) -> None: - aio_general_serializer = GeneralSerializer(code_model=self.code_model, env=env, async_mode=True) - - aio_path = namespace_path / Path("aio") - - # Write the __init__ file - self.write_file( - aio_path / Path("__init__.py"), - aio_general_serializer.serialize_init_file(clients), - ) - - # Write the service client - self._serialize_client_and_config_files( - namespace_path, aio_general_serializer, async_mode=True, clients=clients - ) - if self.code_model.need_vendored_code(async_mode=True): - self.write_file( - aio_path / Path("_vendor.py"), - aio_general_serializer.serialize_vendor_file(clients), - ) - - def _serialize_and_write_metadata(self, env: Environment, namespace_path: Path) -> None: - metadata_serializer = MetadataSerializer(self.code_model, env) - self.write_file(namespace_path / Path("_metadata.json"), metadata_serializer.serialize()) - - @property - def _namespace_from_package_name(self) -> str: - return get_namespace_from_package_name(self.code_model.options["package_name"]) - - def _name_space(self) -> str: - if self.code_model.namespace.count(".") >= self._namespace_from_package_name.count("."): - return self.code_model.namespace - - return self._namespace_from_package_name - - # find root folder where "setup.py" is - def _package_root_folder(self, namespace_path: Path) -> Path: - return namespace_path / Path("../" * (self._name_space().count(".") + 1)) - - @property - def _additional_folder(self) -> Path: - namespace_config = get_namespace_config(self.code_model.namespace, self.code_model.options["multiapi"]) - num_of_namespace = namespace_config.count(".") + 1 - num_of_package_namespace = self._namespace_from_package_name.count(".") + 1 - if num_of_namespace > num_of_package_namespace: - return Path("/".join(namespace_config.split(".")[num_of_package_namespace:])) - return Path("") - - def _serialize_and_write_sample(self, env: Environment, namespace_path: Path): - out_path = self._package_root_folder(namespace_path) / Path("generated_samples") - for client in self.code_model.clients: - for op_group in client.operation_groups: - for operation in op_group.operations: - if ( - self.code_model.options["multiapi"] - and operation.api_versions[0] != self.code_model.options["default_api_version"] - ): - continue - samples = operation.yaml_data["samples"] - if not samples or operation.name.startswith("_"): - continue - for value in samples.values(): - file = value.get("x-ms-original-file", "sample.json") - file_name = to_snake_case(extract_sample_name(file)) + ".py" - try: - self.write_file( - out_path / self._additional_folder / _sample_output_path(file) / file_name, - SampleSerializer( - code_model=self.code_model, - env=env, - operation_group=op_group, - operation=operation, - sample=value, - file_name=file_name, - ).serialize(), - ) - except Exception as e: # pylint: disable=broad-except - # sample generation shall not block code generation, so just log error - log_error = f"error happens in sample {file}: {e}" - _LOGGER.error(log_error) - - def _serialize_and_write_test(self, env: Environment, namespace_path: Path): - self.code_model.for_test = True - out_path = self._package_root_folder(namespace_path) / Path("generated_tests") - general_serializer = TestGeneralSerializer(code_model=self.code_model, env=env) - self.write_file(out_path / "conftest.py", general_serializer.serialize_conftest()) - for is_async in (True, False): - async_suffix = "_async" if is_async else "" - general_serializer.is_async = is_async - self.write_file( - out_path / f"testpreparer{async_suffix}.py", - general_serializer.serialize_testpreparer(), - ) - - for client in self.code_model.clients: - for og in client.operation_groups: - test_serializer = TestSerializer(self.code_model, env, client=client, operation_group=og) - for is_async in (True, False): - try: - test_serializer.is_async = is_async - self.write_file( - out_path / f"{to_snake_case(test_serializer.test_class_name)}.py", - test_serializer.serialize_test(), - ) - except Exception as e: # pylint: disable=broad-except - # test generation shall not block code generation, so just log error - log_error = f"error happens in test generation for operation group {og.class_name}: {e}" - _LOGGER.error(log_error) - self.code_model.for_test = False diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/base_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/base_serializer.py deleted file mode 100644 index 0ac623166a1..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/base_serializer.py +++ /dev/null @@ -1,21 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from jinja2 import Environment -from ..models import ( - FileImport, - CodeModel, -) - - -class BaseSerializer: - """Base serializer for SDK root level files""" - - def __init__(self, code_model: CodeModel, env: Environment): - self.code_model = code_model - self.env = env - - def init_file_import(self) -> FileImport: - return FileImport(self.code_model) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/builder_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/builder_serializer.py deleted file mode 100644 index 007661f9116..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/builder_serializer.py +++ /dev/null @@ -1,1453 +0,0 @@ -# pylint: disable=too-many-lines,multiple-statements -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from abc import abstractmethod -from collections import defaultdict -from typing import Generic, List, Type, TypeVar, Dict, Union, Optional, cast - -from ..models import ( - Operation, - PagingOperation, - CodeModel, - LROOperation, - LROPagingOperation, - ModelType, - DictionaryType, - ListType, - RequestBuilder, - ParameterLocation, - Response, - BinaryType, - BodyParameter, - ParameterMethodLocation, - RequestBuilderBodyParameter, - OverloadedRequestBuilder, - Property, - RequestBuilderType, - CombinedType, - JSONModelType, - DPGModelType, - ParameterListType, - ByteArraySchema, -) -from .parameter_serializer import ParameterSerializer, PopKwargType -from ..models.parameter_list import ParameterType -from . import utils -from ...utils import JSON_REGEXP - -T = TypeVar("T") -OrderedSet = Dict[T, None] - -BuilderType = TypeVar( - "BuilderType", - bound=Union[ - RequestBuilder, - Operation, - PagingOperation, - LROOperation, - LROPagingOperation, - OverloadedRequestBuilder, - ], -) -OperationType = TypeVar( - "OperationType", - bound=Union[Operation, PagingOperation, LROOperation, LROPagingOperation], -) - - -def _json_serializable(content_type: str) -> bool: - return bool(JSON_REGEXP.match(content_type.split(";")[0].strip().lower())) - - -def _need_type_ignore(builder: OperationType) -> bool: - for excep in builder.non_default_errors: - for status_code in excep.status_codes: - if status_code in (401, 404, 409, 304): - return True - return False - - -def _xml_config(send_xml: bool, content_types: List[str]) -> str: - if not (send_xml and "xml" in str(content_types)): - return "" - if len(content_types) == 1: - return ", is_xml=True" - return ", is_xml='xml' in str(content_type)" - - -def _escape_str(input_str: str) -> str: - replace = input_str.replace("'", "\\'") - return f'"{replace}"' - - -def _get_polymorphic_subtype_template(polymorphic_subtype: ModelType) -> List[str]: - retval: List[str] = [] - retval.append("") - retval.append(f'# JSON input template for discriminator value "{polymorphic_subtype.discriminator_value}":') - subtype_template = utils.json_dumps_template( - polymorphic_subtype.get_json_template_representation(), - ) - - def _get_polymorphic_parent( - polymorphic_subtype: Optional[ModelType], - ) -> Optional[ModelType]: - if not polymorphic_subtype: - return None - try: - return next(p for p in polymorphic_subtype.parents if p.discriminated_subtypes) - except StopIteration: - return None - - polymorphic_parent = _get_polymorphic_parent(polymorphic_subtype) - while _get_polymorphic_parent(polymorphic_parent): - polymorphic_parent = _get_polymorphic_parent(polymorphic_parent) - retval.extend(f"{cast(ModelType, polymorphic_parent).snake_case_name} = {subtype_template}".splitlines()) - return retval - - -def _serialize_grouped_body(builder: BuilderType) -> List[str]: - retval: List[str] = [] - for grouped_parameter in builder.parameters.grouped: - retval.append(f"{grouped_parameter.client_name} = None") - groupers = [p for p in builder.parameters if p.grouper] - for grouper in groupers: - retval.append(f"if {grouper.client_name} is not None:") - retval.extend( - [ - f" {parameter} = {grouper.client_name}.{property}" - for property, parameter in grouper.property_to_parameter_name.items() - ] - ) - return retval - - -def _serialize_flattened_body(body_parameter: BodyParameter) -> List[str]: - retval: List[str] = [] - if not body_parameter.property_to_parameter_name: - raise ValueError("This method can't be called if the operation doesn't need parameter flattening") - - parameter_string = ", ".join( - f"{property_name}={parameter_name}" - for property_name, parameter_name in body_parameter.property_to_parameter_name.items() - ) - model_type = cast(ModelType, body_parameter.type) - retval.append(f"{body_parameter.client_name} = _models.{model_type.name}({parameter_string})") - return retval - - -def _serialize_json_model_body(body_parameter: BodyParameter, parameters: List[ParameterType]) -> List[str]: - retval: List[str] = [] - if not body_parameter.property_to_parameter_name: - raise ValueError("This method can't be called if the operation doesn't need parameter flattening") - - retval.append(f"if {body_parameter.client_name} is _Unset:") - for p in parameters: - if p.client_default_value is None and not p.optional and p.default_to_unset_sentinel: - retval.append(f" if {p.client_name} is _Unset:") - retval.append(f" raise TypeError('missing required argument: {p.client_name}')") - parameter_string = ", \n".join( - f'"{property_name}": {parameter_name}' - for property_name, parameter_name in body_parameter.property_to_parameter_name.items() - ) - model_type = cast(ModelType, body_parameter.type) - if isinstance(model_type, CombinedType) and model_type.target_model_subtype((JSONModelType,)): - model_type = model_type.target_model_subtype((JSONModelType,)) - retval.append(f" {body_parameter.client_name} = {{{parameter_string}}}") - retval.append(f" {body_parameter.client_name} = {{") - retval.append(f" k: v for k, v in {body_parameter.client_name}.items() if v is not None") - retval.append(" }") - return retval - - -def _serialize_multipart_body(builder: BuilderType) -> List[str]: - retval: List[str] = [] - body_param = builder.parameters.body_parameter - # we have to construct our form data before passing to the request as well - retval.append("# Construct form data") - retval.append(f"_{body_param.client_name} = {{") - for param in body_param.entries: - retval.append(f' "{param.wire_name}": {param.client_name},') - retval.append("}") - return retval - - -def _get_json_response_template_to_status_codes( - builder: OperationType, -) -> Dict[str, List[str]]: - retval = defaultdict(list) - for response in builder.responses: - json_template = response.get_json_template_representation() - if not json_template: - continue - status_codes = [str(status_code) for status_code in response.status_codes] - response_json = utils.json_dumps_template(json_template) - retval[response_json].extend(status_codes) - return retval - - -def _api_version_validation(builder: OperationType) -> str: - retval: List[str] = [] - if builder.added_on: - retval.append(f' method_added_on="{builder.added_on}",') - params_added_on = defaultdict(list) - for parameter in builder.parameters: - if parameter.added_on: - params_added_on[parameter.added_on].append(parameter.client_name) - if params_added_on: - retval.append(f" params_added_on={dict(params_added_on)},") - if retval: - retval_str = "\n".join(retval) - return f"@api_version_validation(\n{retval_str}\n){builder.pylint_disable}" - return "" - - -def is_json_model_type(parameters: ParameterListType) -> bool: - return ( - parameters.has_body - and parameters.body_parameter.has_json_model_type - and any(p.in_flattened_body for p in parameters.parameters) - ) - - -class _BuilderBaseSerializer(Generic[BuilderType]): # pylint: disable=abstract-method - def __init__(self, code_model: CodeModel, async_mode: bool) -> None: - self.code_model = code_model - self.async_mode = async_mode - self.parameter_serializer = ParameterSerializer() - - @property - @abstractmethod - def _need_self_param(self) -> bool: ... - - @property - @abstractmethod - def _function_def(self) -> str: - """The def keyword for the builder we're serializing, i.e. 'def' or 'async def'""" - - @property - @abstractmethod - def _call_method(self) -> str: - """How to call network calls. Await if we have to await network calls""" - - @property - @abstractmethod - def serializer_name(self) -> str: - """Name of serializer""" - - @abstractmethod - def response_docstring(self, builder: BuilderType) -> List[str]: - """Response portion of the docstring""" - - def decorators(self, builder: BuilderType) -> List[str]: - """Decorators for the method""" - retval: List[str] = [] - if builder.is_overload: - return ["@overload"] - if self.code_model.options["tracing"] and builder.want_tracing: - retval.append(f"@distributed_trace{'_async' if self.async_mode else ''}") - return retval - - def _method_signature(self, builder: BuilderType) -> str: - return self.parameter_serializer.serialize_method( - function_def=self._function_def, - method_name=builder.name, - need_self_param=self._need_self_param, - method_param_signatures=builder.method_signature(self.async_mode), - pylint_disable=builder.pylint_disable, - ) - - def method_signature_and_response_type_annotation( - self, builder: BuilderType, *, want_decorators: Optional[bool] = True - ) -> str: - response_type_annotation = builder.response_type_annotation(async_mode=self.async_mode) - method_signature = self._method_signature(builder) - decorators = self.decorators(builder) - decorators_str = "" - if decorators and want_decorators: - decorators_str = "\n".join(decorators) + "\n" - return decorators_str + utils.method_signature_and_response_type_annotation_template( - method_signature=method_signature, - response_type_annotation=response_type_annotation, - ) - - def description_and_summary(self, builder: BuilderType) -> List[str]: - description_list: List[str] = [] - description_list.append(f"{builder.summary.strip() if builder.summary else builder.description.strip()}") - if builder.summary and builder.description: - description_list.append("") - description_list.append(builder.description.strip()) - description_list.append("") - return description_list - - @staticmethod - def line_too_long(docs: List[str]) -> bool: - return any(len(line) > 120 for line in docs) - - def example_template(self, builder: BuilderType) -> List[str]: - template = [] - if builder.abstract: - return [] - if self._json_input_example_template(builder): - template.append("") - template += self._json_input_example_template(builder) - return template - - def param_description(self, builder: BuilderType) -> List[str]: - description_list: List[str] = [] - for param in builder.parameters.method: - if ( - not param.in_docstring - or param.hide_in_operation_signature - or param.method_location == ParameterMethodLocation.KWARG - ): - continue - description_list.extend( - f":{param.description_keyword} {param.client_name}: {param.description}".replace("\n", "\n ").split( - "\n" - ) - ) - docstring_type = param.docstring_type( - async_mode=self.async_mode, - ) - description_list.append(f":{param.docstring_type_keyword} {param.client_name}: {docstring_type}") - return description_list - - def param_description_and_response_docstring(self, builder: BuilderType) -> List[str]: - if builder.abstract: - return [] - return self.param_description(builder) + self.response_docstring(builder) - - @property - @abstractmethod - def _json_response_template_name(self) -> str: ... - - def _json_input_example_template(self, builder: BuilderType) -> List[str]: - template: List[str] = [] - if not builder.parameters.has_body or builder.parameters.body_parameter.flattened: - # No input template if now body parameter - return template - - body_param = builder.parameters.body_parameter - if not isinstance(body_param.type, (ListType, DictionaryType, ModelType, CombinedType)): - return template - - if ( - isinstance(body_param.type, (ListType, DictionaryType)) - and self.code_model.options["models_mode"] == "msrest" - ): - return template - - if isinstance(body_param.type, ModelType) and body_param.type.base == "msrest": - return template - - json_type = body_param.type - if isinstance(body_param.type, CombinedType): - target_model_type = body_param.type.target_model_subtype((JSONModelType, DPGModelType)) - if target_model_type is None: - return template - json_type = target_model_type - - polymorphic_subtypes: List[ModelType] = [] - json_type.get_polymorphic_subtypes(polymorphic_subtypes) - if polymorphic_subtypes: - # we just assume one kind of polymorphic body for input - discriminator_name = cast(Property, polymorphic_subtypes[0].discriminator).wire_name - template.append( - "# The input is polymorphic. The following are possible polymorphic " - f'inputs based off discriminator "{discriminator_name}":' - ) - for idx in range( - min( - self.code_model.options["polymorphic_examples"], - len(polymorphic_subtypes), - ) - ): - template.extend(_get_polymorphic_subtype_template(polymorphic_subtypes[idx])) - template.append("") - template.append("# JSON input template you can fill out and use as your body input.") - json_template = utils.json_dumps_template( - json_type.get_json_template_representation(), - ) - template.extend(f"{builder.parameters.body_parameter.client_name} = {json_template}".splitlines()) - return template - - def serialize_path(self, builder: BuilderType) -> List[str]: - return self.parameter_serializer.serialize_path(builder.parameters.path, self.serializer_name) - - @property - def pipeline_name(self) -> str: - return f"{'_' if self.code_model.is_azure_flavor else ''}pipeline" - - -############################## REQUEST BUILDERS ############################## - - -class RequestBuilderSerializer(_BuilderBaseSerializer[RequestBuilderType]): # pylint: disable=abstract-method - def description_and_summary(self, builder: RequestBuilderType) -> List[str]: - retval = super().description_and_summary(builder) - retval += [ - "See https://aka.ms/azsdk/dpcodegen/python/send_request for how to incorporate this " - "request builder into your code flow.", - "", - ] - return retval - - @property - def _call_method(self) -> str: - return "" - - @property - def serializer_name(self) -> str: - return "_SERIALIZER" - - @property - def _json_response_template_name(self) -> str: - return "response.json()" - - @staticmethod - def declare_non_inputtable_constants(builder: RequestBuilderType) -> List[str]: - def _get_value(param): - if param.location in [ParameterLocation.HEADER, ParameterLocation.QUERY]: - kwarg_dict = "headers" if param.location == ParameterLocation.HEADER else "params" - return f"_{kwarg_dict}.pop('{param.wire_name}', {param.get_declaration()})" - return f"{param.get_declaration()}" - - return [f"{p.client_name} = {_get_value(p)}" for p in builder.parameters.constant if not p.in_method_signature] - - @property - def _function_def(self) -> str: - return "def" - - @property - def _need_self_param(self) -> bool: - return False - - def response_docstring(self, builder: RequestBuilderType) -> List[str]: - request_full_path = f"{self.code_model.core_library}.rest.HttpRequest" - response_str = ( - f":return: Returns an :class:`~{request_full_path}` that you will pass to the client's " - + "`send_request` method. See https://aka.ms/azsdk/dpcodegen/python/send_request for how to " - + "incorporate this response into your code flow." - ) - rtype_str = f":rtype: ~{request_full_path}" - return [response_str, rtype_str] - - def pop_kwargs_from_signature(self, builder: RequestBuilderType) -> List[str]: - return self.parameter_serializer.pop_kwargs_from_signature( - builder.parameters.kwargs_to_pop, - check_kwarg_dict=True, - pop_headers_kwarg=(PopKwargType.CASE_INSENSITIVE if bool(builder.parameters.headers) else PopKwargType.NO), - pop_params_kwarg=(PopKwargType.CASE_INSENSITIVE if bool(builder.parameters.query) else PopKwargType.NO), - ) - - @staticmethod - def create_http_request(builder: RequestBuilderType) -> List[str]: - retval = ["return HttpRequest("] - retval.append(f' method="{builder.method}",') - retval.append(" url=_url,") - if builder.parameters.query: - retval.append(" params=_params,") - if builder.parameters.headers: - retval.append(" headers=_headers,") - if builder.parameters.has_body and builder.parameters.body_parameter.in_method_signature: - body_param = builder.parameters.body_parameter - if body_param.constant or body_param.method_location != ParameterMethodLocation.KWARG: - # we only need to pass it through if it's not a kwarg or it's a popped kwarg - retval.append( - f" {builder.parameters.body_parameter.client_name}=" - f"{builder.parameters.body_parameter.client_name}," - ) - retval.append(" **kwargs") - retval.append(")") - return retval - - def serialize_headers(self, builder: RequestBuilderType) -> List[str]: - headers = [ - h - for h in builder.parameters.headers - if not builder.has_form_data_body or h.wire_name.lower() != "content-type" - ] - retval = ["# Construct headers"] if headers else [] - for header in headers: - retval.extend( - self.parameter_serializer.serialize_query_header( - header, - "headers", - self.serializer_name, - self.code_model.is_legacy, - ) - ) - return retval - - def serialize_query(self, builder: RequestBuilderType) -> List[str]: - retval = ["# Construct parameters"] - for parameter in builder.parameters.query: - retval.extend( - self.parameter_serializer.serialize_query_header( - parameter, - "params", - self.serializer_name, - self.code_model.is_legacy, - ) - ) - return retval - - def construct_url(self, builder: RequestBuilderType) -> str: - if any(o for o in ["low_level_client", "version_tolerant"] if self.code_model.options.get(o)): - url_value = _escape_str(builder.url) - else: - url_value = f'kwargs.pop("template_url", {_escape_str(builder.url)})' - return f"_url = {url_value}{' # pylint: disable=line-too-long' if len(url_value) > 114 else ''}" - - -############################## NORMAL OPERATIONS ############################## - - -class _OperationSerializer(_BuilderBaseSerializer[OperationType]): # pylint: disable=abstract-method - def description_and_summary(self, builder: OperationType) -> List[str]: - retval = super().description_and_summary(builder) - if builder.deprecated: - retval.append(".. warning::") - retval.append(" This method is deprecated") - retval.append("") - if builder.external_docs and builder.external_docs.get("url"): - retval.append(".. seealso::") - retval.append(f" - {builder.external_docs['url']}") - retval.append("") - return retval - - @property - def _json_response_template_name(self) -> str: - return "response" - - def example_template(self, builder: OperationType) -> List[str]: - retval = super().example_template(builder) - if self.code_model.options["models_mode"] == "msrest": - return retval - for response in builder.responses: - polymorphic_subtypes: List[ModelType] = [] - if not response.type: - continue - response.get_polymorphic_subtypes(polymorphic_subtypes) - if polymorphic_subtypes: - # we just assume one kind of polymorphic body for input - discriminator_name = cast(Property, polymorphic_subtypes[0].discriminator).wire_name - retval.append("") - retval.append( - "# The response is polymorphic. The following are possible polymorphic " - f'responses based off discriminator "{discriminator_name}":' - ) - for idx in range( - min( - self.code_model.options["polymorphic_examples"], - len(polymorphic_subtypes), - ) - ): - retval.extend(_get_polymorphic_subtype_template(polymorphic_subtypes[idx])) - - if _get_json_response_template_to_status_codes(builder): - retval.append("") - for ( - response_body, - status_codes, - ) in _get_json_response_template_to_status_codes(builder).items(): - retval.append("# response body for status code(s): {}".format(", ".join(status_codes))) - retval.extend(f"{self._json_response_template_name} == {response_body}".splitlines()) - return retval - - def make_pipeline_call(self, builder: OperationType) -> List[str]: - type_ignore = self.async_mode and builder.group_name == "" # is in a mixin - stream_value = ( - f'kwargs.pop("stream", {builder.has_stream_response})' - if builder.expose_stream_keyword and builder.has_response_body - else builder.has_stream_response - ) - return [ - f"_stream = {stream_value}", - f"pipeline_response: PipelineResponse = {self._call_method}self._client.{self.pipeline_name}.run( " - + f"{'# type: ignore' if type_ignore else ''} # pylint: disable=protected-access", - " _request,", - " stream=_stream,", - " **kwargs", - ")", - ] - - @property - def _function_def(self) -> str: - return "async def" if self.async_mode else "def" - - @property - def _need_self_param(self) -> bool: - return True - - @property - def serializer_name(self) -> str: - return "self._serialize" - - def decorators(self, builder: OperationType) -> List[str]: - """Decorators for the method""" - retval = super().decorators(builder) - if _api_version_validation(builder): - retval.append(_api_version_validation(builder)) - return retval - - def pop_kwargs_from_signature(self, builder: OperationType) -> List[str]: - kwargs_to_pop = builder.parameters.kwargs_to_pop - kwargs = self.parameter_serializer.pop_kwargs_from_signature( - kwargs_to_pop, - check_kwarg_dict=True, - pop_headers_kwarg=( - PopKwargType.CASE_INSENSITIVE - if builder.has_kwargs_to_pop_with_default(kwargs_to_pop, ParameterLocation.HEADER) # type: ignore - else PopKwargType.SIMPLE - ), - pop_params_kwarg=( - PopKwargType.CASE_INSENSITIVE - if builder.has_kwargs_to_pop_with_default(kwargs_to_pop, ParameterLocation.QUERY) # type: ignore - else PopKwargType.SIMPLE - ), - check_client_input=not self.code_model.options["multiapi"], - operation_name=f"('{builder.name}')" if builder.group_name == "" else "", - ) - for p in builder.parameters.parameters: - if p.hide_in_operation_signature: - kwargs.append(f'{p.client_name} = kwargs.pop("{p.client_name}", None)') - cls_annotation = builder.cls_type_annotation(async_mode=self.async_mode) - pylint_disable = "" - if any(x.startswith("_") for x in cls_annotation.split(".")): - pylint_disable = " # pylint: disable=protected-access" - kwargs.append(f"cls: {cls_annotation} = kwargs.pop({pylint_disable}\n 'cls', None\n)") - return kwargs - - def response_docstring(self, builder: OperationType) -> List[str]: - response_str = f":return: {builder.response_docstring_text(async_mode=self.async_mode)}" - rtype_str = f":rtype: {builder.response_docstring_type(async_mode=self.async_mode)}" - return [ - response_str, - rtype_str, - f":raises ~{self.code_model.core_library}.exceptions.HttpResponseError:", - ] - - def _serialize_body_parameter(self, builder: OperationType) -> List[str]: - """We need to serialize params if they're not meant to be streamed in. - - This function serializes the body params that need to be serialized. - """ - retval: List[str] = [] - body_param = builder.parameters.body_parameter - if body_param.is_form_data: - model_type = cast( - ModelType, - ( - body_param.type.target_model_subtype((JSONModelType, DPGModelType)) - if isinstance(body_param.type, CombinedType) - else body_param.type - ), - ) - file_fields = [p.wire_name for p in model_type.properties if p.is_multipart_file_input] - data_fields = [p.wire_name for p in model_type.properties if not p.is_multipart_file_input] - retval.extend( - [ - "_body = (", - f" {body_param.client_name}.as_dict()", - f" if isinstance({body_param.client_name}, _model_base.Model) else", - f" {body_param.client_name}", - ")", - f"_file_fields: List[str] = {file_fields}", - f"_data_fields: List[str] = {data_fields}", - "_files, _data = prepare_multipart_form_data(_body, _file_fields, _data_fields)", - ] - ) - return retval - - body_kwarg_name = builder.request_builder.parameters.body_parameter.client_name - send_xml = builder.parameters.body_parameter.type.is_xml - xml_serialization_ctxt = body_param.type.xml_serialization_ctxt if send_xml else None - ser_ctxt_name = "serialization_ctxt" - if xml_serialization_ctxt and self.code_model.options["models_mode"]: - retval.append(f'{ser_ctxt_name} = {{"xml": {{{xml_serialization_ctxt}}}}}') - if self.code_model.options["models_mode"] == "msrest": - is_xml_cmd = _xml_config(send_xml, builder.parameters.body_parameter.content_types) - serialization_ctxt_cmd = f", {ser_ctxt_name}={ser_ctxt_name}" if xml_serialization_ctxt else "" - create_body_call = ( - f"_{body_kwarg_name} = self._serialize.body({body_param.client_name}, " - f"'{body_param.type.serialization_type}'{is_xml_cmd}{serialization_ctxt_cmd})" - ) - elif self.code_model.options["models_mode"] == "dpg": - if _json_serializable(body_param.default_content_type): - if hasattr(body_param.type, "encode") and body_param.type.encode: # type: ignore - create_body_call = ( - f"_{body_kwarg_name} = json.dumps({body_param.client_name}, " - "cls=SdkJSONEncoder, exclude_readonly=True, " - f"format='{body_param.type.encode}') # type: ignore" # type: ignore - ) - else: - create_body_call = ( - f"_{body_kwarg_name} = json.dumps({body_param.client_name}, " - "cls=SdkJSONEncoder, exclude_readonly=True) # type: ignore" - ) - else: - create_body_call = f"_{body_kwarg_name} = {body_param.client_name}" - else: - create_body_call = f"_{body_kwarg_name} = {body_param.client_name}" - if body_param.optional: - retval.append(f"if {body_param.client_name} is not None:") - retval.append(" " + create_body_call) - retval.append("else:") - retval.append(f" _{body_kwarg_name} = None") - else: - retval.append(create_body_call) - return retval - - def _create_body_parameter( - self, - builder: OperationType, - ) -> List[str]: - """Create the body parameter before we pass it as either json or content to the request builder""" - retval = [] - body_param = builder.parameters.body_parameter - if body_param.entries: - return _serialize_multipart_body(builder) - body_kwarg_name = builder.request_builder.parameters.body_parameter.client_name - body_param_type = body_param.type - if isinstance(body_param_type, BinaryType) or ( - isinstance(body_param.type, ByteArraySchema) and body_param.default_content_type != "application/json" - ): - retval.append(f"_{body_kwarg_name} = {body_param.client_name}") - if ( - not body_param.default_content_type - and not next(p for p in builder.parameters if p.wire_name.lower() == "content-type").optional - ): - content_types = "'" + "', '".join(body_param.content_types) + "'" - retval.extend( - [ - "if not content_type:", - f' raise TypeError("Missing required keyword-only argument: content_type. ' - f'Known values are:" + "{content_types}")', - ] - ) - else: - retval.extend(self._serialize_body_parameter(builder)) - return retval - - def _initialize_overloads(self, builder: OperationType, is_paging: bool = False) -> List[str]: - retval: List[str] = [] - # For paging, we put body parameter in local place outside `prepare_request` - if is_paging: - return retval - same_content_type = len(set(o.parameters.body_parameter.default_content_type for o in builder.overloads)) == 1 - if same_content_type: - default_content_type = builder.overloads[0].parameters.body_parameter.default_content_type - retval.append(f'content_type = content_type or "{default_content_type}"') - client_names = [ - overload.request_builder.parameters.body_parameter.client_name for overload in builder.overloads - ] - for v in sorted(set(client_names), key=client_names.index): - retval.append(f"_{v} = None") - try: - # if there is a binary overload, we do a binary check first. - binary_overload = cast( - OperationType, - next((o for o in builder.overloads if isinstance(o.parameters.body_parameter.type, BinaryType))), - ) - binary_body_param = binary_overload.parameters.body_parameter - retval.append(f"if {binary_body_param.type.instance_check_template.format(binary_body_param.client_name)}:") - if binary_body_param.default_content_type and not same_content_type: - retval.append(f' content_type = content_type or "{binary_body_param.default_content_type}"') - retval.extend(f" {l}" for l in self._create_body_parameter(binary_overload)) - retval.append("else:") - other_overload = cast( - OperationType, - next((o for o in builder.overloads if not isinstance(o.parameters.body_parameter.type, BinaryType))), - ) - retval.extend(f" {l}" for l in self._create_body_parameter(other_overload)) - if other_overload.parameters.body_parameter.default_content_type and not same_content_type: - retval.append( - " content_type = content_type or " - f'"{other_overload.parameters.body_parameter.default_content_type}"' - ) - except StopIteration: - for idx, overload in enumerate(builder.overloads): - if_statement = "if" if idx == 0 else "elif" - body_param = overload.parameters.body_parameter - retval.append( - f"{if_statement} {body_param.type.instance_check_template.format(body_param.client_name)}:" - ) - if body_param.default_content_type and not same_content_type: - retval.append(f' content_type = content_type or "{body_param.default_content_type}"') - retval.extend(f" {l}" for l in self._create_body_parameter(cast(OperationType, overload))) - return retval - - def _create_request_builder_call( - self, - builder: OperationType, - request_builder: RequestBuilderType, - is_next_request: bool = False, - ) -> List[str]: - retval: List[str] = [] - if self.code_model.options["builders_visibility"] == "embedded": - request_path_name = request_builder.name - else: - group_name = request_builder.group_name - request_path_name = "rest{}.{}".format( - ("_" + group_name) if group_name else "", - request_builder.name, - ) - retval.append(f"_request = {request_path_name}(") - for parameter in request_builder.parameters.method: - if parameter.location == ParameterLocation.BODY: - # going to pass in body later based off of overloads - continue - if ( - is_next_request - and builder.operation_type == "paging" - and not bool(builder.next_request_builder) # type: ignore - and parameter.location == ParameterLocation.QUERY - ): - # if we don't want to reformat query parameters for next link calls - # in paging operations with a single swagger operation defintion, - # we skip passing query params when building the next request - continue - type_ignore = ( - parameter.grouped_by - and parameter.client_default_value is not None - and next(p for p in builder.parameters if p.grouper and p.client_name == parameter.grouped_by).optional - ) - retval.append( - f" {parameter.client_name}={parameter.name_in_high_level_operation}," - f"{' # type: ignore' if type_ignore else ''}" - ) - if builder.parameters.has_body and builder.parameters.body_parameter.entries: - # this is for legacy - client_name = builder.parameters.body_parameter.client_name - retval.append(f" {client_name}=_{client_name},") - elif request_builder.has_form_data_body: - retval.append(" files=_files,") - retval.append(" data=_data,") - elif request_builder.overloads: - seen_body_params = set() - for overload in request_builder.overloads: - body_param = cast(RequestBuilderBodyParameter, overload.parameters.body_parameter) - if body_param.client_name in seen_body_params: - continue - seen_body_params.add(body_param.client_name) - - retval.append(f" {body_param.client_name}={body_param.name_in_high_level_operation},") - elif request_builder.parameters.has_body: - body_param = cast(RequestBuilderBodyParameter, request_builder.parameters.body_parameter) - retval.append(f" {body_param.client_name}={body_param.name_in_high_level_operation},") - retval.append(" headers=_headers,") - retval.append(" params=_params,") - retval.append(")") - return retval - - def _postprocess_http_request(self, builder: OperationType, template_url: Optional[str] = None) -> List[str]: - retval: List[str] = [] - if builder.parameters.path: - retval.extend(self.serialize_path(builder)) - url_to_format = "_request.url" - if self.code_model.options["version_tolerant"] and template_url: - url_to_format = template_url - retval.append( - "_request.url = self._client.format_url({}{})".format( - url_to_format, - ", **path_format_arguments" if builder.parameters.path else "", - ) - ) - return retval - - def _call_request_builder_helper( # pylint: disable=too-many-statements - self, - builder: OperationType, - request_builder: RequestBuilderType, - template_url: Optional[str] = None, - is_next_request: bool = False, - is_paging: bool = False, - ) -> List[str]: - retval = [] - if builder.parameters.grouped: - # request builders don't allow grouped parameters, so we group them before making the call - retval.extend(_serialize_grouped_body(builder)) - if builder.parameters.has_body and builder.parameters.body_parameter.flattened: - # unflatten before passing to request builder as well - retval.extend(_serialize_flattened_body(builder.parameters.body_parameter)) - if is_json_model_type(builder.parameters): - retval.extend(_serialize_json_model_body(builder.parameters.body_parameter, builder.parameters.parameters)) - if builder.has_form_data_body: - retval.extend(self._create_body_parameter(builder)) - elif builder.overloads: - # we are only dealing with two overloads. If there are three, we generate an abstract operation - retval.extend(self._initialize_overloads(builder, is_paging=is_paging)) - elif builder.parameters.has_body: - # non-overloaded body - retval.extend(self._create_body_parameter(builder)) - retval.append("") - retval.extend(self._create_request_builder_call(builder, request_builder, is_next_request)) - retval.extend(self._postprocess_http_request(builder, template_url)) - return retval - - def call_request_builder(self, builder: OperationType, is_paging: bool = False) -> List[str]: - return self._call_request_builder_helper(builder, builder.request_builder, is_paging=is_paging) - - def response_headers_and_deserialization( - self, - builder: OperationType, - response: Response, - ) -> List[str]: - retval: List[str] = [ - ( - f"response_headers['{response_header.wire_name}']=self._deserialize(" - f"'{response_header.serialization_type}', response.headers.get('{response_header.wire_name}'))" - ) - for response_header in response.headers - ] - if response.headers: - retval.append("") - deserialize_code: List[str] = [] - stream_logic = True - if builder.has_stream_response: - if isinstance(response.type, ByteArraySchema): - deserialized = f"{'await ' if self.async_mode else ''}response.read()" - else: - stream_logic = False - if self.code_model.options["version_tolerant"]: - deserialized = "response.iter_bytes()" - else: - deserialized = f"response.stream_download(self._client.{self.pipeline_name})" - deserialize_code.append(f"deserialized = {deserialized}") - elif response.type: - pylint_disable = "" - if isinstance(response.type, ModelType) and response.type.internal: - pylint_disable = " # pylint: disable=protected-access" - if self.code_model.options["models_mode"] == "msrest": - deserialize_code.append("deserialized = self._deserialize(") - deserialize_code.append(f" '{response.serialization_type}',{pylint_disable}") - deserialize_code.append(" pipeline_response") - deserialize_code.append(")") - elif self.code_model.options["models_mode"] == "dpg": - if builder.has_stream_response: - deserialize_code.append("deserialized = response.content") - else: - format_filed = ( - f', format="{response.type.encode}"' - if isinstance(response.type, ByteArraySchema) - and response.default_content_type == "application/json" - else "" - ) - response_attr = "json" if _json_serializable(str(response.default_content_type)) else "text" - deserialize_code.append("deserialized = _deserialize(") - deserialize_code.append( - f" {response.type.type_annotation(is_operation_file=True)},{pylint_disable}" - ) - deserialize_code.append(f" response.{response_attr}(){response.result_property}{format_filed}") - deserialize_code.append(")") - - else: - deserialized_value = "ET.fromstring(response.text())" if response.type.is_xml else "response.json()" - deserialize_code.append("if response.content:") - deserialize_code.append(f" deserialized = {deserialized_value}") - deserialize_code.append("else:") - deserialize_code.append(" deserialized = None") - if len(deserialize_code) > 0: - if builder.expose_stream_keyword and stream_logic: - retval.append("if _stream:") - retval.append(" deserialized = response.iter_bytes()") - retval.append("else:") - retval.extend([f" {dc}" for dc in deserialize_code]) - else: - retval.extend(deserialize_code) - return retval - - def handle_error_response(self, builder: OperationType) -> List[str]: - async_await = "await " if self.async_mode else "" - retval = [f"if response.status_code not in {str(builder.success_status_codes)}:"] - retval.extend( - [ - " if _stream:", - f" {async_await} response.read() # Load the body in memory and close the socket", - ] - ) - type_ignore = " # type: ignore" if _need_type_ignore(builder) else "" - retval.append( - f" map_error(status_code=response.status_code, response=response, error_map=error_map){type_ignore}" - ) - error_model = "" - if builder.default_error_deserialization and self.code_model.options["models_mode"]: - if self.code_model.options["models_mode"] == "dpg": - retval.append(f" error = _deserialize({builder.default_error_deserialization}, response.json())") - else: - retval.append( - f" error = self._deserialize.failsafe_deserialize({builder.default_error_deserialization}, " - "pipeline_response)" - ) - error_model = ", model=error" - retval.append( - " raise HttpResponseError(response=response{}{})".format( - error_model, - (", error_format=ARMErrorFormat" if self.code_model.options["azure_arm"] else ""), - ) - ) - return retval - - def handle_response(self, builder: OperationType) -> List[str]: - retval: List[str] = ["response = pipeline_response.http_response"] - retval.append("") - retval.extend(self.handle_error_response(builder)) - retval.append("") - if builder.has_optional_return_type: - retval.append("deserialized = None") - if builder.any_response_has_headers: - retval.append("response_headers = {}") - if builder.has_response_body or builder.any_response_has_headers: - if len(builder.responses) > 1: - for status_code in builder.success_status_codes: - response = builder.get_response_from_status(status_code) - if response.headers or response.type: - retval.append(f"if response.status_code == {status_code}:") - retval.extend( - [f" {line}" for line in self.response_headers_and_deserialization(builder, response)] - ) - retval.append("") - else: - retval.extend(self.response_headers_and_deserialization(builder, builder.responses[0])) - retval.append("") - if builder.has_optional_return_type or self.code_model.options["models_mode"]: - deserialized = "deserialized" - else: - deserialized = f"cast({builder.response_type_annotation(async_mode=self.async_mode)}, deserialized)" - retval.append("if cls:") - retval.append( - " return cls(pipeline_response, {}, {}){}".format( - deserialized if builder.has_response_body else "None", - "response_headers" if builder.any_response_has_headers else "{}", - " # type: ignore", - ) - ) - if builder.has_response_body and any( - response.is_stream_response or response.type for response in builder.responses - ): - retval.append("") - retval.append(f"return {deserialized} # type: ignore") - if builder.request_builder.method == "HEAD" and self.code_model.options["head_as_boolean"]: - retval.append("return 200 <= response.status_code <= 299") - return retval - - def error_map(self, builder: OperationType) -> List[str]: - retval = ["error_map: MutableMapping[int, Type[HttpResponseError]] = {"] - if builder.non_default_errors: - if not 401 in builder.non_default_error_status_codes: - retval.append(" 401: ClientAuthenticationError,") - if not 404 in builder.non_default_error_status_codes: - retval.append(" 404: ResourceNotFoundError,") - if not 409 in builder.non_default_error_status_codes: - retval.append(" 409: ResourceExistsError,") - if not 304 in builder.non_default_error_status_codes: - retval.append(" 304: ResourceNotModifiedError,") - for excep in builder.non_default_errors: - error_model_str = "" - if isinstance(excep.type, ModelType): - if self.code_model.options["models_mode"] == "msrest": - error_model_str = ( - f", model=self._deserialize(" f"_models.{excep.type.serialization_type}, response)" - ) - elif self.code_model.options["models_mode"] == "dpg": - error_model_str = f", model=_deserialize(_models.{excep.type.name}, response.json())" - error_format_str = ", error_format=ARMErrorFormat" if self.code_model.options["azure_arm"] else "" - for status_code in excep.status_codes: - if status_code == 401: - retval.append( - " 401: cast(Type[HttpResponseError], " - "lambda response: ClientAuthenticationError(response=response" - f"{error_model_str}{error_format_str}))," - ) - elif status_code == 404: - retval.append( - " 404: cast(Type[HttpResponseError], " - "lambda response: ResourceNotFoundError(response=response" - f"{error_model_str}{error_format_str}))," - ) - elif status_code == 409: - retval.append( - " 409: cast(Type[HttpResponseError], " - "lambda response: ResourceExistsError(response=response" - f"{error_model_str}{error_format_str}))," - ) - elif status_code == 304: - retval.append( - " 304: cast(Type[HttpResponseError], " - "lambda response: ResourceNotModifiedError(response=response" - f"{error_model_str}{error_format_str}))," - ) - elif not error_model_str and not error_format_str: - retval.append(f" {status_code}: HttpResponseError,") - else: - retval.append( - f" {status_code}: cast(Type[HttpResponseError], " - "lambda response: HttpResponseError(response=response" - f"{error_model_str}{error_format_str}))," - ) - else: - retval.append( - " 401: ClientAuthenticationError, 404: ResourceNotFoundError, 409: ResourceExistsError, " - "304: ResourceNotModifiedError" - ) - retval.append("}") - if builder.has_etag: - retval.extend( - [ - "if match_condition == MatchConditions.IfNotModified:", - " error_map[412] = ResourceModifiedError", - "elif match_condition == MatchConditions.IfPresent:", - " error_map[412] = ResourceNotFoundError", - "elif match_condition == MatchConditions.IfMissing:", - " error_map[412] = ResourceExistsError", - ] - ) - retval.append("error_map.update(kwargs.pop('error_map', {}) or {})") - return retval - - @property - def _call_method(self) -> str: - return "await " if self.async_mode else "" - - -class OperationSerializer(_OperationSerializer[Operation]): ... - - -############################## PAGING OPERATIONS ############################## - -PagingOperationType = TypeVar("PagingOperationType", bound=Union[PagingOperation, LROPagingOperation]) - - -class _PagingOperationSerializer(_OperationSerializer[PagingOperationType]): # pylint: disable=abstract-method - def __init__(self, code_model: CodeModel, async_mode: bool) -> None: - # for pylint reasons need to redefine init - # probably because inheritance is going too deep - super().__init__(code_model, async_mode) - self.code_model = code_model - self.async_mode = async_mode - self.parameter_serializer = ParameterSerializer() - - def serialize_path(self, builder: PagingOperationType) -> List[str]: - return self.parameter_serializer.serialize_path(builder.parameters.path, self.serializer_name) - - def decorators(self, builder: PagingOperationType) -> List[str]: - """Decorators for the method""" - retval: List[str] = [] - if builder.is_overload: - return ["@overload"] - if self.code_model.options["tracing"] and builder.want_tracing: - retval.append("@distributed_trace") - if _api_version_validation(builder): - retval.append(_api_version_validation(builder)) - return retval - - def call_next_link_request_builder(self, builder: PagingOperationType) -> List[str]: - if builder.next_request_builder: - request_builder = builder.next_request_builder - template_url = None - else: - request_builder = builder.request_builder - template_url = "next_link" - - request_builder = builder.next_request_builder or builder.request_builder - if builder.next_request_builder: - return self._call_request_builder_helper( - builder, - request_builder, - template_url=template_url, - is_next_request=True, - ) - retval: List[str] = [] - query_str = "" - next_link_str = "next_link" - try: - api_version_param = next( - p for p in builder.client.parameters if p.is_api_version and p.location == ParameterLocation.QUERY - ) - retval.append("# make call to next link with the client's api-version") - retval.append("_parsed_next_link = urllib.parse.urlparse(next_link)") - retval.extend( - [ - "_next_request_params = case_insensitive_dict({", - " key: [urllib.parse.quote(v) for v in value]" - " for key, value in urllib.parse.parse_qs(_parsed_next_link.query).items()" - "})", - ] - ) - api_version = ( - "self._api_version" - if self.code_model.options["multiapi"] and builder.group_name - else api_version_param.full_client_name - ) - retval.append(f'_next_request_params["api-version"] = {api_version}') - query_str = ", params=_next_request_params" - next_link_str = "urllib.parse.urljoin(next_link, _parsed_next_link.path)" - except StopIteration: - pass - - retval.append(f'_request = HttpRequest("GET", {next_link_str}{query_str})') - retval.extend(self._postprocess_http_request(builder, "_request.url")) - - return retval - - def _prepare_request_callback(self, builder: PagingOperationType) -> List[str]: - retval = self._initialize_overloads(builder) - retval.append("def prepare_request(next_link=None):") - retval.append(" if not next_link:") - retval.extend([f" {line}" for line in self.call_request_builder(builder, is_paging=True)]) - retval.append("") - retval.append(" else:") - retval.extend([f" {line}" for line in self.call_next_link_request_builder(builder)]) - if not builder.next_request_builder and self.code_model.is_legacy: - retval.append(' _request.method = "GET"') - else: - retval.append("") - retval.append(" return _request") - return retval - - @property - def _function_def(self) -> str: - return "def" - - def _extract_data_callback(self, builder: PagingOperationType) -> List[str]: - retval = [f"{'async ' if self.async_mode else ''}def extract_data(pipeline_response):"] - response = builder.responses[0] - deserialized = "pipeline_response.http_response.json()" - if self.code_model.options["models_mode"] == "msrest": - deserialize_type = response.serialization_type - pylint_disable = " # pylint: disable=protected-access" - if isinstance(response.type, ModelType) and not response.type.internal: - deserialize_type = f'"{response.serialization_type}"' - pylint_disable = "" - deserialized = f"self._deserialize(\n {deserialize_type},{pylint_disable}\n pipeline_response\n)" - retval.append(f" deserialized = {deserialized}") - elif self.code_model.options["models_mode"] == "dpg": - # we don't want to generate paging models for DPG - retval.append(f" deserialized = {deserialized}") - else: - retval.append(f" deserialized = {deserialized}") - item_name = builder.item_name - access = f".{item_name}" if self.code_model.options["models_mode"] == "msrest" else f'["{item_name}"]' - list_of_elem_deserialized = "" - if self.code_model.options["models_mode"] == "dpg": - item_type = builder.item_type.type_annotation(is_operation_file=True) - list_of_elem_deserialized = f"_deserialize({item_type}, deserialized{access})" - else: - list_of_elem_deserialized = f"deserialized{access}" - retval.append(f" list_of_elem = {list_of_elem_deserialized}") - retval.append(" if cls:") - retval.append(" list_of_elem = cls(list_of_elem) # type: ignore") - - continuation_token_name = builder.continuation_token_name - if not continuation_token_name: - cont_token_property = "None" - elif self.code_model.options["models_mode"] == "msrest": - cont_token_property = f"deserialized.{continuation_token_name} or None" - else: - cont_token_property = f'deserialized.get("{continuation_token_name}") or None' - list_type = "AsyncList" if self.async_mode else "iter" - retval.append(f" return {cont_token_property}, {list_type}(list_of_elem)") - return retval - - def _get_next_callback(self, builder: PagingOperationType) -> List[str]: - retval = [f"{'async ' if self.async_mode else ''}def get_next(next_link=None):"] - retval.append(" _request = prepare_request(next_link)") - retval.append("") - retval.extend([f" {l}" for l in self.make_pipeline_call(builder)]) - retval.append(" response = pipeline_response.http_response") - retval.append("") - retval.extend([f" {line}" for line in self.handle_error_response(builder)]) - retval.append("") - retval.append(" return pipeline_response") - return retval - - def set_up_params_for_pager(self, builder: PagingOperationType) -> List[str]: - retval = [] - retval.extend(self.error_map(builder)) - retval.extend(self._prepare_request_callback(builder)) - retval.append("") - retval.extend(self._extract_data_callback(builder)) - retval.append("") - retval.extend(self._get_next_callback(builder)) - return retval - - -class PagingOperationSerializer(_PagingOperationSerializer[PagingOperation]): ... - - -############################## LRO OPERATIONS ############################## - -LROOperationType = TypeVar("LROOperationType", bound=Union[LROOperation, LROPagingOperation]) - - -class _LROOperationSerializer(_OperationSerializer[LROOperationType]): - def __init__(self, code_model: CodeModel, async_mode: bool) -> None: - # for pylint reasons need to redefine init - # probably because inheritance is going too deep - super().__init__(code_model, async_mode) - self.code_model = code_model - self.async_mode = async_mode - self.parameter_serializer = ParameterSerializer() - - def serialize_path(self, builder: LROOperationType) -> List[str]: - return self.parameter_serializer.serialize_path(builder.parameters.path, self.serializer_name) - - def initial_call(self, builder: LROOperationType) -> List[str]: - retval = [ - f"polling: Union[bool, {builder.get_base_polling_method(self.async_mode)}] = kwargs.pop('polling', True)", - ] - retval.append("lro_delay = kwargs.pop(") - retval.append(" 'polling_interval',") - retval.append(" self._config.polling_interval") - retval.append(")") - retval.append("cont_token: Optional[str] = kwargs.pop('continuation_token', None)") - retval.append("if cont_token is None:") - retval.append( - f" raw_result = {self._call_method}self.{builder.initial_operation.name}(" - f"{'' if any(rsp.type for rsp in builder.initial_operation.responses) else ' # type: ignore'}" - ) - retval.extend( - [f" {parameter.client_name}={parameter.client_name}," for parameter in builder.parameters.method] - ) - retval.append(" cls=lambda x,y,z: x,") - retval.append(" headers=_headers,") - retval.append(" params=_params,") - retval.append(" **kwargs") - retval.append(" )") - retval.append("kwargs.pop('error_map', None)") - return retval - - def return_lro_poller(self, builder: LROOperationType) -> List[str]: - retval = [] - lro_options_str = ( - "lro_options={'final-state-via': '" + builder.lro_options["final-state-via"] + "'}," - if builder.lro_options - else "" - ) - path_format_arguments_str = "" - if builder.parameters.path: - path_format_arguments_str = "path_format_arguments=path_format_arguments," - retval.extend(self.serialize_path(builder)) - retval.append("") - retval.extend( - [ - "if polling is True:", - f" polling_method: {builder.get_base_polling_method(self.async_mode)} " - + f"= cast({builder.get_base_polling_method(self.async_mode)}, " - f"{builder.get_polling_method(self.async_mode)}(", - " lro_delay,", - f" {lro_options_str}", - f" {path_format_arguments_str}", - " **kwargs", - "))", - ] - ) - retval.append( - f"elif polling is False: polling_method = cast({builder.get_base_polling_method(self.async_mode)}, " - f"{builder.get_no_polling_method(self.async_mode)}())" - ) - retval.append("else: polling_method = polling") - retval.append("if cont_token:") - retval.append(f" return {builder.get_poller_with_response_type(self.async_mode)}.from_continuation_token(") - retval.append(" polling_method=polling_method,") - retval.append(" continuation_token=cont_token,") - retval.append(" client=self._client,") - retval.append(" deserialization_callback=get_long_running_output") - retval.append(" )") - retval.append(f"return {builder.get_poller_with_response_type(self.async_mode)}(") - retval.append(" self._client, raw_result, get_long_running_output, polling_method # type: ignore") - retval.append(" )") - return retval - - def get_long_running_output(self, builder: LROOperationType) -> List[str]: - pylint_disable = "" - if not builder.lro_response: - pylint_disable = " # pylint: disable=inconsistent-return-statements" - retval = [f"def get_long_running_output(pipeline_response):{pylint_disable}"] - if builder.lro_response: - if builder.lro_response.headers: - retval.append(" response_headers = {}") - if ( - not self.code_model.options["models_mode"] - or self.code_model.options["models_mode"] == "dpg" - or builder.lro_response.headers - ): - retval.append(" response = pipeline_response.http_response") - retval.extend( - [f" {line}" for line in self.response_headers_and_deserialization(builder, builder.lro_response)] - ) - retval.append(" if cls:") - retval.append( - " return cls(pipeline_response, {}, {}){}".format( - ("deserialized" if builder.lro_response and builder.lro_response.type else "None"), - ("response_headers" if builder.lro_response and builder.lro_response.headers else "{}"), - " # type: ignore", - ) - ) - if builder.lro_response and builder.lro_response.type: - retval.append(" return deserialized") - return retval - - -class LROOperationSerializer(_LROOperationSerializer[LROOperation]): ... - - -############################## LRO PAGING OPERATIONS ############################## - - -class LROPagingOperationSerializer( - _LROOperationSerializer[LROPagingOperation], - _PagingOperationSerializer[LROPagingOperation], -): # pylint: disable=abstract-method - @property - def _call_method(self) -> str: - return "await " if self.async_mode else "" - - @property - def _function_def(self) -> str: - return "async def" if self.async_mode else "def" - - def get_long_running_output(self, builder: LROPagingOperation) -> List[str]: - retval = ["def get_long_running_output(pipeline_response):"] - retval.append(f" {self._function_def} internal_get_next(next_link=None):") - retval.append(" if next_link is None:") - retval.append(" return pipeline_response") - retval.append(f" return {self._call_method}get_next(next_link)") - retval.append("") - retval.append(f" return {builder.get_pager(self.async_mode)}(") - retval.append(" internal_get_next, extract_data") - retval.append(" )") - return retval - - def decorators(self, builder: LROPagingOperation) -> List[str]: - """Decorators for the method""" - return _LROOperationSerializer.decorators(self, builder) # type: ignore - - -def get_operation_serializer( - builder: Operation, - code_model, - async_mode: bool, -) -> Union[ - OperationSerializer, - PagingOperationSerializer, - LROOperationSerializer, - LROPagingOperationSerializer, -]: - retcls: Union[ - Type[OperationSerializer], - Type[PagingOperationSerializer], - Type[LROOperationSerializer], - Type[LROPagingOperationSerializer], - ] = OperationSerializer - if builder.operation_type == "lropaging": - retcls = LROPagingOperationSerializer - elif builder.operation_type == "lro": - retcls = LROOperationSerializer - elif builder.operation_type == "paging": - retcls = PagingOperationSerializer - return retcls(code_model, async_mode) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/client_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/client_serializer.py deleted file mode 100644 index 28041370efb..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/client_serializer.py +++ /dev/null @@ -1,295 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import List - -from . import utils -from ..models import Client, ParameterMethodLocation -from .parameter_serializer import ParameterSerializer, PopKwargType -from ...utils import build_policies - - -class ClientSerializer: - def __init__(self, client: Client) -> None: - self.client = client - self.parameter_serializer = ParameterSerializer() - - def _init_signature(self, async_mode: bool) -> str: - pylint_disable = "" - if not self.client.parameters.credential: - pylint_disable = " # pylint: disable=missing-client-constructor-parameter-credential" - return self.parameter_serializer.serialize_method( - function_def="def", - method_name="__init__", - need_self_param=True, - method_param_signatures=self.client.parameters.method_signature(async_mode), - pylint_disable=pylint_disable, - ) - - def init_signature_and_response_type_annotation(self, async_mode: bool) -> str: - init_signature = self._init_signature(async_mode) - return utils.method_signature_and_response_type_annotation_template( - method_signature=init_signature, - response_type_annotation="None", - ) - - def pop_kwargs_from_signature(self) -> List[str]: - return self.parameter_serializer.pop_kwargs_from_signature( - self.client.parameters.kwargs_to_pop, - check_kwarg_dict=False, - pop_headers_kwarg=PopKwargType.NO, - pop_params_kwarg=PopKwargType.NO, - ) - - @property - def class_definition(self) -> str: - class_name = self.client.name - base_class = "" - if self.client.has_mixin: - base_class = f"{class_name}OperationsMixin" - pylint_disable = self.client.pylint_disable - if base_class: - return f"class {class_name}({base_class}):{pylint_disable}" - return f"class {class_name}:{pylint_disable}" - - def property_descriptions(self, async_mode: bool) -> List[str]: - retval: List[str] = [] - operations_folder = ".aio.operations." if async_mode else ".operations." - for og in [og for og in self.client.operation_groups if not og.is_mixin]: - retval.append(f":ivar {og.property_name}: {og.class_name} operations") - property_type = f"{self.client.code_model.namespace}{operations_folder}{og.class_name}" - retval.append(f":vartype {og.property_name}: {property_type}") - for param in self.client.parameters.method: - retval.append(f":{param.description_keyword} {param.client_name}: {param.description}") - retval.append( - f":{param.docstring_type_keyword} {param.client_name}: {param.docstring_type(async_mode=async_mode)}" - ) - if self.client.has_public_lro_operations: - retval.append( - ":keyword int polling_interval: Default waiting time between two polls for LRO operations " - "if no Retry-After header is present." - ) - retval = [s.replace("\\", "\\\\") for s in retval] - retval.append('"""') - return retval - - def initialize_config(self) -> str: - config_name = f"{self.client.name}Configuration" - config_call = ", ".join( - [ - f"{p.client_name}={p.client_name}" - for p in self.client.config.parameters.method - if p.method_location != ParameterMethodLocation.KWARG - ] - + ["**kwargs"] - ) - return f"self._config = {config_name}({config_call})" - - @property - def host_variable_name(self) -> str: - try: - return next(p for p in self.client.parameters if p.is_host).client_name - except StopIteration: - return "_endpoint" - - @property - def should_init_super(self) -> bool: - return any(og for og in self.client.operation_groups if og.is_mixin and og.has_abstract_operations) - - def initialize_pipeline_client(self, async_mode: bool) -> List[str]: - result = [] - pipeline_client_name = self.client.pipeline_class(async_mode) - endpoint_name = "base_url" if self.client.code_model.is_azure_flavor else "endpoint" - params = { - endpoint_name: self.host_variable_name, - "policies": "_policies", - } - if not self.client.code_model.is_legacy and self.client.request_id_header_name: - result.append(f'kwargs["request_id_header_name"] = "{self.client.request_id_header_name}"') - policies = build_policies( - self.client.code_model.options["azure_arm"], - async_mode, - is_azure_flavor=self.client.code_model.is_azure_flavor, - tracing=self.client.code_model.options["tracing"], - ) - result.extend( - [ - "_policies = kwargs.pop('policies', None)", - "if _policies is None:", - f' _policies = [{",".join(policies)}]', - f"self._client: {pipeline_client_name} = {pipeline_client_name}(" - f"{', '.join(f'{k}={v}' for k, v in params.items())}, **kwargs)", - ] - ) - return result - - def serializers_and_operation_groups_properties(self) -> List[str]: - retval = [] - - def _get_client_models_value(models_dict_name: str) -> str: - if self.client.code_model.model_types: - return f"{{k: v for k, v in {models_dict_name}.__dict__.items() if isinstance(v, type)}}" - return "{}" - - is_msrest_model = self.client.code_model.options["models_mode"] == "msrest" - if is_msrest_model: - add_private_models = len(self.client.code_model.model_types) != len( - self.client.code_model.public_model_types - ) - model_dict_name = f"_models.{self.client.code_model.models_filename}" if add_private_models else "_models" - retval.append( - f"client_models{': Dict[str, Any]' if not self.client.code_model.model_types else ''}" - f" = {_get_client_models_value(model_dict_name)}" - ) - if add_private_models and self.client.code_model.model_types: - update_dict = "{k: v for k, v in _models.__dict__.items() if isinstance(v, type)}" - retval.append(f"client_models.update({update_dict})") - client_models_str = "client_models" if is_msrest_model else "" - retval.append(f"self._serialize = Serializer({client_models_str})") - retval.append(f"self._deserialize = Deserializer({client_models_str})") - if not self.client.code_model.options["client_side_validation"]: - retval.append("self._serialize.client_side_validation = False") - operation_groups = [og for og in self.client.operation_groups if not og.is_mixin] - for og in operation_groups: - if og.code_model.options["multiapi"]: - api_version = f", '{og.api_versions[0]}'" if og.api_versions else ", None" - else: - api_version = "" - retval.extend( - [ - f"self.{og.property_name} = {og.class_name}(", - f" self._client, self._config, self._serialize, self._deserialize{api_version}", - ")", - ] - ) - return retval - - def _send_request_signature(self) -> str: - send_request_signature = [ - "request: HttpRequest, *, stream: bool = False," - ] + self.client.parameters.method_signature_kwargs - return self.parameter_serializer.serialize_method( - function_def="def", - method_name=self.client.send_request_name, - need_self_param=True, - method_param_signatures=send_request_signature, - ) - - def send_request_signature_and_response_type_annotation(self, async_mode: bool) -> str: - send_request_signature = self._send_request_signature() - return utils.method_signature_and_response_type_annotation_template( - method_signature=send_request_signature, - response_type_annotation=("Awaitable[AsyncHttpResponse]" if async_mode else "HttpResponse"), - ) - - def _example_make_call(self, async_mode: bool) -> List[str]: - http_response = "AsyncHttpResponse" if async_mode else "HttpResponse" - retval = [f">>> response = {'await ' if async_mode else ''}client.{self.client.send_request_name}(request)"] - retval.append(f"<{http_response}: 200 OK>") - return retval - - def _request_builder_example(self, async_mode: bool) -> List[str]: - retval = [ - "We have helper methods to create requests specific to this service in " - + f"`{self.client.code_model.namespace}.{self.client.code_model.rest_layer_name}`." - ] - retval.append("Use these helper methods to create the request you pass to this method.") - retval.append("") - - request_builder = self.client.request_builders[0] - request_builder_signature = ", ".join(request_builder.parameters.call) - if request_builder.group_name: - rest_imported = request_builder.group_name - request_builder_name = f"{request_builder.group_name}.{request_builder.name}" - else: - rest_imported = request_builder.name - request_builder_name = request_builder.name - full_path = f"{self.client.code_model.namespace}.{self.client.code_model.rest_layer_name}" - retval.append(f">>> from {full_path} import {rest_imported}") - retval.append(f">>> request = {request_builder_name}({request_builder_signature})") - retval.append(f"") - retval.extend(self._example_make_call(async_mode)) - return retval - - def _rest_request_example(self, async_mode: bool) -> List[str]: - retval = [f">>> from {self.client.code_model.core_library}.rest import HttpRequest"] - retval.append('>>> request = HttpRequest("GET", "https://www.example.org/")') - retval.append("") - retval.extend(self._example_make_call(async_mode)) - return retval - - def send_request_description(self, async_mode: bool) -> List[str]: - rest_library = f"{self.client.code_model.core_library}.rest" - retval = ['"""Runs the network request through the client\'s chained policies.'] - retval.append("") - if self.client.code_model.options["builders_visibility"] != "embedded": - retval.extend(self._request_builder_example(async_mode)) - else: - retval.extend(self._rest_request_example(async_mode)) - retval.append("") - retval.append("For more information on this code flow, see https://aka.ms/azsdk/dpcodegen/python/send_request") - retval.append("") - retval.append(":param request: The network request you want to make. Required.") - retval.append(f":type request: ~{rest_library}.HttpRequest") - retval.append(":keyword bool stream: Whether the response payload will be streamed. Defaults to False.") - retval.append(":return: The response of your network call. Does not do error handling on your response.") - http_response = "AsyncHttpResponse" if async_mode else "HttpResponse" - retval.append(f":rtype: ~{rest_library}.{http_response}") - retval.append('"""') - return retval - - def serialize_path(self) -> List[str]: - return self.parameter_serializer.serialize_path(self.client.parameters.path, "self._serialize") - - -class ConfigSerializer: - def __init__(self, client: Client) -> None: - self.client = client - self.parameter_serializer = ParameterSerializer() - - def _init_signature(self, async_mode: bool) -> str: - return self.parameter_serializer.serialize_method( - function_def="def", - method_name="__init__", - need_self_param=True, - method_param_signatures=self.client.config.parameters.method_signature(async_mode), - ) - - def init_signature_and_response_type_annotation(self, async_mode: bool) -> str: - init_signature = self._init_signature(async_mode) - return utils.method_signature_and_response_type_annotation_template( - method_signature=init_signature, - response_type_annotation="None", - ) - - def pop_kwargs_from_signature(self) -> List[str]: - return self.parameter_serializer.pop_kwargs_from_signature( - self.client.config.parameters.kwargs_to_pop, - check_kwarg_dict=False, - pop_headers_kwarg=PopKwargType.NO, - pop_params_kwarg=PopKwargType.NO, - ) - - def set_constants(self) -> List[str]: - return [ - f"self.{p.client_name} = {p.client_default_value_declaration}" - for p in self.client.config.parameters.constant - if p not in self.client.config.parameters.method - ] - - def check_required_parameters(self) -> List[str]: - return [ - f"if {p.client_name} is None:\n" f" raise ValueError(\"Parameter '{p.client_name}' must not be None.\")" - for p in self.client.config.parameters.method - if not (p.optional or p.constant) - ] - - def property_descriptions(self, async_mode: bool) -> List[str]: - retval: List[str] = [] - for p in self.client.config.parameters.method: - retval.append(f":{p.description_keyword} {p.client_name}: {p.description}") - retval.append(f":{p.docstring_type_keyword} {p.client_name}: {p.docstring_type(async_mode=async_mode)}") - retval.append('"""') - return retval diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/enum_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/enum_serializer.py deleted file mode 100644 index 4b9ce87e3fc..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/enum_serializer.py +++ /dev/null @@ -1,15 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -from .base_serializer import BaseSerializer -from ..models import FileImport - - -class EnumSerializer(BaseSerializer): - def serialize(self) -> str: - # Generate the enum file - template = self.env.get_template("enum_container.py.jinja2") - return template.render(code_model=self.code_model, file_import=FileImport(self.code_model)) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/general_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/general_serializer.py deleted file mode 100644 index b3b0877da3c..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/general_serializer.py +++ /dev/null @@ -1,212 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import json -from typing import Any, List -from jinja2 import Environment -from .import_serializer import FileImportSerializer, TypingSection -from ..models.imports import MsrestImportType, FileImport -from ..models import ( - ImportType, - CodeModel, - TokenCredentialType, - Client, -) -from .client_serializer import ClientSerializer, ConfigSerializer -from .base_serializer import BaseSerializer - - -class GeneralSerializer(BaseSerializer): - """General serializer for SDK root level files""" - - def __init__(self, code_model: CodeModel, env: Environment, async_mode: bool): - super().__init__(code_model, env) - self.async_mode = async_mode - - def serialize_setup_file(self) -> str: - template = self.env.get_template("packaging_templates/setup.py.jinja2") - params = {} - params.update(self.code_model.options) - return template.render(code_model=self.code_model, **params) - - def serialize_package_file(self, template_name: str, **kwargs: Any) -> str: - template = self.env.get_template(template_name) - package_parts = (self.code_model.options["package_name"] or "").split("-")[:-1] - token_credential = any( - c for c in self.code_model.clients if isinstance(getattr(c.credential, "type", None), TokenCredentialType) - ) - version = self.code_model.options["package_version"] - if any(x in version for x in ["a", "b", "rc"]) or version[0] == "0": - dev_status = "4 - Beta" - else: - dev_status = "5 - Production/Stable" - params = { - "code_model": self.code_model, - "dev_status": dev_status, - "token_credential": token_credential, - "pkgutil_names": [".".join(package_parts[: i + 1]) for i in range(len(package_parts))], - "init_names": ["/".join(package_parts[: i + 1]) + "/__init__.py" for i in range(len(package_parts))], - "client_name": self.code_model.clients[0].name, - "namespace": self.code_model.namespace, - } - params.update(self.code_model.options) - params.update(kwargs) - return template.render(file_import=FileImport(self.code_model), **params) - - def serialize_pkgutil_init_file(self) -> str: - template = self.env.get_template("pkgutil_init.py.jinja2") - return template.render() - - def serialize_init_file(self, clients: List[Client]) -> str: - template = self.env.get_template("init.py.jinja2") - return template.render( - code_model=self.code_model, - clients=clients, - async_mode=self.async_mode, - ) - - def serialize_service_client_file(self, clients: List[Client]) -> str: - template = self.env.get_template("client_container.py.jinja2") - - imports = FileImport(self.code_model) - for client in clients: - imports.merge(client.imports(self.async_mode)) - - return template.render( - code_model=self.code_model, - clients=clients, - async_mode=self.async_mode, - get_serializer=ClientSerializer, - imports=FileImportSerializer(imports), - ) - - def serialize_vendor_file(self, clients: List[Client]) -> str: - template = self.env.get_template("vendor.py.jinja2") - - # configure imports - file_import = FileImport(self.code_model) - if self.code_model.need_mixin_abc: - file_import.add_submodule_import( - "abc", - "ABC", - ImportType.STDLIB, - ) - file_import.add_submodule_import( - "" if self.code_model.is_azure_flavor else "runtime", - f"{'Async' if self.async_mode else ''}PipelineClient", - ImportType.SDKCORE, - TypingSection.TYPING, - ) - file_import.add_msrest_import( - relative_path=".." if self.async_mode else ".", - msrest_import_type=MsrestImportType.SerializerDeserializer, - typing_section=TypingSection.TYPING, - ) - for client in clients: - file_import.add_submodule_import( - "._configuration", - f"{client.name}Configuration", - ImportType.LOCAL, - ) - if self.code_model.has_etag: - file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) - file_import.add_submodule_import( - "", - "MatchConditions", - ImportType.SDKCORE, - ) - if self.code_model.has_form_data and self.code_model.options["models_mode"] == "dpg" and not self.async_mode: - file_import.add_submodule_import("typing", "IO", ImportType.STDLIB) - file_import.add_submodule_import("typing", "Tuple", ImportType.STDLIB) - file_import.add_submodule_import("typing", "Union", ImportType.STDLIB) - file_import.add_submodule_import("typing", "Optional", ImportType.STDLIB) - file_import.add_submodule_import("typing", "Mapping", ImportType.STDLIB) - file_import.add_submodule_import("typing", "Sequence", ImportType.STDLIB) - file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) - file_import.add_submodule_import("typing", "List", ImportType.STDLIB) - file_import.add_submodule_import( - "._model_base", - "SdkJSONEncoder", - ImportType.LOCAL, - ) - file_import.add_submodule_import( - "._model_base", - "Model", - ImportType.LOCAL, - ) - file_import.add_import("json", ImportType.STDLIB) - - return template.render( - code_model=self.code_model, - imports=FileImportSerializer( - file_import, - ), - async_mode=self.async_mode, - clients=clients, - ) - - def serialize_config_file(self, clients: List[Client]) -> str: - template = self.env.get_template("config_container.py.jinja2") - imports = FileImport(self.code_model) - for client in self.code_model.clients: - imports.merge(client.config.imports(self.async_mode)) - return template.render( - code_model=self.code_model, - async_mode=self.async_mode, - imports=FileImportSerializer(imports), - get_serializer=ConfigSerializer, - clients=clients, - ) - - def serialize_version_file(self) -> str: - template = self.env.get_template("version.py.jinja2") - return template.render(code_model=self.code_model) - - def serialize_serialization_file(self) -> str: - template = self.env.get_template("serialization.py.jinja2") - return template.render( - code_model=self.code_model, - ) - - def serialize_model_base_file(self) -> str: - template = self.env.get_template("model_base.py.jinja2") - return template.render(code_model=self.code_model, file_import=FileImport(self.code_model)) - - def serialize_validation_file(self) -> str: - template = self.env.get_template("validation.py.jinja2") - return template.render(code_model=self.code_model) - - def serialize_cross_language_definition_file(self) -> str: - cross_langauge_def_dict = { - f"{self.code_model.namespace}.models.{model.name}": model.cross_language_definition_id - for model in self.code_model.model_types - } - cross_langauge_def_dict.update( - { - f"{self.code_model.namespace}.models.{enum.name}": enum.cross_language_definition_id - for enum in self.code_model.enums - } - ) - cross_langauge_def_dict.update( - { - ( - f"{self.code_model.namespace}.{client.name}." - + ("" if operation_group.is_mixin else f"{operation_group.property_name}.") - + f"{operation.name}" - ): operation.cross_language_definition_id - for client in self.code_model.clients - for operation_group in client.operation_groups - for operation in operation_group.operations - if not operation.name.startswith("_") - } - ) - return json.dumps( - { - "CrossLanguagePackageId": self.code_model.cross_language_package_id, - "CrossLanguageDefinitionId": cross_langauge_def_dict, - }, - indent=4, - ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/import_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/import_serializer.py deleted file mode 100644 index 2824a1e6750..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/import_serializer.py +++ /dev/null @@ -1,127 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from copy import deepcopy -from typing import List -from ..models.imports import ( - ImportType, - FileImport, - ImportModel, - TypingSection, - TypeDefinition, -) - - -def _serialize_package(imports: List[ImportModel], delimiter: str) -> str: - buffer = [] - if any(i for i in imports if i.submodule_name is None): - buffer.append(f"import {imports[0].module_name}{f' as {imports[0].alias}' if imports[0].alias else ''}") - else: - import_str = ", ".join( - sorted( - set( - f"{i.submodule_name} as {i.alias}" if i.alias else i.submodule_name for i in imports # type: ignore - ) - ) - ) - buffer.append(f"from {imports[0].module_name} import {import_str}") - return delimiter.join(buffer) - - -def _serialize_versioned_package(i: ImportModel, delimiter: str) -> str: - if not i.version_modules: - return "" - buffer = [] - for n, (version, module_name, comment) in enumerate(i.version_modules): - buffer.append("{} sys.version_info >= {}:".format("if" if n == 0 else "elif", version)) - buffer.append( - f" from {module_name} import {i.submodule_name}{f' as {i.alias}' if i.alias else ''}" - f"{f' # {comment}' if comment else ''}" - ) - buffer.append("else:") - buffer.append( - f" from {i.module_name} import {i.submodule_name}{f' as {i.alias}' if i.alias else ''}" - " # type: ignore # pylint: disable=ungrouped-imports" - ) - return delimiter.join(buffer) - - -def _serialize_import_type(imports: List[ImportModel], delimiter: str) -> str: - """Serialize a given import type.""" - import_list = [] - for module_name in sorted(set(i.module_name for i in imports)): - normal_imports = [i for i in imports if i.module_name == module_name and not i.version_modules] - versioned_imports = [i for i in imports if i.module_name == module_name and i.version_modules] - if normal_imports: - import_list.append(_serialize_package(normal_imports, delimiter)) - for i in versioned_imports: - import_list.append(_serialize_versioned_package(i, delimiter)) - return delimiter.join(import_list) - - -def _get_import_clauses(imports: List[ImportModel], delimiter: str) -> List[str]: - import_clause = [] - for import_type in ImportType: - imports_with_import_type = [i for i in imports if i.import_type == import_type] - if imports_with_import_type: - import_clause.append(_serialize_import_type(imports_with_import_type, delimiter)) - return import_clause - - -class FileImportSerializer: - def __init__(self, file_import: FileImport, async_mode: bool = False) -> None: - self.file_import = file_import - self.async_mode = async_mode - - def _get_imports_list(self, baseline_typing_section: TypingSection, add_conditional_typing: bool): - # If this is a python 3 file, our regular imports include the CONDITIONAL category - # If this is not a python 3 file, our typing imports include the CONDITIONAL category - file_import_copy = deepcopy(self.file_import) - if add_conditional_typing and any(self.file_import.get_imports_from_section(TypingSection.CONDITIONAL)): - # we switch the TypingSection key for the CONDITIONAL typing imports so we can merge - # the imports together - for i in file_import_copy.imports: - if i.typing_section == TypingSection.CONDITIONAL: - i.typing_section = baseline_typing_section - return file_import_copy.get_imports_from_section(baseline_typing_section) - - def _add_type_checking_import(self): - if any(self.file_import.get_imports_from_section(TypingSection.TYPING)): - self.file_import.add_submodule_import("typing", "TYPE_CHECKING", ImportType.STDLIB) - - def get_typing_definitions(self) -> str: - def declare_definition(type_name: str, type_definition: TypeDefinition) -> List[str]: - ret: List[str] = [] - definition_value = type_definition.async_definition if self.async_mode else type_definition.sync_definition - ret.append("{} = {}".format(type_name, definition_value)) - return ret - - if not self.file_import.type_definitions: - return "" - declarations: List[str] = [""] - for type_name, value in self.file_import.type_definitions.items(): - declarations.extend(declare_definition(type_name, value)) - return "\n".join(declarations) - - def __str__(self) -> str: - self._add_type_checking_import() - regular_imports = "" - regular_imports_list = self._get_imports_list( - baseline_typing_section=TypingSection.REGULAR, - add_conditional_typing=True, - ) - - if regular_imports_list: - regular_imports = "\n\n".join(_get_import_clauses(regular_imports_list, "\n")) - - typing_imports = "" - typing_imports_list = self._get_imports_list( - baseline_typing_section=TypingSection.TYPING, - add_conditional_typing=False, - ) - if typing_imports_list: - typing_imports += "\n\nif TYPE_CHECKING:\n # pylint: disable=unused-import,ungrouped-imports\n " - typing_imports += "\n\n ".join(_get_import_clauses(typing_imports_list, "\n ")) - return regular_imports + typing_imports + self.get_typing_definitions() diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/metadata_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/metadata_serializer.py deleted file mode 100644 index 35d374983fc..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/metadata_serializer.py +++ /dev/null @@ -1,198 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import functools -import json -from typing import List, Optional, Set, Tuple, Dict, Union, Any -from jinja2 import Environment -from ..models import ( - OperationGroup, - LROOperation, - PagingOperation, - TypingSection, - ImportType, - CodeModel, -) -from .builder_serializer import get_operation_serializer -from .import_serializer import FileImportSerializer - - -def _to_string(data: Union[Tuple[Any], List[Any], str]) -> str: - if isinstance(data, (list, tuple)): - return "".join([_to_string(item) for item in data]) - return str(data) - - -def _json_serialize_imports( - imports: Dict[ - TypingSection, - Dict[ - ImportType, - Dict[ - str, - Set[ - Optional[ - Union[ - str, - Tuple[str, str], - Tuple[ - str, - Optional[str], - Tuple[Tuple[Tuple[int, int], str, Optional[str]]], - ], - ] - ] - ], - ], - ], - ] -) -> str: - if not imports: - return "" - - json_serialize_imports = {} - # need to make name_import set -> list to make the dictionary json serializable - # not using an OrderedDict since we're iterating through a set and the order there varies - # going to sort the list instead - - for typing_section_key, typing_section_value in imports.items(): - json_import_type_dictionary = {} - for import_type_key, import_type_value in typing_section_value.items(): - json_package_name_dictionary = {} - for package_name, name_imports in import_type_value.items(): - name_import_ordered_list = [] - if name_imports: - name_import_ordered_list = list(name_imports) - name_import_ordered_list.sort( - key=lambda e: ( - _to_string(e) # type: ignore - if isinstance(e, (list, tuple)) - else e if isinstance(e, str) else "" - ) - ) - json_package_name_dictionary[package_name] = name_import_ordered_list - json_import_type_dictionary[import_type_key] = json_package_name_dictionary - json_serialize_imports[typing_section_key] = json_import_type_dictionary - return json.dumps(json_serialize_imports) - - -def _mixin_imports( - mixin_operation_group: Optional[OperationGroup], -) -> Tuple[Optional[str], Optional[str]]: - if not mixin_operation_group: - return None, None - - sync_mixin_imports = mixin_operation_group.imports_for_multiapi(async_mode=False) - async_mixin_imports = mixin_operation_group.imports_for_multiapi(async_mode=True) - - return _json_serialize_imports(sync_mixin_imports.to_dict()), _json_serialize_imports(async_mixin_imports.to_dict()) - - -def _mixin_typing_definitions( - mixin_operation_group: Optional[OperationGroup], -) -> Tuple[Optional[str], Optional[str]]: - if not mixin_operation_group: - return None, None - - sync_mixin_imports = mixin_operation_group.imports_for_multiapi(async_mode=False) - async_mixin_imports = mixin_operation_group.imports_for_multiapi(async_mode=True) - sync_mixin_typing_definitions = FileImportSerializer(sync_mixin_imports, False).get_typing_definitions() - async_mixin_typing_definitions = FileImportSerializer(async_mixin_imports, True).get_typing_definitions() - - return sync_mixin_typing_definitions, async_mixin_typing_definitions - - -class MetadataSerializer: - def __init__(self, code_model: CodeModel, env: Environment) -> None: - self.code_model = code_model - self.client = self.code_model.clients[0] # we only do one client for multiapi - self.env = env - - def _choose_api_version(self) -> Tuple[str, List[str]]: - chosen_version = "" - total_api_version_set: Set[str] = set() - for client in self.code_model.clients: - for operation_group in client.operation_groups: - total_api_version_set.update(operation_group.api_versions) - - total_api_version_list = list(total_api_version_set) - total_api_version_list.sort() - - # switching ' to " so json can decode the dict we end up writing to file - total_api_version_list = [str(api_version).replace("'", '"') for api_version in total_api_version_list] - if len(total_api_version_list) == 1: - chosen_version = total_api_version_list[0] - elif len(total_api_version_list) > 1: - module_version = self.code_model.namespace.split(".")[-1] - for api_version in total_api_version_list: - if "v{}".format(api_version.replace("-", "_")) == module_version: - chosen_version = api_version - - return chosen_version, total_api_version_list - - def serialize(self) -> str: - def _is_lro(operation): - return isinstance(operation, LROOperation) - - def _is_paging(operation): - return isinstance(operation, PagingOperation) - - mixin_operation_group: Optional[OperationGroup] = next( - ( - operation_group - for client in self.code_model.clients - for operation_group in client.operation_groups - if operation_group.is_mixin - ), - None, - ) - mixin_operations = mixin_operation_group.operations if mixin_operation_group else [] - sync_mixin_imports, async_mixin_imports = _mixin_imports(mixin_operation_group) - ( - sync_mixin_typing_definitions, - async_mixin_typing_definitions, - ) = _mixin_typing_definitions(mixin_operation_group) - - chosen_version, total_api_version_list = self._choose_api_version() - - # setting to true, because for multiapi we always generate with a version file with version 0.1.0 - self.code_model.options["package_version"] = "0.1.0" - template = self.env.get_template("metadata.json.jinja2") - - return template.render( - code_model=self.code_model, - chosen_version=chosen_version, - total_api_version_list=total_api_version_list, - client=self.client, - global_parameters=self.client.parameters, - mixin_operations=mixin_operations, - any=any, - is_lro=_is_lro, - is_paging=_is_paging, - str=str, - sync_mixin_imports=sync_mixin_imports, - async_mixin_imports=async_mixin_imports, - sync_mixin_typing_definitions=sync_mixin_typing_definitions, - async_mixin_typing_definitions=async_mixin_typing_definitions, - sync_client_imports=_json_serialize_imports(self.client.imports_for_multiapi(async_mode=False).to_dict()), - async_client_imports=_json_serialize_imports(self.client.imports_for_multiapi(async_mode=True).to_dict()), - sync_config_imports=_json_serialize_imports( - self.client.config.imports_for_multiapi(async_mode=False).to_dict() - ), - async_config_imports=_json_serialize_imports( - self.client.config.imports_for_multiapi(async_mode=True).to_dict() - ), - get_async_operation_serializer=functools.partial( - get_operation_serializer, - code_model=self.client.code_model, - async_mode=True, - ), - get_sync_operation_serializer=functools.partial( - get_operation_serializer, - code_model=self.client.code_model, - async_mode=False, - ), - has_credential=bool(self.client.credential), - ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/model_init_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/model_init_serializer.py deleted file mode 100644 index 5df688adbb8..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/model_init_serializer.py +++ /dev/null @@ -1,33 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from jinja2 import Environment -from ..models import CodeModel - - -class ModelInitSerializer: - def __init__(self, code_model: CodeModel, env: Environment) -> None: - self.code_model = code_model - self.env = env - - def serialize(self) -> str: - schemas = [s.name for s in self.code_model.public_model_types] - schemas.sort() - enums = [e.name for e in self.code_model.enums if not e.internal] if self.code_model.enums else None - - if enums: - enums.sort() - - # check to see if we have any duplicate names between enum and object schemas - model_enum_name_intersection = set(schemas).intersection(set(enums)) - if model_enum_name_intersection: - raise ValueError( - "We have models and enums sharing the following names: {}".format( - ", ".join(model_enum_name_intersection) - ) - ) - - template = self.env.get_template("model_init.py.jinja2") - return template.render(code_model=self.code_model, schemas=schemas, enums=enums) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/model_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/model_serializer.py deleted file mode 100644 index b42d35e9709..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/model_serializer.py +++ /dev/null @@ -1,287 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import List -from abc import ABC, abstractmethod - -from ..models import ModelType, Property, ConstantType, EnumValue -from ..models.imports import FileImport, TypingSection, MsrestImportType, ImportType -from .import_serializer import FileImportSerializer -from .base_serializer import BaseSerializer - - -def _documentation_string(prop: Property, description_keyword: str, docstring_type_keyword: str) -> List[str]: - retval: List[str] = [] - sphinx_prefix = f":{description_keyword} {prop.client_name}:" - description = prop.description(is_operation_file=False).replace("\\", "\\\\") - retval.append(f"{sphinx_prefix} {description}" if description else sphinx_prefix) - retval.append(f":{docstring_type_keyword} {prop.client_name}: {prop.type.docstring_type()}") - return retval - - -class _ModelSerializer(BaseSerializer, ABC): - @abstractmethod - def imports(self) -> FileImport: ... - - def serialize(self) -> str: - # Generate the models - template = self.env.get_template("model_container.py.jinja2") - return template.render( - code_model=self.code_model, - imports=FileImportSerializer(self.imports()), - str=str, - serializer=self, - ) - - @abstractmethod - def declare_model(self, model: ModelType) -> str: ... - - @staticmethod - def escape_dot(s: str): - return s.replace(".", "\\\\.") - - @staticmethod - def input_documentation_string(prop: Property) -> List[str]: - # building the param line of the property doc - return _documentation_string(prop, "keyword", "paramtype") - - @staticmethod - def variable_documentation_string(prop: Property) -> List[str]: - return _documentation_string(prop, "ivar", "vartype") - - def super_call(self, model: ModelType) -> List[str]: - return [f"super().__init__({self.properties_to_pass_to_super(model)})"] - - @staticmethod - def initialize_discriminator_property(model: ModelType, prop: Property) -> str: - discriminator_value = f"'{model.discriminator_value}'" if model.discriminator_value else None - if not discriminator_value: - typing = "Optional[str]" - else: - typing = "str" - return f"self.{prop.client_name}: {typing} = {discriminator_value}" - - @staticmethod - def initialize_standard_property(prop: Property): - if not (prop.optional or prop.client_default_value is not None): - return f"{prop.client_name}: {prop.type_annotation()},{prop.pylint_disable}" - return ( - f"{prop.client_name}: {prop.type_annotation()} = " - f"{prop.client_default_value_declaration},{prop.pylint_disable}" - ) - - @staticmethod - def discriminator_docstring(model: ModelType) -> str: - return ( - "You probably want to use the sub-classes and not this class directly. " - f"Known sub-classes are: {', '.join(v.name for v in model.discriminated_subtypes.values())}" - ) - - @staticmethod - def _init_line_parameters(model: ModelType): - return [p for p in model.properties if not p.readonly and not p.is_discriminator and not p.constant] - - def init_line(self, model: ModelType) -> List[str]: - init_properties_declaration = [] - init_line_parameters = self._init_line_parameters(model) - init_line_parameters.sort(key=lambda x: x.optional) - if init_line_parameters: - init_properties_declaration.append("*,") - for param in init_line_parameters: - init_properties_declaration.append(self.initialize_standard_property(param)) - - return init_properties_declaration - - @staticmethod - def properties_to_pass_to_super(model: ModelType) -> str: - properties_to_pass_to_super = [] - for parent in model.parents: - for prop in model.properties: - if prop in parent.properties and not prop.is_discriminator and not prop.constant and not prop.readonly: - properties_to_pass_to_super.append(f"{prop.client_name}={prop.client_name}") - properties_to_pass_to_super.append("**kwargs") - return ", ".join(properties_to_pass_to_super) - - -class MsrestModelSerializer(_ModelSerializer): - def imports(self) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_msrest_import( - relative_path="..", - msrest_import_type=MsrestImportType.Module, - typing_section=TypingSection.REGULAR, - ) - for model in self.code_model.model_types: - file_import.merge(model.imports(is_operation_file=False)) - for param in self._init_line_parameters(model): - file_import.merge(param.imports()) - - return file_import - - def declare_model(self, model: ModelType) -> str: - basename = ( - "msrest.serialization.Model" - if self.code_model.options["client_side_validation"] - else "_serialization.Model" - ) - if model.parents: - basename = ", ".join([m.name for m in model.parents]) - return f"class {model.name}({basename}):{model.pylint_disable}" - - @staticmethod - def get_properties_to_initialize(model: ModelType) -> List[Property]: - if model.parents: - properties_to_initialize = list( - { - p.client_name: p - for bm in model.parents - for p in model.properties - if p not in bm.properties or p.is_discriminator or p.constant - }.values() - ) - else: - properties_to_initialize = model.properties - return properties_to_initialize - - def initialize_properties(self, model: ModelType) -> List[str]: - init_args = [] - for prop in self.get_properties_to_initialize(model): - if prop.is_discriminator: - init_args.append(self.initialize_discriminator_property(model, prop)) - elif prop.readonly: - init_args.append(f"self.{prop.client_name} = None") - elif not prop.constant: - init_args.append(f"self.{prop.client_name} = {prop.client_name}") - return init_args - - @staticmethod - def declare_property(prop: Property) -> str: - if prop.flattened_names: - attribute_key = ".".join(_ModelSerializer.escape_dot(n) for n in prop.flattened_names) - else: - attribute_key = _ModelSerializer.escape_dot(prop.wire_name) - if prop.type.xml_serialization_ctxt: - xml_metadata = f", 'xml': {{{prop.type.xml_serialization_ctxt}}}" - else: - xml_metadata = "" - return ( - f'"{prop.client_name}": {{"key": "{attribute_key}",' - f' "type": "{prop.msrest_deserialization_key}"{xml_metadata}}},' - ) - - -class DpgModelSerializer(_ModelSerializer): - def super_call(self, model: ModelType) -> List[str]: - super_call = f"super().__init__({self.properties_to_pass_to_super(model)})" - if model.flattened_property: - return [ - "_flattened_input = {k: kwargs.pop(k) for k in kwargs.keys() & self.__flattened_items}", - super_call, - "for k, v in _flattened_input.items():", - " setattr(self, k, v)", - ] - return [super_call] - - def imports(self) -> FileImport: - file_import = FileImport(self.code_model) - file_import.add_submodule_import( - "..", - "_model_base", - ImportType.LOCAL, - TypingSection.REGULAR, - ) - - for model in self.code_model.model_types: - file_import.merge(model.imports(is_operation_file=False)) - for prop in model.properties: - file_import.merge(prop.imports()) - if model.is_polymorphic: - file_import.add_submodule_import("typing", "Dict", ImportType.STDLIB) - if not model.internal and self.init_line(model): - file_import.add_submodule_import("typing", "overload", ImportType.STDLIB) - file_import.add_submodule_import("typing", "Mapping", ImportType.STDLIB) - file_import.add_submodule_import("typing", "Any", ImportType.STDLIB) - return file_import - - def declare_model(self, model: ModelType) -> str: - basename = "_model_base.Model" - if model.parents: - basename = ", ".join([m.name for m in model.parents]) - if model.discriminator_value: - basename += f", discriminator='{model.discriminator_value}'" - return f"class {model.name}({basename}):{model.pylint_disable}" - - @staticmethod - def get_properties_to_declare(model: ModelType) -> List[Property]: - if model.parents: - parent_properties = [p for bm in model.parents for p in bm.properties] - properties_to_declare = [ - p - for p in model.properties - if not any( - p.client_name == pp.client_name - and p.type_annotation() == pp.type_annotation() - and not p.is_base_discriminator - for pp in parent_properties - ) - ] - else: - properties_to_declare = model.properties - if any(p for p in properties_to_declare if p.client_name == "_"): - raise ValueError("We do not generate anonymous properties") - return properties_to_declare - - @staticmethod - def declare_property(prop: Property) -> str: - args = [] - if prop.client_name != prop.wire_name or prop.is_discriminator: - args.append(f'name="{prop.wire_name}"') - if prop.visibility: - v_list = ", ".join(f'"{x}"' for x in prop.visibility) - args.append(f"visibility=[{v_list}]") - if prop.client_default_value is not None: - args.append(f"default={prop.client_default_value_declaration}") - - if prop.is_multipart_file_input: - args.append("is_multipart_file_input=True") - elif hasattr(prop.type, "encode") and prop.type.encode: # type: ignore - args.append(f'format="{prop.type.encode}"') # type: ignore - - field = "rest_discriminator" if prop.is_discriminator else "rest_field" - type_ignore = prop.is_discriminator and isinstance(prop.type, (ConstantType, EnumValue)) and prop.type.value - return ( - f"{prop.client_name}: {prop.type_annotation()} =" - f' {field}({", ".join(args)}){" # type: ignore" if type_ignore else ""}' - ) - - def initialize_properties(self, model: ModelType) -> List[str]: - init_args = [] - for prop in self.get_properties_to_declare(model): - if prop.constant and not prop.is_base_discriminator: - init_args.append(f"self.{prop.client_name}: {prop.type_annotation()} = " f"{prop.get_declaration()}") - return init_args - - @staticmethod - def _init_line_parameters(model: ModelType): - return [ - p - for p in model.properties - if p.is_base_discriminator or not p.is_discriminator and not p.constant and p.visibility != ["read"] - ] - - @staticmethod - def properties_to_pass_to_super(model: ModelType) -> str: - properties_to_pass_to_super = ["*args"] - for parent in model.parents: - for prop in model.properties: - if ( - prop.client_name in [prop.client_name for prop in parent.properties if prop.is_base_discriminator] - and prop.is_discriminator - and not prop.constant - and not prop.readonly - ): - properties_to_pass_to_super.append(f"{prop.client_name}={prop.get_declaration()}") - properties_to_pass_to_super.append("**kwargs") - return ", ".join(properties_to_pass_to_super) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/operation_groups_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/operation_groups_serializer.py deleted file mode 100644 index 4c04efbcd2e..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/operation_groups_serializer.py +++ /dev/null @@ -1,89 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Optional, List, Union -import functools -from jinja2 import Environment - -from .utils import get_all_operation_groups_recursively -from ..models import ( - CodeModel, - OperationGroup, - RequestBuilder, - OverloadedRequestBuilder, - Client, - FileImport, -) -from .import_serializer import FileImportSerializer -from .builder_serializer import ( - get_operation_serializer, - RequestBuilderSerializer, -) -from .base_serializer import BaseSerializer - - -class OperationGroupsSerializer(BaseSerializer): - def __init__( - self, - code_model: CodeModel, - clients: List[Client], - env: Environment, - async_mode: bool, - operation_group: Optional[OperationGroup] = None, - ): - super().__init__(code_model, env) - self.clients = clients - self.async_mode = async_mode - self.operation_group = operation_group - - def _get_request_builders( - self, operation_group: OperationGroup - ) -> List[Union[OverloadedRequestBuilder, RequestBuilder]]: - return [ - r - for client in self.clients - for r in client.request_builders - if r.client.name == operation_group.client.name - and r.group_name == operation_group.identify_name - and not r.is_overload - and not r.abstract - and not r.is_lro # lro has already initial builder - ] - - def serialize(self) -> str: - if self.operation_group: - operation_groups = [self.operation_group] - else: - operation_groups = get_all_operation_groups_recursively(self.clients) - - imports = FileImport(self.code_model) - for operation_group in operation_groups: - imports.merge( - operation_group.imports( - async_mode=self.async_mode, - ) - ) - - template = self.env.get_or_select_template("operation_groups_container.py.jinja2") - - return template.render( - code_model=self.code_model, - operation_groups=operation_groups, - imports=FileImportSerializer( - imports, - async_mode=self.async_mode, - ), - async_mode=self.async_mode, - get_operation_serializer=functools.partial( - get_operation_serializer, - code_model=self.code_model, - async_mode=self.async_mode, - ), - request_builder_serializer=RequestBuilderSerializer( - self.code_model, - async_mode=False, - ), - get_request_builders=self._get_request_builders, - ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/operations_init_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/operations_init_serializer.py deleted file mode 100644 index 02232c527c5..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/operations_init_serializer.py +++ /dev/null @@ -1,44 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import List -from jinja2 import Environment - -from ..models.operation_group import OperationGroup -from ..models import CodeModel, Client - - -class OperationsInitSerializer: - def __init__( - self, - code_model: CodeModel, - clients: List[Client], - env: Environment, - async_mode: bool, - ) -> None: - self.code_model = code_model - self.clients = clients - self.env = env - self.async_mode = async_mode - - def operation_group_imports(self) -> List[str]: - def _get_filename(operation_group: OperationGroup) -> str: - return "_operations" if self.code_model.options["combine_operation_files"] else operation_group.filename - - return [ - f"from .{_get_filename(og)} import {og.class_name}" - for client in self.clients - for og in client.operation_groups - ] - - def serialize(self) -> str: - operation_group_init_template = self.env.get_template("operations_folder_init.py.jinja2") - - return operation_group_init_template.render( - code_model=self.code_model, - async_mode=self.async_mode, - operation_group_imports=self.operation_group_imports, - clients=self.clients, - ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/parameter_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/parameter_serializer.py deleted file mode 100644 index 8a41c850a7d..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/parameter_serializer.py +++ /dev/null @@ -1,221 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import List, Sequence, Union, Optional, Dict -from enum import Enum, auto - -from ..models import ( - Parameter, - ParameterLocation, - ListType, - ParameterDelimeter, - RequestBuilderParameter, - ClientParameter, - ConfigParameter, - ParameterType, -) -from ..models.parameter import _ParameterBase - - -class PopKwargType(Enum): - NO = auto() - SIMPLE = auto() - CASE_INSENSITIVE = auto() - - -SPECIAL_HEADER_SERIALIZATION: Dict[str, List[str]] = { - "repeatability-request-id": [ - """if "Repeatability-Request-ID" not in _headers:""", - """ _headers["Repeatability-Request-ID"] = str(uuid.uuid4())""", - ], - "repeatability-first-sent": [ - """if "Repeatability-First-Sent" not in _headers:""", - """ _headers["Repeatability-First-Sent"] = _SERIALIZER.serialize_data(""", - """ datetime.datetime.now(datetime.timezone.utc), "rfc-1123")""", - ], - "client-request-id": [], - "x-ms-client-request-id": [], - "return-client-request-id": [], - "etag": [ - """if_match = prep_if_match(etag, match_condition)""", - """if if_match is not None:""", - """ _headers["If-Match"] = _SERIALIZER.header("if_match", if_match, "str")""", - ], - "match-condition": [ - """if_none_match = prep_if_none_match(etag, match_condition)""", - """if if_none_match is not None:""", - """ _headers["If-None-Match"] = _SERIALIZER.header("if_none_match", if_none_match, "str")""", - ], -} - - -class ParameterSerializer: - @staticmethod - def serialize_parameter(parameter: ParameterType, serializer_name: str) -> str: - optional_parameters = [] - - if parameter.skip_url_encoding: - optional_parameters.append("skip_quote=True") - - if parameter.delimiter and not parameter.explode: - if parameter.delimiter == ParameterDelimeter.COMMA: - div_char = "," - elif parameter.delimiter == ParameterDelimeter.SPACE: - div_char = " " - elif parameter.delimiter == ParameterDelimeter.PIPE: - div_char = "|" - elif parameter.delimiter == ParameterDelimeter.TAB: - div_char = "\t" - else: - raise ValueError(f"We do not support {parameter.delimiter} yet") - optional_parameters.append(f"div='{div_char}'") - - if parameter.explode: - if not isinstance(parameter.type, ListType): - raise ValueError("Got a explode boolean on a non-array schema") - type = parameter.type.element_type - else: - type = parameter.type - - serialization_constraints = type.serialization_constraints - if serialization_constraints: - optional_parameters += serialization_constraints - - origin_name = parameter.full_client_name - - parameters = [ - f'"{origin_name.lstrip("_")}"', - "q" if parameter.explode else origin_name, - f"'{type.serialization_type}'", - *optional_parameters, - ] - parameters_line = ", ".join(parameters) - - msrest_function_name = { - ParameterLocation.PATH: "url", - ParameterLocation.ENDPOINT_PATH: "url", - ParameterLocation.HEADER: "header", - ParameterLocation.QUERY: "query", - }[parameter.location] - - serialize_line = f"{serializer_name}.{msrest_function_name}({parameters_line})" - - if parameter.explode: - return f"[{serialize_line} if q is not None else '' for q in {origin_name}]" - return serialize_line - - @staticmethod - def serialize_path( - parameters: Union[ - List[Parameter], - List[RequestBuilderParameter], - List[ClientParameter], - List[ConfigParameter], - ], - serializer_name: str, - ) -> List[str]: - retval = ["path_format_arguments = {"] - retval.extend( - [ - ' "{}": {},'.format( - path_parameter.wire_name, - ParameterSerializer.serialize_parameter(path_parameter, serializer_name), - ) - for path_parameter in parameters - ] - ) - retval.append("}") - return retval - - @staticmethod - def serialize_query_header( - param: Parameter, - kwarg_name: str, - serializer_name: str, - is_legacy: bool, - ) -> List[str]: - if ( - not is_legacy - and param.location == ParameterLocation.HEADER - and param.wire_name.lower() in SPECIAL_HEADER_SERIALIZATION - ): - return SPECIAL_HEADER_SERIALIZATION[param.wire_name.lower()] - - set_parameter = "_{}['{}'] = {}".format( - kwarg_name, - param.wire_name, - ParameterSerializer.serialize_parameter(param, serializer_name), - ) - if not param.optional: - retval = [set_parameter] - else: - retval = [ - f"if {param.full_client_name} is not None:", - f" {set_parameter}", - ] - return retval - - @staticmethod - def pop_kwargs_from_signature( - parameters: Sequence[_ParameterBase], - check_kwarg_dict: bool, - pop_headers_kwarg: PopKwargType, - pop_params_kwarg: PopKwargType, - check_client_input: bool = False, - operation_name: Optional[str] = None, - ) -> List[str]: - retval = [] - - def append_pop_kwarg(key: str, pop_type: PopKwargType) -> None: - if PopKwargType.CASE_INSENSITIVE == pop_type: - retval.append(f'_{key} = case_insensitive_dict(kwargs.pop("{key}", {{}}) or {{}})') - elif PopKwargType.SIMPLE == pop_type: - retval.append(f'_{key} = kwargs.pop("{key}", {{}}) or {{}}') - - append_pop_kwarg("headers", pop_headers_kwarg) - append_pop_kwarg("params", pop_params_kwarg) - if pop_headers_kwarg != PopKwargType.NO or pop_params_kwarg != PopKwargType.NO: - retval.append("") - for kwarg in parameters: - type_annot = kwarg.type_annotation() - if kwarg.client_default_value is not None or kwarg.optional: - if check_client_input and kwarg.check_client_input: - default_value = f"self._config.{kwarg.client_name}" - else: - default_value = kwarg.client_default_value_declaration - if check_kwarg_dict and (kwarg.location in [ParameterLocation.HEADER, ParameterLocation.QUERY]): - kwarg_dict = "headers" if kwarg.location == ParameterLocation.HEADER else "params" - if ( - kwarg.client_name == "api_version" - and kwarg.code_model.options["multiapi"] - and operation_name is not None - ): - default_value = f"self._api_version{operation_name} or {default_value}" - default_value = f"_{kwarg_dict}.pop('{kwarg.wire_name}', {default_value})" - - retval.append( - f"{kwarg.client_name}: {type_annot} = kwargs.pop('{kwarg.client_name}', " + f"{default_value})" - ) - else: - retval.append(f"{kwarg.client_name}: {type_annot} = kwargs.pop('{kwarg.client_name}')") - return retval - - @staticmethod - def serialize_method( - *, - function_def: str, - method_name: str, - need_self_param: bool, - method_param_signatures: List[str], - pylint_disable: str = "", - ): - lines: List[str] = [] - first_line = f"{function_def} {method_name}({pylint_disable}" - lines.append(first_line) - if need_self_param: - lines.append(" self,") - lines.extend([(" " + line) for line in method_param_signatures]) - lines.append(")") - return "\n".join(lines) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/patch_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/patch_serializer.py deleted file mode 100644 index 8e9c8ef8e7f..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/patch_serializer.py +++ /dev/null @@ -1,19 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from .import_serializer import FileImportSerializer -from ..models import ImportType, FileImport -from .base_serializer import BaseSerializer - - -class PatchSerializer(BaseSerializer): - def serialize(self) -> str: - template = self.env.get_template("patch.py.jinja2") - imports = FileImport(self.code_model) - imports.add_submodule_import("typing", "List", ImportType.STDLIB) - return template.render( - code_model=self.code_model, - imports=FileImportSerializer(imports), - ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/request_builders_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/request_builders_serializer.py deleted file mode 100644 index f9829843646..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/request_builders_serializer.py +++ /dev/null @@ -1,52 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import List -from jinja2 import Environment - -from ..models import FileImport -from .import_serializer import FileImportSerializer -from ..models import CodeModel, RequestBuilderType -from .builder_serializer import RequestBuilderSerializer -from .base_serializer import BaseSerializer - - -class RequestBuildersSerializer(BaseSerializer): - def __init__( - self, - code_model: CodeModel, - env: Environment, - request_builders: List[RequestBuilderType], - ) -> None: - super().__init__(code_model, env) - self.request_builders = request_builders - self.group_name = request_builders[0].group_name - - @property - def imports(self) -> FileImport: - file_import = FileImport(self.code_model) - for request_builder in self.request_builders: - if request_builder.group_name == self.group_name: - file_import.merge(request_builder.imports()) - return file_import - - def serialize_init(self) -> str: - template = self.env.get_template("rest_init.py.jinja2") - return template.render( - code_model=self.code_model, - request_builders=[r for r in self.request_builders if not r.is_overload], - ) - - def serialize_request_builders(self) -> str: - template = self.env.get_template("request_builders.py.jinja2") - - return template.render( - code_model=self.code_model, - request_builders=[rb for rb in self.request_builders if not rb.abstract], - imports=FileImportSerializer( - self.imports, - ), - request_builder_serializer=RequestBuilderSerializer(self.code_model, async_mode=False), - ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/sample_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/sample_serializer.py deleted file mode 100644 index 5694804c687..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/sample_serializer.py +++ /dev/null @@ -1,163 +0,0 @@ -# pylint: disable=too-many-lines -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import logging -from typing import Dict, Any, Union, Tuple -from jinja2 import Environment - -from ..models.operation import OperationBase -from .import_serializer import FileImportSerializer -from .base_serializer import BaseSerializer -from ..models import ( - CodeModel, - KeyCredentialType, - TokenCredentialType, - ImportType, - OperationGroup, - Parameter, - BodyParameter, - FileImport, -) -from .utils import get_namespace_config, get_namespace_from_package_name -from ...utils import to_snake_case - -_LOGGER = logging.getLogger(__name__) - - -class SampleSerializer(BaseSerializer): - def __init__( - self, - code_model: CodeModel, - env: Environment, - operation_group: OperationGroup, - operation: OperationBase[Any], - sample: Dict[str, Any], - file_name: str, - ) -> None: - super().__init__(code_model, env) - self.operation_group = operation_group - self.operation = operation - self.sample = sample - self.file_name = file_name - self.sample_params = {to_snake_case(k): v for k, v in sample.get("parameters", {}).items()} - - def _imports(self) -> FileImportSerializer: - imports = FileImport(self.code_model) - namespace_from_package_name = get_namespace_from_package_name(self.code_model.options["package_name"]) - namespace_config = get_namespace_config(self.code_model.namespace, self.code_model.options["multiapi"]) - namespace = namespace_from_package_name or namespace_config - # mainly for "azure-mgmt-rdbms" - if not self.code_model.options["multiapi"] and namespace_config.count(".") > namespace_from_package_name.count( - "." - ): - namespace = namespace_config - client = self.code_model.clients[0] - imports.add_submodule_import(namespace, client.name, ImportType.LOCAL) - credential_type = getattr(client.credential, "type", None) - if isinstance(credential_type, TokenCredentialType): - imports.add_submodule_import("azure.identity", "DefaultAzureCredential", ImportType.SDKCORE) - elif isinstance(credential_type, KeyCredentialType): - imports.add_import("os", ImportType.STDLIB) - imports.add_submodule_import( - "credentials", - "AzureKeyCredential", - ImportType.SDKCORE, - ) - for param in self.operation.parameters.positional: - if not param.client_default_value and not param.optional and param.client_name in self.sample_params: - imports.merge(param.type.imports_for_sample()) - return FileImportSerializer(imports, True) - - def _client_params(self) -> Dict[str, Any]: - # client params - special_param = {} - credential_type = getattr(self.code_model.clients[0].credential, "type", None) - if isinstance(credential_type, TokenCredentialType): - special_param.update({"credential": "DefaultAzureCredential()"}) - elif isinstance(credential_type, KeyCredentialType): - special_param.update({"credential": 'AzureKeyCredential(key=os.getenv("AZURE_KEY"))'}) - - params_positional = [ - p for p in self.code_model.clients[0].parameters.positional if not (p.optional or p.client_default_value) - ] - client_params = { - p.client_name: special_param.get( - p.client_name, - f'"{self.sample_params.get(p.client_name) or p.client_name.upper()}"', - ) - for p in params_positional - } - - return client_params - - @staticmethod - def handle_param(param: Union[Parameter, BodyParameter], param_value: Any) -> str: - if isinstance(param_value, str): - if any(i in param_value for i in '\r\n"'): - return f'"""{param_value}"""' - - return param.type.serialize_sample_value(param_value) - - # prepare operation parameters - def _operation_params(self) -> Dict[str, Any]: - params_positional = [p for p in self.operation.parameters.positional if not p.client_default_value] - failure_info = "fail to find required param named {}" - operation_params = {} - for param in params_positional: - name = param.client_name - param_value = self.sample_params.get(name) - if not param.optional: - if not param_value: - raise Exception(failure_info.format(name)) # pylint: disable=broad-exception-raised - operation_params[param.client_name] = self.handle_param(param, param_value) - return operation_params - - def _operation_group_name(self) -> str: - if self.operation_group.is_mixin: - return "" - return f".{self.operation_group.property_name}" - - def _operation_result(self) -> Tuple[str, str]: - is_response_none = "None" in self.operation.response_type_annotation(async_mode=False) - lro = ".result()" - if is_response_none: - paging, normal_print, return_var = "", "", "" - else: - paging = "\n for item in response:\n print(item)" - normal_print = "\n print(response)" - return_var = "response = " - - if self.operation.operation_type == "paging": - return paging, return_var - if self.operation.operation_type == "lro": - return lro + normal_print, return_var - if self.operation.operation_type == "lropaging": - return lro + paging, return_var - return normal_print, return_var - - def _operation_name(self) -> str: - return f".{self.operation.name}" - - def _origin_file(self) -> str: - name = self.sample.get("x-ms-original-file", "") - if "specification" in name: - return "specification" + name.split("specification")[-1] - return "" - - def serialize(self) -> str: - operation_result, return_var = self._operation_result() - return self.env.get_template("sample.py.jinja2").render( - code_model=self.code_model, - file_name=self.file_name, - operation_result=operation_result, - operation_params=self._operation_params(), - operation_group_name=self._operation_group_name(), - operation_name=self._operation_name(), - imports=self._imports(), - client_params=self._client_params(), - origin_file=self._origin_file(), - return_var=return_var, - ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/test_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/test_serializer.py deleted file mode 100644 index 5314ff2c768..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/test_serializer.py +++ /dev/null @@ -1,263 +0,0 @@ -# pylint: disable=too-many-lines -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Dict, Any, List, Optional -from jinja2 import Environment - -from .import_serializer import FileImportSerializer -from .base_serializer import BaseSerializer -from ..models import ( - CodeModel, - ImportType, - OperationGroup, - Client, - OperationType, - ModelType, - BaseType, - CombinedType, -) -from .utils import get_namespace_from_package_name, json_dumps_template - - -class TestName: - def __init__(self, client_name: str, *, is_async: bool = False) -> None: - self.client_name = client_name - self.is_async = is_async - - @property - def async_suffix_capt(self) -> str: - return "Async" if self.is_async else "" - - @property - def create_client_name(self) -> str: - return "create_async_client" if self.is_async else "create_client" - - @property - def prefix(self) -> str: - return self.client_name.replace("Client", "") - - @property - def preparer_name(self) -> str: - return self.prefix + "Preparer" - - @property - def base_test_class_name(self) -> str: - return f"{self.client_name}TestBase{self.async_suffix_capt}" - - -class TestCase: - def __init__( - self, - operation_groups: List[OperationGroup], - params: Dict[str, Any], - operation: OperationType, - *, - is_async: bool = False, - ) -> None: - self.operation_groups = operation_groups - self.params = params - self.operation = operation - self.is_async = is_async - - @property - def operation_group_prefix(self) -> str: - if self.operation_groups[-1].is_mixin: - return "" - return "." + ".".join([og.property_name for og in self.operation_groups]) - - @property - def response(self) -> str: - if self.is_async: - if self.operation.operation_type == "lropaging": - return "response = await (await " - return "response = await " - return "response = " - - @property - def lro_comment(self) -> str: - return " # poll until service return final result" - - @property - def operation_suffix(self) -> str: - if self.operation.operation_type == "lropaging": - extra = ")" if self.is_async else "" - return f"{extra}.result(){self.lro_comment}" - return "" - - @property - def extra_operation(self) -> str: - if self.is_async: - if self.operation.operation_type == "lro": - return f"result = await response.result(){self.lro_comment}" - if self.operation.operation_type == ("lropaging", "paging"): - return "result = [r async for r in response]" - else: - if self.operation.operation_type == "lro": - return f"result = response.result(){self.lro_comment}" - if self.operation.operation_type in ("lropaging", "paging"): - return "result = [r for r in response]" - return "" - - -class Test(TestName): - def __init__( - self, - client_name: str, - operation_group: OperationGroup, - testcases: List[TestCase], - test_class_name: str, - *, - is_async: bool = False, - ) -> None: - super().__init__(client_name, is_async=is_async) - self.operation_group = operation_group - self.testcases = testcases - self.test_class_name = test_class_name - - -class TestGeneralSerializer(BaseSerializer): - def __init__(self, code_model: CodeModel, env: Environment, *, is_async: bool = False) -> None: - super().__init__(code_model, env) - self.is_async = is_async - - @property - def aio_str(self) -> str: - return ".aio" if self.is_async else "" - - @property - def test_names(self) -> List[TestName]: - return [TestName(c.name, is_async=self.is_async) for c in self.code_model.clients] - - @property - def import_clients(self) -> FileImportSerializer: - imports = self.init_file_import() - namespace = get_namespace_from_package_name(self.code_model.options["package_name"]) - - imports.add_submodule_import("devtools_testutils", "AzureRecordedTestCase", ImportType.STDLIB) - if not self.is_async: - imports.add_import("functools", ImportType.STDLIB) - imports.add_submodule_import("devtools_testutils", "PowerShellPreparer", ImportType.STDLIB) - for client in self.code_model.clients: - imports.add_submodule_import(namespace + self.aio_str, client.name, ImportType.STDLIB) - return FileImportSerializer(imports, self.is_async) - - def serialize_conftest(self) -> str: - return self.env.get_template("conftest.py.jinja2").render( - test_names=self.test_names, - code_model=self.code_model, - ) - - def serialize_testpreparer(self) -> str: - return self.env.get_template("testpreparer.py.jinja2").render( - test_names=self.test_names, - imports=self.import_clients, - code_model=self.code_model, - ) - - -class TestSerializer(TestGeneralSerializer): - def __init__( - self, - code_model: CodeModel, - env: Environment, - *, - client: Client, - operation_group: OperationGroup, - is_async: bool = False, - ) -> None: - super().__init__(code_model, env, is_async=is_async) - self.client = client - self.operation_group = operation_group - - @property - def import_test(self) -> FileImportSerializer: - imports = self.init_file_import() - test_name = TestName(self.client.name, is_async=self.is_async) - async_suffix = "_async" if self.is_async else "" - imports.add_submodule_import( - "testpreparer" + async_suffix, - test_name.base_test_class_name, - ImportType.LOCAL, - ) - imports.add_submodule_import("testpreparer", test_name.preparer_name, ImportType.LOCAL) - imports.add_submodule_import( - "devtools_testutils" + self.aio_str, - "recorded_by_proxy" + async_suffix, - ImportType.LOCAL, - ) - return FileImportSerializer(imports, self.is_async) - - @property - def breadth_search_operation_group(self) -> List[List[OperationGroup]]: - result = [] - queue = [[self.operation_group]] - while queue: - current = queue.pop(0) - if current[-1].operations: - result.append(current) - if current[-1].operation_groups: - queue.extend([current + [og] for og in current[-1].operation_groups]) - return result - - def get_sub_type(self, param_type: ModelType) -> ModelType: - if param_type.discriminated_subtypes: - for item in param_type.discriminated_subtypes.values(): - return self.get_sub_type(item) - return param_type - - def get_model_type(self, param_type: BaseType) -> Optional[ModelType]: - if isinstance(param_type, ModelType): - return param_type - if isinstance(param_type, CombinedType): - return param_type.target_model_subtype((ModelType,)) - return None - - def get_operation_params(self, operation: OperationType) -> Dict[str, Any]: - operation_params = {} - required_params = [p for p in operation.parameters.method if not p.optional] - for param in required_params: - model_type = self.get_model_type(param.type) - param_type = self.get_sub_type(model_type) if model_type else param.type - operation_params[param.client_name] = json_dumps_template(param_type.get_json_template_representation()) - return operation_params - - def get_test(self) -> Test: - testcases = [] - for operation_groups in self.breadth_search_operation_group: - for operation in operation_groups[-1].operations: - if operation.internal or operation.is_lro_initial_operation: - continue - operation_params = self.get_operation_params(operation) - testcase = TestCase( - operation_groups=operation_groups, - params=operation_params, - operation=operation, - is_async=self.is_async, - ) - testcases.append(testcase) - if not testcases: - raise Exception("no public operation to test") # pylint: disable=broad-exception-raised - - return Test( - client_name=self.client.name, - operation_group=self.operation_group, - testcases=testcases, - test_class_name=self.test_class_name, - is_async=self.is_async, - ) - - @property - def test_class_name(self) -> str: - test_name = TestName(self.client.name, is_async=self.is_async) - class_name = "" if self.operation_group.is_mixin else self.operation_group.class_name - return f"Test{test_name.prefix}{class_name}{test_name.async_suffix_capt}" - - def serialize_test(self) -> str: - return self.env.get_template("test.py.jinja2").render( - imports=self.import_test, - code_model=self.code_model, - test=self.get_test(), - ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/types_serializer.py b/packages/autorest.python/generator/pygen/codegen/serializers/types_serializer.py deleted file mode 100644 index 3e143f5209e..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/types_serializer.py +++ /dev/null @@ -1,31 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from ..models.imports import FileImport, ImportType -from .import_serializer import FileImportSerializer -from .base_serializer import BaseSerializer - - -class TypesSerializer(BaseSerializer): - def imports(self) -> FileImport: - file_import = FileImport(self.code_model) - if self.code_model.named_unions: - file_import.add_submodule_import( - "typing", - "Union", - ImportType.STDLIB, - ) - for nu in self.code_model.named_unions: - file_import.merge(nu.imports(relative_path=".", model_typing=True, is_types_file=True)) - return file_import - - def serialize(self) -> str: - # Generate the models - template = self.env.get_template("types.py.jinja2") - return template.render( - code_model=self.code_model, - imports=FileImportSerializer(self.imports()), - serializer=self, - ) diff --git a/packages/autorest.python/generator/pygen/codegen/serializers/utils.py b/packages/autorest.python/generator/pygen/codegen/serializers/utils.py deleted file mode 100644 index 7cd6f7c9267..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/serializers/utils.py +++ /dev/null @@ -1,68 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import json -from typing import Optional, List, Any -from pathlib import Path - -from ..models import Client, OperationGroup - - -def method_signature_and_response_type_annotation_template( - *, - method_signature: str, - response_type_annotation: str, -) -> str: - return f"{method_signature} -> {response_type_annotation}:" - - -def extract_sample_name(file_path: str) -> str: - file = file_path.split("specification")[-1] - return Path(file).parts[-1].replace(".json", "") - - -def strip_end(namespace: str) -> str: - return ".".join(namespace.split(".")[:-1]) - - -def get_namespace_config(namespace: str, multiapi: bool) -> str: - return strip_end(namespace) if multiapi else namespace - - -def get_namespace_from_package_name(package_name: Optional[str]) -> str: - return (package_name or "").replace("-", ".") - - -def get_all_operation_groups_recursively(clients: List[Client]) -> List[OperationGroup]: - operation_groups = [] - queue = [] - for client in clients: - queue.extend(client.operation_groups) - while queue: - operation_groups.append(queue.pop(0)) - if operation_groups[-1].operation_groups: - queue.extend(operation_groups[-1].operation_groups) - return operation_groups - - -def _improve_json_string(template_representation: str) -> Any: - origin = template_representation.split("\n") - final = [] - for line in origin: - idx0 = line.find("#") - idx1 = line.rfind('"') - modified_line = "" - if idx0 > -1 and idx1 > -1: - modified_line = line[:idx0] + line[idx1:] + " " + line[idx0:idx1] + "\n" - else: - modified_line = line + "\n" - modified_line = modified_line.replace('"', "").replace("\\", '"') - final.append(modified_line) - return "".join(final) - - -def json_dumps_template(template_representation: Any) -> Any: - # only for template use, since it wraps everything in strings - return _improve_json_string(json.dumps(template_representation, indent=4)) diff --git a/packages/autorest.python/generator/pygen/codegen/templates/client.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/client.py.jinja2 deleted file mode 100644 index 57e27f1f90f..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/client.py.jinja2 +++ /dev/null @@ -1,37 +0,0 @@ -{{ serializer.class_definition }} - """{{ op_tools.wrap_string(client.description, "\n") | indent }} - - {{ op_tools.serialize_with_wrap(serializer.property_descriptions(async_mode), "\n ") | indent }} - {{ serializer.init_signature_and_response_type_annotation(async_mode) | indent }} - {% if serializer.should_init_super %} - super().__init__() - {% endif %} - {% if client.has_parameterized_host %} - {{ serializer.host_variable_name }} = {{ keywords.escape_str(client.url) }}{{ client.url_pylint_disable }} - {% endif %} - {{ serializer.initialize_config() }} - {{ op_tools.serialize(serializer.initialize_pipeline_client(async_mode)) | indent(8) }} - - {{ op_tools.serialize(serializer.serializers_and_operation_groups_properties()) | indent(8) }} - - {% set http_response = keywords.async_class + "HttpResponse" %} - {{ serializer.send_request_signature_and_response_type_annotation(async_mode) | indent }} - {{ op_tools.serialize(serializer.send_request_description(async_mode)) | indent(8) }} - request_copy = deepcopy(request) - {% if client.parameters.path %} - {{ op_tools.serialize(serializer.serialize_path()) | indent(8) }} - request_copy.url = self._client.format_url(request_copy.url, **path_format_arguments) - {% else %} - request_copy.url = self._client.format_url(request_copy.url) - {% endif %} - return self._client.send_request(request_copy, stream=stream, **kwargs) # type: ignore - - {{ keywords.def }} close(self) -> None: - {{ keywords.await }}self._client.close() - - {{ keywords.def }} __{{ keywords.async_prefix }}enter__(self){{ " -> \"" + client.name + "\"" }}: - {{ keywords.await }}self._client.__{{ keywords.async_prefix }}enter__() - return self - - {{ keywords.def }} __{{ keywords.async_prefix }}exit__(self, *exc_details: Any) -> None: - {{ keywords.await }}self._client.__{{ keywords.async_prefix }}exit__(*exc_details) diff --git a/packages/autorest.python/generator/pygen/codegen/templates/client_container.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/client_container.py.jinja2 deleted file mode 100644 index 33de339affe..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/client_container.py.jinja2 +++ /dev/null @@ -1,12 +0,0 @@ -{% import 'keywords.jinja2' as keywords with context %} -{% import 'operation_tools.jinja2' as op_tools %} -{# actual template starts here #} -# coding=utf-8 -{{ code_model.options['license_header'] }} - -{{ imports }} - -{% for client in clients %} - {% set serializer = get_serializer(client) %} -{% include "client.py.jinja2" %} -{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/config.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/config.py.jinja2 deleted file mode 100644 index b267c8e97d8..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/config.py.jinja2 +++ /dev/null @@ -1,73 +0,0 @@ -class {{ client.name }}Configuration: {{ client.config.pylint_disable }} - """Configuration for {{ client.name }}. - - Note that all parameters used to create this instance are saved as instance - attributes. -{% if client.config.parameters.method | first %} - -{% endif %} - {{ op_tools.serialize_with_wrap(serializer.property_descriptions(async_mode), "\n ") | indent }} - {{ serializer.init_signature_and_response_type_annotation(async_mode) | indent }} - {% if client.config.parameters.kwargs_to_pop %} - {{ op_tools.serialize(serializer.pop_kwargs_from_signature()) | indent(8) }} - {% endif %} -{% if serializer.check_required_parameters() %} - {{ op_tools.serialize(serializer.check_required_parameters()) | indent(8) -}} -{% endif %} - -{% for parameter in client.config.parameters.method %} - self.{{ parameter.client_name }} = {{ parameter.client_name }} -{% endfor %} -{% if serializer.set_constants() %} - {{ op_tools.serialize(serializer.set_constants()) | indent(8) -}} -{% endif %} -{% if client.credential %} - {% set cred_scopes = client.credential.type if client.credential.type.policy is defined and client.credential.type.policy.credential_scopes is defined %} - {% if not cred_scopes %} - {% set cred_scopes = client.credential.type.types | selectattr("policy.credential_scopes") | first if client.credential.type.types is defined %} - {% endif %} - {% if cred_scopes %} - self.credential_scopes = kwargs.pop('credential_scopes', {{ cred_scopes.policy.credential_scopes }}) - {% endif %} -{% endif %} - kwargs.setdefault('sdk_moniker', '{{ client.config.sdk_moniker }}/{}'.format(VERSION)) - self.polling_interval = kwargs.get("polling_interval", 30) - self._configure(**kwargs) - -{% if client.credential and client.credential.type.types is defined %} - def _infer_policy(self, **kwargs): - {% for cred_type in client.credential.type.types %} - if {{ cred_type.instance_check_template.format("self.credential") }}: - return {{ cred_type.policy.call(async_mode) }} - {% endfor %} - raise TypeError(f"Unsupported credential: {self.credential}") -{% endif %} - - def _configure( - self, - **kwargs: Any - ) -> None: - self.user_agent_policy = kwargs.get('user_agent_policy') or policies.UserAgentPolicy(**kwargs) - self.headers_policy = kwargs.get('headers_policy') or policies.HeadersPolicy(**kwargs) - self.proxy_policy = kwargs.get('proxy_policy') or policies.ProxyPolicy(**kwargs) - self.logging_policy = kwargs.get('logging_policy') or policies.NetworkTraceLoggingPolicy(**kwargs) - {% if code_model.is_azure_flavor %} - self.http_logging_policy = kwargs.get('http_logging_policy') or {{ "ARM" if client.code_model.options['azure_arm'] else "policies." }}HttpLoggingPolicy(**kwargs) - self.custom_hook_policy = kwargs.get('custom_hook_policy') or policies.CustomHookPolicy(**kwargs) - self.redirect_policy = kwargs.get('redirect_policy') or policies.{{ keywords.async_class }}RedirectPolicy(**kwargs) - {% endif %} - self.retry_policy = kwargs.get('retry_policy') or policies.{{ keywords.async_class }}RetryPolicy(**kwargs) - self.authentication_policy = kwargs.get('authentication_policy') - {% if client.credential and client.credential.type.policy is defined %} - {# only adding this if credential_scopes is not passed during code generation #} - {% if client.credential.type.policy.credential_scopes is defined and client.credential.type.policy.credential_scopes | length == 0 %} - if not self.credential_scopes and not self.authentication_policy: - raise ValueError("You must provide either credential_scopes or authentication_policy as kwargs") - {% endif %} - if self.credential and not self.authentication_policy: - self.authentication_policy = {{ client.credential.type.policy.call(async_mode) }} - {% endif %} - {% if client.credential and client.credential.type.types is defined %} - if self.credential and not self.authentication_policy: - self.authentication_policy = self._infer_policy(**kwargs) - {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/config_container.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/config_container.py.jinja2 deleted file mode 100644 index 9a3c263e2ff..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/config_container.py.jinja2 +++ /dev/null @@ -1,16 +0,0 @@ -{% import 'keywords.jinja2' as keywords with context %} -{% import 'operation_tools.jinja2' as op_tools %} -{# actual template starts here #} -# coding=utf-8 -{{ code_model.options['license_header'] }} - -{{ imports }} - -{% if not code_model.options['package_version'] %} -VERSION = "unknown" -{% endif %} - -{% for client in clients %} - {% set serializer = get_serializer(client) %} -{% include "config.py.jinja2" %} -{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/conftest.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/conftest.py.jinja2 deleted file mode 100644 index 3d56b93351d..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/conftest.py.jinja2 +++ /dev/null @@ -1,28 +0,0 @@ -# coding=utf-8 -{{ code_model.options['license_header'] }} -import os -import pytest -from dotenv import load_dotenv -from devtools_testutils import test_proxy, add_general_regex_sanitizer, add_body_key_sanitizer, add_header_regex_sanitizer - -load_dotenv() - -# aovid record sensitive identity information in recordings -@pytest.fixture(scope="session", autouse=True) -def add_sanitizers(test_proxy): - {% for test_name in test_names %} - {% set prefix_upper = test_name.prefix|upper %} - {% set prefix_lower = test_name.prefix|lower %} - {{ prefix_lower }}_subscription_id = os.environ.get("{{ prefix_upper }}_SUBSCRIPTION_ID", "00000000-0000-0000-0000-000000000000") - {{ prefix_lower }}_tenant_id = os.environ.get("{{ prefix_upper }}_TENANT_ID", "00000000-0000-0000-0000-000000000000") - {{ prefix_lower }}_client_id = os.environ.get("{{ prefix_upper }}_CLIENT_ID", "00000000-0000-0000-0000-000000000000") - {{ prefix_lower }}_client_secret = os.environ.get("{{ prefix_upper }}_CLIENT_SECRET", "00000000-0000-0000-0000-000000000000") - add_general_regex_sanitizer(regex={{ prefix_lower }}_subscription_id, value="00000000-0000-0000-0000-000000000000") - add_general_regex_sanitizer(regex={{ prefix_lower }}_tenant_id, value="00000000-0000-0000-0000-000000000000") - add_general_regex_sanitizer(regex={{ prefix_lower }}_client_id, value="00000000-0000-0000-0000-000000000000") - add_general_regex_sanitizer(regex={{ prefix_lower }}_client_secret, value="00000000-0000-0000-0000-000000000000") - - {% endfor %} - add_header_regex_sanitizer(key="Set-Cookie", value="[set-cookie;]") - add_header_regex_sanitizer(key="Cookie", value="cookie;") - add_body_key_sanitizer(json_path="$..access_token", value="access_token") diff --git a/packages/autorest.python/generator/pygen/codegen/templates/enum.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/enum.py.jinja2 deleted file mode 100644 index 047d64c30d6..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/enum.py.jinja2 +++ /dev/null @@ -1,13 +0,0 @@ - -class {{ enum.name }}({{ enum.value_type.type_annotation(is_operation_file=False) }}, Enum, metaclass=CaseInsensitiveEnumMeta): - {% if enum.yaml_data.get("description") %} - """{{ op_tools.wrap_string(enum.yaml_data["description"], "\n ") }} - """ - {% endif %} - - {% for value in enum.values %} - {{ value.name }} = {{ enum.value_type.get_declaration(value.value) }} - {% if value.description(is_operation_file=False) %} - """{{ op_tools.wrap_string(value.description(is_operation_file=False), "\n ") }}""" - {% endif %} - {% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/enum_container.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/enum_container.py.jinja2 deleted file mode 100644 index 099fbb072a6..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/enum_container.py.jinja2 +++ /dev/null @@ -1,10 +0,0 @@ -{% import 'operation_tools.jinja2' as op_tools %} -# coding=utf-8 -{{ code_model.options['license_header'] }} - -from enum import Enum -from {{ code_model.core_library }}{{ "" if code_model.is_azure_flavor else ".utils" }} import CaseInsensitiveEnumMeta - -{% for enum in code_model.enums | sort %} -{% include "enum.py.jinja2" %} -{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/init.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/init.py.jinja2 deleted file mode 100644 index 9f70f40bd6b..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/init.py.jinja2 +++ /dev/null @@ -1,24 +0,0 @@ -{% import 'keywords.jinja2' as keywords %} -# coding=utf-8 -{{ code_model.options['license_header'] }} - -{% if clients %} - {% for client in clients %} -from .{{ client.filename }} import {{ client.name }} - {% endfor %} -{% endif %} -{% if not async_mode and code_model.options['package_version']%} -from ._version import VERSION - -__version__ = VERSION -{% endif %} - -{{ keywords.patch_imports(try_except=True) }} -__all__ = [ - {% for client in clients %} - {{ keywords.escape_str(client.name) }}, - {% endfor %} -] -{{ keywords.extend_all }} - -_patch_sdk() diff --git a/packages/autorest.python/generator/pygen/codegen/templates/keywords.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/keywords.jinja2 deleted file mode 100644 index 6ee92a41416..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/keywords.jinja2 +++ /dev/null @@ -1,19 +0,0 @@ -{% set def = "async def" if async_mode else "def" %} -{% set async_prefix = "a" if async_mode else "" %} -{% set await = "await " if async_mode else "" %} -{% set async_class = "Async" if async_mode else "" %} -{% macro escape_str(s) %}'{{ s|replace("'", "\\'") }}'{% endmacro %} -{% set kwargs_declaration = "**kwargs: Any" %} -{% set extend_all = "__all__.extend([p for p in _patch_all if p not in __all__])" %} -{% macro patch_imports(try_except=False) %} -{% set indentation = " " if try_except else "" %} -{% if try_except %} -try: -{% endif %} -{{ indentation }}from ._patch import __all__ as _patch_all -{{ indentation }}from ._patch import * # pylint: disable=unused-wildcard-import -{% if try_except %} -except ImportError: - _patch_all = [] -{% endif %} -from ._patch import patch_sdk as _patch_sdk{% endmacro %} \ No newline at end of file diff --git a/packages/autorest.python/generator/pygen/codegen/templates/lro_operation.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/lro_operation.py.jinja2 deleted file mode 100644 index bca9dcf2e4a..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/lro_operation.py.jinja2 +++ /dev/null @@ -1,16 +0,0 @@ -{% import 'operation_tools.jinja2' as op_tools with context %} -{# actual template starts here #} -{% if operation.overloads and operation.include_documentation %} -{{ op_tools.generate_overloads(operation_serializer, operation) }} -{% endif %} -{{ operation_serializer.method_signature_and_response_type_annotation(operation) }} - {{ op_tools.description(operation, operation_serializer) | indent -}} - {% if not operation.abstract %} - {% if operation_serializer.pop_kwargs_from_signature(operation) %} - {{ op_tools.serialize(operation_serializer.pop_kwargs_from_signature(operation)) | indent }} - {%- endif %} - {{ op_tools.serialize(operation_serializer.initial_call(operation)) | indent }} - {{ op_tools.serialize(operation_serializer.get_long_running_output(operation)) | indent }} - - {{ op_tools.serialize(operation_serializer.return_lro_poller(operation)) | indent }} - {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 deleted file mode 100644 index 09b50a00d42..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 +++ /dev/null @@ -1,18 +0,0 @@ -{% import 'operation_tools.jinja2' as op_tools with context %} -{% import 'keywords.jinja2' as keywords with context %} -{# actual template starts here #} -{% if operation.overloads and operation.include_documentation %} -{{ op_tools.generate_overloads(operation_serializer, operation) }} -{% endif %} -{{ operation_serializer.method_signature_and_response_type_annotation(operation) }} - {{ op_tools.description(operation, operation_serializer) | indent }} - {% if not operation.abstract %} - {% if operation_serializer.pop_kwargs_from_signature(operation) %} - {{ op_tools.serialize(operation_serializer.pop_kwargs_from_signature(operation)) | indent }} - {% endif %} - {{ op_tools.serialize(operation_serializer.set_up_params_for_pager(operation)) | indent }} - - {{ op_tools.serialize(operation_serializer.initial_call(operation)) | indent }} - {{ op_tools.serialize(operation_serializer.get_long_running_output(operation)) | indent }} - {{ op_tools.serialize(operation_serializer.return_lro_poller(operation)) | indent }} - {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/macros.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/macros.jinja2 deleted file mode 100644 index 64981bedb86..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/macros.jinja2 +++ /dev/null @@ -1,12 +0,0 @@ -{% macro wrap_model_string(doc_string, wrap_string, suffix_string="") %} -{% set orignal_result = doc_string | wordwrap(width=95, break_long_words=False, break_on_hyphens=False, wrapstring=wrap_string) %} -{% set list_result = orignal_result.split('\n') %} -{% for line in list_result %} - {% set suffix = suffix_string if list_result | length == loop.index %} - {% if line | length > 120 %} -{{ line + " # pylint: disable=line-too-long" }}{{ suffix }} - {% else %} -{{ line }}{{ suffix }} - {% endif %} -{% endfor %} -{% endmacro %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/metadata.json.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/metadata.json.jinja2 deleted file mode 100644 index 946baa79170..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/metadata.json.jinja2 +++ /dev/null @@ -1,167 +0,0 @@ -{% import 'operation_tools.jinja2' as op_tools %} -{% import 'keywords.jinja2' as keywords %} -{ - "chosen_version": {{ chosen_version | tojson }}, - "total_api_version_list": {{ total_api_version_list | tojson }}, - "client": { - "name": {{ client.name | tojson }}, - "filename": {{ ("_" + client.legacy_filename) | tojson }}, - "description": {{ client.description | tojson }}, - "host_value": {{ (client.parameters.host.client_default_value_declaration if not client.has_parameterized_host else None) | tojson }}, - "parameterized_host_template": {{ (keywords.escape_str(client.url) if client.has_parameterized_host else None) | tojson }}, - "azure_arm": {{ client.code_model.options["azure_arm"] | tojson }}, - "has_public_lro_operations": {{ client.has_public_lro_operations | tojson }}, - "client_side_validation": {{ client.code_model.options["client_side_validation"] | tojson }}, - "sync_imports": {{ sync_client_imports | tojson }}, - "async_imports": {{ async_client_imports | tojson }} - }, - "global_parameters": { - "sync": { - {% for gp in global_parameters.method | rejectattr("client_name", "equalto", "api_version") | rejectattr("is_host") %} - {{ gp.client_name | tojson }}: { - "signature": {{ gp.method_signature(async_mode=False) | tojson }}, - "description": {{ gp.description | tojson }}, - "docstring_type": {{ gp.docstring_type(async_mode=False) | tojson }}, - "required": {{ (not gp.optional) | tojson }}, - "method_location": {{ gp.method_location | tojson }} - }{{ "," if not loop.last else "" }} - {% endfor %} - }, - "async": { - {% for gp in global_parameters.method | rejectattr("client_name", "equalto", "api_version") | rejectattr("is_host") %} - {{ gp.client_name | tojson }}: { - "signature": {{ (gp.method_signature(async_mode=True)) | tojson }}, - "description": {{ gp.description | tojson }}, - "docstring_type": {{ gp.docstring_type(async_mode=True) | tojson }}, - "required": {{ (not gp.optional) | tojson }} - }{{ "," if not loop.last else "" }} - {% endfor %} - }, - "constant": { - {% for gp in client.parameters.constant | rejectattr("client_name", "equalto", "api_version") %} - {{ gp.client_name | tojson }}: {{ gp.constant_declaration | tojson }}{{ "," if not loop.last else "" }} - {% endfor %} - }, - "call": {{ client.parameters.method | rejectattr("client_name", "equalto", "api_version") | rejectattr("is_host") | map(attribute="client_name") | join(', ') | tojson }}, - "service_client_specific": { - "sync": { - "api_version": { - "signature": "api_version: Optional[str]=None,", - "description": "API version to use if no profile is provided, or if missing in profile.", - "docstring_type": "str", - "required": false, - "method_location": "positional" - }, - {% if not client.has_parameterized_host %} - "base_url": { - "signature": {{ client.parameters.host.method_signature(async_mode=False) | tojson }}, - "description": "Service URL", - "docstring_type": "str", - "required": false, - "method_location": "positional" - }, - {% endif %} - "profile": { - "signature": "profile: KnownProfiles=KnownProfiles.default,", - "description": "A profile definition, from KnownProfiles to dict.", - "docstring_type": "azure.profiles.KnownProfiles", - "required": false, - "method_location": "positional" - } - }, - "async": { - "api_version": { - "signature": "api_version: Optional[str] = None,", - "description": "API version to use if no profile is provided, or if missing in profile.", - "docstring_type": "str", - "required": false, - "method_location": "positional" - }, - {% if not client.has_parameterized_host %} - "base_url": { - "signature": {{ client.parameters.host.method_signature(async_mode=True) | tojson }}, - "description": "Service URL", - "docstring_type": "str", - "required": false, - "method_location": "positional" - }, - {% endif %} - "profile": { - "signature": "profile: KnownProfiles = KnownProfiles.default,", - "description": "A profile definition, from KnownProfiles to dict.", - "docstring_type": "azure.profiles.KnownProfiles", - "required": false, - "method_location": "positional" - } - } - } - }, - "config": { - "credential": {{ has_credential | tojson }}, - "credential_scopes": {{ (client.credential.type.policy.credential_scopes if has_credential and client.credential.type.policy.credential_scopes is defined else None)| tojson}}, - "credential_call_sync": {{ (client.credential.type.policy.call(async_mode=False) if has_credential else None) | tojson }}, - "credential_call_async": {{ (client.credential.type.policy.call(async_mode=True) if has_credential else None) | tojson }}, - "sync_imports": {{ sync_config_imports | tojson }}, - "async_imports": {{ async_config_imports | tojson }} - }, - "operation_groups": { - {% for operation_group in client.operation_groups | rejectattr('is_mixin') %} - {{ operation_group.property_name | tojson }}: {{ operation_group.class_name | tojson }}{{ "," if not loop.last else "" }} - {% endfor %} - }{{ "," if mixin_operations }} - {% if mixin_operations %} - "operation_mixins": { - "sync_imports": {{ str(sync_mixin_imports) | tojson }}, - "async_imports": {{ str(async_mixin_imports) | tojson }}, - "sync_mixin_typing_definitions": {{ str(sync_mixin_typing_definitions) | tojson }}, - "async_mixin_typing_definitions": {{ str(async_mixin_typing_definitions) | tojson }}, - "operations": { - {% for operation in mixin_operations %} - {{ operation.name | tojson }} : { - {% set request_builder = operation.request_builder %} - "sync": { - {% set operation_serializer = get_sync_operation_serializer(operation) %} - {% if is_lro(operation) and is_paging(operation) %} - {% from "lro_paging_operation.py.jinja2" import operation_docstring with context %} - {% set sync_return_type_wrapper = [operation.get_poller(async_mode=False), operation.get_pager(async_mode=False)] %} - {% elif is_lro(operation) %} - {% from "lro_operation.py.jinja2" import operation_docstring with context %} - {% set sync_return_type_wrapper = [operation.get_poller(async_mode=False)] %} - {% elif is_paging(operation) %} - {% from "paging_operation.py.jinja2" import operation_docstring with context %} - {% set sync_return_type_wrapper = [operation.get_pager(async_mode=False)] %} - {% else %} - {% from "operation.py.jinja2" import operation_docstring with context %} - {% set sync_return_type_wrapper = "" %} - {% endif %} - "signature": {{ (operation_serializer.method_signature_and_response_type_annotation(operation, want_decorators=False) + "\n") | tojson }}, - "doc": {{ op_tools.description(operation, operation_serializer).rstrip("\n") | tojson }}, - "call": {{ operation.parameters.call | join(', ') | tojson }} - }, - "async": { - {% set coroutine = False if (is_paging(operation) and not is_lro(operation)) else True %} - {% set operation_serializer = get_async_operation_serializer(operation) %} - "coroutine": {{ coroutine | tojson }}, - {% if is_lro(operation) and is_paging(operation) %} - {% from "lro_paging_operation.py.jinja2" import operation_docstring with context %} - {% set async_return_type_wrapper = [operation.get_poller(async_mode=True), operation.get_pager(async_mode=True)] %} - {% elif is_lro(operation) %} - {% from "lro_operation.py.jinja2" import operation_docstring with context %} - {% set async_return_type_wrapper = [operation.get_poller(async_mode=True)] %} - {% elif is_paging(operation) %} - {% from "paging_operation.py.jinja2" import operation_docstring with context %} - {% set async_return_type_wrapper = [operation.get_pager(async_mode=True)] %} - {% else %} - {% from "operation.py.jinja2" import operation_docstring with context %} - {% set async_return_type_wrapper = "" %} - {% endif %} - "signature": {{ (operation_serializer.method_signature_and_response_type_annotation(operation, want_decorators=False) + "\n") | tojson }}, - "doc": {{ op_tools.description(operation, operation_serializer).rstrip("\n") | tojson }}, - "call": {{ operation.parameters.call | join(', ') | tojson }} - } - }{{ "," if not loop.last else "" }} - {% endfor %} - } - } - {% endif %} -} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/model_base.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/model_base.py.jinja2 deleted file mode 100644 index 386a0a2e2eb..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/model_base.py.jinja2 +++ /dev/null @@ -1,898 +0,0 @@ -# coding=utf-8 -# -------------------------------------------------------------------------- -# Copyright (c) {{ code_model.options["company_name"] }} Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -# pylint: disable=protected-access, arguments-differ, signature-differs, broad-except - -import copy -import calendar -import decimal -import functools -import sys -import logging -import base64 -import re -import typing -import enum -import email.utils -from datetime import datetime, date, time, timedelta, timezone -from json import JSONEncoder -from typing_extensions import Self -import isodate -from {{ code_model.core_library }}.exceptions import DeserializationError -from {{ code_model.core_library }}{{ "" if code_model.is_azure_flavor else ".utils" }} import CaseInsensitiveEnumMeta -from {{ code_model.core_library }}.{{ "" if code_model.is_azure_flavor else "runtime." }}pipeline import PipelineResponse -from {{ code_model.core_library }}.serialization import _Null - -if sys.version_info >= (3, 9): - from collections.abc import MutableMapping -else: - from typing import MutableMapping - -_LOGGER = logging.getLogger(__name__) - -__all__ = ["SdkJSONEncoder", "Model", "rest_field", "rest_discriminator"] - -TZ_UTC = timezone.utc -_T = typing.TypeVar("_T") - - -def _timedelta_as_isostr(td: timedelta) -> str: - """Converts a datetime.timedelta object into an ISO 8601 formatted string, e.g. 'P4DT12H30M05S' - - Function adapted from the Tin Can Python project: https://github.com/RusticiSoftware/TinCanPython - - :param timedelta td: The timedelta to convert - :rtype: str - :return: ISO8601 version of this timedelta - """ - - # Split seconds to larger units - seconds = td.total_seconds() - minutes, seconds = divmod(seconds, 60) - hours, minutes = divmod(minutes, 60) - days, hours = divmod(hours, 24) - - days, hours, minutes = list(map(int, (days, hours, minutes))) - seconds = round(seconds, 6) - - # Build date - date_str = "" - if days: - date_str = "%sD" % days - - if hours or minutes or seconds: - # Build time - time_str = "T" - - # Hours - bigger_exists = date_str or hours - if bigger_exists: - time_str += "{:02}H".format(hours) - - # Minutes - bigger_exists = bigger_exists or minutes - if bigger_exists: - time_str += "{:02}M".format(minutes) - - # Seconds - try: - if seconds.is_integer(): - seconds_string = "{:02}".format(int(seconds)) - else: - # 9 chars long w/ leading 0, 6 digits after decimal - seconds_string = "%09.6f" % seconds - # Remove trailing zeros - seconds_string = seconds_string.rstrip("0") - except AttributeError: # int.is_integer() raises - seconds_string = "{:02}".format(seconds) - - time_str += "{}S".format(seconds_string) - else: - time_str = "" - - return "P" + date_str + time_str - - -def _serialize_bytes(o, format: typing.Optional[str] = None) -> str: - encoded = base64.b64encode(o).decode() - if format == "base64url": - return encoded.strip("=").replace("+", "-").replace("/", "_") - return encoded - - -def _serialize_datetime(o, format: typing.Optional[str] = None): - if hasattr(o, "year") and hasattr(o, "hour"): - if format == "rfc7231": - return email.utils.format_datetime(o, usegmt=True) - if format == "unix-timestamp": - return int(calendar.timegm(o.utctimetuple())) - - # astimezone() fails for naive times in Python 2.7, so make make sure o is aware (tzinfo is set) - if not o.tzinfo: - iso_formatted = o.replace(tzinfo=TZ_UTC).isoformat() - else: - iso_formatted = o.astimezone(TZ_UTC).isoformat() - # Replace the trailing "+00:00" UTC offset with "Z" (RFC 3339: https://www.ietf.org/rfc/rfc3339.txt) - return iso_formatted.replace("+00:00", "Z") - # Next try datetime.date or datetime.time - return o.isoformat() - - -def _is_readonly(p): - try: - return p._visibility == ["read"] # pylint: disable=protected-access - except AttributeError: - return False - - -class SdkJSONEncoder(JSONEncoder): - """A JSON encoder that's capable of serializing datetime objects and bytes.""" - - def __init__(self, *args, exclude_readonly: bool = False, format: typing.Optional[str] = None, **kwargs): - super().__init__(*args, **kwargs) - self.exclude_readonly = exclude_readonly - self.format = format - - def default(self, o): # pylint: disable=too-many-return-statements - if _is_model(o): - if self.exclude_readonly: - readonly_props = [p._rest_name for p in o._attr_to_rest_field.values() if _is_readonly(p)] - return {k: v for k, v in o.items() if k not in readonly_props} - return dict(o.items()) - try: - return super(SdkJSONEncoder, self).default(o) - except TypeError: - if isinstance(o, _Null): - return None - if isinstance(o, decimal.Decimal): - return float(o) - if isinstance(o, (bytes, bytearray)): - return _serialize_bytes(o, self.format) - try: - # First try datetime.datetime - return _serialize_datetime(o, self.format) - except AttributeError: - pass - # Last, try datetime.timedelta - try: - return _timedelta_as_isostr(o) - except AttributeError: - # This will be raised when it hits value.total_seconds in the method above - pass - return super(SdkJSONEncoder, self).default(o) - - -_VALID_DATE = re.compile(r"\d{4}[-]\d{2}[-]\d{2}T\d{2}:\d{2}:\d{2}" + r"\.?\d*Z?[-+]?[\d{2}]?:?[\d{2}]?") -_VALID_RFC7231 = re.compile( - r"(Mon|Tue|Wed|Thu|Fri|Sat|Sun),\s\d{2}\s" - r"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s\d{4}\s\d{2}:\d{2}:\d{2}\sGMT" -) - - -def _deserialize_datetime(attr: typing.Union[str, datetime]) -> datetime: - """Deserialize ISO-8601 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: ~datetime.datetime - :returns: The datetime object from that input - """ - if isinstance(attr, datetime): - # i'm already deserialized - return attr - attr = attr.upper() - match = _VALID_DATE.match(attr) - if not match: - raise ValueError("Invalid datetime string: " + attr) - - check_decimal = attr.split(".") - if len(check_decimal) > 1: - decimal_str = "" - for digit in check_decimal[1]: - if digit.isdigit(): - decimal_str += digit - else: - break - if len(decimal_str) > 6: - attr = attr.replace(decimal_str, decimal_str[0:6]) - - date_obj = isodate.parse_datetime(attr) - test_utc = date_obj.utctimetuple() - if test_utc.tm_year > 9999 or test_utc.tm_year < 1: - raise OverflowError("Hit max or min date") - return date_obj - - -def _deserialize_datetime_rfc7231(attr: typing.Union[str, datetime]) -> datetime: - """Deserialize RFC7231 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: ~datetime.datetime - :returns: The datetime object from that input - """ - if isinstance(attr, datetime): - # i'm already deserialized - return attr - match = _VALID_RFC7231.match(attr) - if not match: - raise ValueError("Invalid datetime string: " + attr) - - return email.utils.parsedate_to_datetime(attr) - - -def _deserialize_datetime_unix_timestamp(attr: typing.Union[float, datetime]) -> datetime: - """Deserialize unix timestamp into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: ~datetime.datetime - :returns: The datetime object from that input - """ - if isinstance(attr, datetime): - # i'm already deserialized - return attr - return datetime.fromtimestamp(attr, TZ_UTC) - - -def _deserialize_date(attr: typing.Union[str, date]) -> date: - """Deserialize ISO-8601 formatted string into Date object. - :param str attr: response string to be deserialized. - :rtype: date - :returns: The date object from that input - """ - # This must NOT use defaultmonth/defaultday. Using None ensure this raises an exception. - if isinstance(attr, date): - return attr - return isodate.parse_date(attr, defaultmonth=None, defaultday=None) # type: ignore - - -def _deserialize_time(attr: typing.Union[str, time]) -> time: - """Deserialize ISO-8601 formatted string into time object. - - :param str attr: response string to be deserialized. - :rtype: datetime.time - :returns: The time object from that input - """ - if isinstance(attr, time): - return attr - return isodate.parse_time(attr) - - -def _deserialize_bytes(attr): - if isinstance(attr, (bytes, bytearray)): - return attr - return bytes(base64.b64decode(attr)) - - -def _deserialize_bytes_base64(attr): - if isinstance(attr, (bytes, bytearray)): - return attr - padding = "=" * (3 - (len(attr) + 3) % 4) # type: ignore - attr = attr + padding # type: ignore - encoded = attr.replace("-", "+").replace("_", "/") - return bytes(base64.b64decode(encoded)) - - -def _deserialize_duration(attr): - if isinstance(attr, timedelta): - return attr - return isodate.parse_duration(attr) - - -def _deserialize_decimal(attr): - if isinstance(attr, decimal.Decimal): - return attr - return decimal.Decimal(str(attr)) - - -_DESERIALIZE_MAPPING = { - datetime: _deserialize_datetime, - date: _deserialize_date, - time: _deserialize_time, - bytes: _deserialize_bytes, - bytearray: _deserialize_bytes, - timedelta: _deserialize_duration, - typing.Any: lambda x: x, - decimal.Decimal: _deserialize_decimal, -} - -_DESERIALIZE_MAPPING_WITHFORMAT = { - "rfc3339": _deserialize_datetime, - "rfc7231": _deserialize_datetime_rfc7231, - "unix-timestamp": _deserialize_datetime_unix_timestamp, - "base64": _deserialize_bytes, - "base64url": _deserialize_bytes_base64, -} - - -def get_deserializer(annotation: typing.Any, rf: typing.Optional["_RestField"] = None): - if rf and rf._format: - return _DESERIALIZE_MAPPING_WITHFORMAT.get(rf._format) - return _DESERIALIZE_MAPPING.get(annotation) - - -def _get_type_alias_type(module_name: str, alias_name: str): - types = { - k: v - for k, v in sys.modules[module_name].__dict__.items() - if isinstance(v, typing._GenericAlias) # type: ignore - } - if alias_name not in types: - return alias_name - return types[alias_name] - - -def _get_model(module_name: str, model_name: str): - models = { - k: v - for k, v in sys.modules[module_name].__dict__.items() - if isinstance(v, type) - } - module_end = module_name.rsplit(".", 1)[0] - models.update({ - k: v - for k, v in sys.modules[module_end].__dict__.items() - if isinstance(v, type) - }) - if isinstance(model_name, str): - model_name = model_name.split(".")[-1] - if model_name not in models: - return model_name - return models[model_name] - - -_UNSET = object() - - -class _MyMutableMapping(MutableMapping[str, typing.Any]): # pylint: disable=unsubscriptable-object - def __init__(self, data: typing.Dict[str, typing.Any]) -> None: - self._data = data - - def __contains__(self, key: typing.Any) -> bool: - return key in self._data - - def __getitem__(self, key: str) -> typing.Any: - return self._data.__getitem__(key) - - def __setitem__(self, key: str, value: typing.Any) -> None: - self._data.__setitem__(key, value) - - def __delitem__(self, key: str) -> None: - self._data.__delitem__(key) - - def __iter__(self) -> typing.Iterator[typing.Any]: - return self._data.__iter__() - - def __len__(self) -> int: - return self._data.__len__() - - def __ne__(self, other: typing.Any) -> bool: - return not self.__eq__(other) - - def keys(self) -> typing.KeysView[str]: - return self._data.keys() - - def values(self) -> typing.ValuesView[typing.Any]: - return self._data.values() - - def items(self) -> typing.ItemsView[str, typing.Any]: - return self._data.items() - - def get(self, key: str, default: typing.Any = None) -> typing.Any: - try: - return self[key] - except KeyError: - return default - - @typing.overload - def pop(self, key: str) -> typing.Any: - ... - - @typing.overload - def pop(self, key: str, default: _T) -> _T: - ... - - @typing.overload - def pop(self, key: str, default: typing.Any) -> typing.Any: - ... - - def pop(self, key: str, default: typing.Any = _UNSET) -> typing.Any: - if default is _UNSET: - return self._data.pop(key) - return self._data.pop(key, default) - - def popitem(self) -> typing.Tuple[str, typing.Any]: - return self._data.popitem() - - def clear(self) -> None: - self._data.clear() - - def update(self, *args: typing.Any, **kwargs: typing.Any) -> None: - self._data.update(*args, **kwargs) - - @typing.overload - def setdefault(self, key: str, default: None = None) -> None: - ... - - @typing.overload - def setdefault(self, key: str, default: typing.Any) -> typing.Any: - ... - - def setdefault(self, key: str, default: typing.Any = _UNSET) -> typing.Any: - if default is _UNSET: - return self._data.setdefault(key) - return self._data.setdefault(key, default) - - def __eq__(self, other: typing.Any) -> bool: - try: - other_model = self.__class__(other) - except Exception: - return False - return self._data == other_model._data - - def __repr__(self) -> str: - return str(self._data) - - -def _is_model(obj: typing.Any) -> bool: - return getattr(obj, "_is_model", False) - - -def _serialize(o, format: typing.Optional[str] = None): # pylint: disable=too-many-return-statements - if isinstance(o, list): - return [_serialize(x, format) for x in o] - if isinstance(o, dict): - return {k: _serialize(v, format) for k, v in o.items()} - if isinstance(o, set): - return {_serialize(x, format) for x in o} - if isinstance(o, tuple): - return tuple(_serialize(x, format) for x in o) - if isinstance(o, (bytes, bytearray)): - return _serialize_bytes(o, format) - if isinstance(o, decimal.Decimal): - return float(o) - if isinstance(o, enum.Enum): - return o.value - try: - # First try datetime.datetime - return _serialize_datetime(o, format) - except AttributeError: - pass - # Last, try datetime.timedelta - try: - return _timedelta_as_isostr(o) - except AttributeError: - # This will be raised when it hits value.total_seconds in the method above - pass - return o - - -def _get_rest_field( - attr_to_rest_field: typing.Dict[str, "_RestField"], rest_name: str -) -> typing.Optional["_RestField"]: - try: - return next(rf for rf in attr_to_rest_field.values() if rf._rest_name == rest_name) - except StopIteration: - return None - - -def _create_value(rf: typing.Optional["_RestField"], value: typing.Any) -> typing.Any: - if not rf: - return _serialize(value, None) - if rf._is_multipart_file_input: - return value - if rf._is_model: - return _deserialize(rf._type, value) - return _serialize(value, rf._format) - - -class Model(_MyMutableMapping): - _is_model = True - - def __init__(self, *args: typing.Any, **kwargs: typing.Any) -> None: - class_name = self.__class__.__name__ - if len(args) > 1: - raise TypeError(f"{class_name}.__init__() takes 2 positional arguments but {len(args) + 1} were given") - dict_to_pass = { - rest_field._rest_name: rest_field._default - for rest_field in self._attr_to_rest_field.values() - if rest_field._default is not _UNSET - } - if args: - dict_to_pass.update( - {k: _create_value(_get_rest_field(self._attr_to_rest_field, k), v) for k, v in args[0].items()} - ) - else: - non_attr_kwargs = [k for k in kwargs if k not in self._attr_to_rest_field] - if non_attr_kwargs: - # actual type errors only throw the first wrong keyword arg they see, so following that. - raise TypeError(f"{class_name}.__init__() got an unexpected keyword argument '{non_attr_kwargs[0]}'") - dict_to_pass.update( - { - self._attr_to_rest_field[k]._rest_name: _create_value(self._attr_to_rest_field[k], v) - for k, v in kwargs.items() - if v is not None - } - ) - super().__init__(dict_to_pass) - - def copy(self) -> "Model": - return Model(self.__dict__) - - def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self: # pylint: disable=unused-argument - # we know the last three classes in mro are going to be 'Model', 'dict', and 'object' - mros = cls.__mro__[:-3][::-1] # ignore model, dict, and object parents, and reverse the mro order - attr_to_rest_field: typing.Dict[str, _RestField] = { # map attribute name to rest_field property - k: v for mro_class in mros for k, v in mro_class.__dict__.items() if k[0] != "_" and hasattr(v, "_type") - } - annotations = { - k: v - for mro_class in mros - if hasattr(mro_class, "__annotations__") # pylint: disable=no-member - for k, v in mro_class.__annotations__.items() # pylint: disable=no-member - } - for attr, rf in attr_to_rest_field.items(): - rf._module = cls.__module__ - if not rf._type: - rf._type = rf._get_deserialize_callable_from_annotation(annotations.get(attr, None)) - if not rf._rest_name_input: - rf._rest_name_input = attr - cls._attr_to_rest_field: typing.Dict[str, _RestField] = dict(attr_to_rest_field.items()) - - return super().__new__(cls) # pylint: disable=no-value-for-parameter - - def __init_subclass__(cls, discriminator: typing.Optional[str] = None) -> None: - for base in cls.__bases__: - if hasattr(base, "__mapping__"): # pylint: disable=no-member - base.__mapping__[discriminator or cls.__name__] = cls # type: ignore # pylint: disable=no-member - - @classmethod - def _get_discriminator(cls, exist_discriminators) -> typing.Optional[str]: - for v in cls.__dict__.values(): - if isinstance(v, _RestField) and v._is_discriminator and v._rest_name not in exist_discriminators: # pylint: disable=protected-access - return v._rest_name # pylint: disable=protected-access - return None - - @classmethod - def _deserialize(cls, data, exist_discriminators): - if not hasattr(cls, "__mapping__"): # pylint: disable=no-member - return cls(data) - discriminator = cls._get_discriminator(exist_discriminators) - exist_discriminators.append(discriminator) - mapped_cls = cls.__mapping__.get( - data.get(discriminator), cls - ) # pyright: ignore # pylint: disable=no-member - if mapped_cls == cls: - return cls(data) - return mapped_cls._deserialize(data, exist_discriminators) # pylint: disable=protected-access - - def as_dict(self, *, exclude_readonly: bool = False) -> typing.Dict[str, typing.Any]: - """Return a dict that can be JSONify using json.dump. - - :keyword bool exclude_readonly: Whether to remove the readonly properties. - :returns: A dict JSON compatible object - :rtype: dict - """ - - result = {} - if exclude_readonly: - readonly_props = [p._rest_name for p in self._attr_to_rest_field.values() if _is_readonly(p)] - for k, v in self.items(): - if exclude_readonly and k in readonly_props: # pyright: ignore - continue - is_multipart_file_input = False - try: - is_multipart_file_input = next(rf for rf in self._attr_to_rest_field.values() if rf._rest_name == k)._is_multipart_file_input - except StopIteration: - pass - result[k] = v if is_multipart_file_input else Model._as_dict_value(v, exclude_readonly=exclude_readonly) - return result - - @staticmethod - def _as_dict_value(v: typing.Any, exclude_readonly: bool = False) -> typing.Any: - if v is None or isinstance(v, _Null): - return None - if isinstance(v, (list, tuple, set)): - return type(v)( - Model._as_dict_value(x, exclude_readonly=exclude_readonly) - for x in v - ) - if isinstance(v, dict): - return { - dk: Model._as_dict_value(dv, exclude_readonly=exclude_readonly) - for dk, dv in v.items() - } - return v.as_dict(exclude_readonly=exclude_readonly) if hasattr(v, "as_dict") else v - -def _deserialize_model(model_deserializer: typing.Optional[typing.Callable], obj): - if _is_model(obj): - return obj - return _deserialize(model_deserializer, obj) - -def _deserialize_with_optional(if_obj_deserializer: typing.Optional[typing.Callable], obj): - if obj is None: - return obj - return _deserialize_with_callable(if_obj_deserializer, obj) - -def _deserialize_with_union(deserializers, obj): - for deserializer in deserializers: - try: - return _deserialize(deserializer, obj) - except DeserializationError: - pass - raise DeserializationError() - -def _deserialize_dict( - value_deserializer: typing.Optional[typing.Callable], - module: typing.Optional[str], - obj: typing.Dict[typing.Any, typing.Any], -): - if obj is None: - return obj - return { - k: _deserialize(value_deserializer, v, module) - for k, v in obj.items() - } - -def _deserialize_multiple_sequence( - entry_deserializers: typing.List[typing.Optional[typing.Callable]], - module: typing.Optional[str], - obj, -): - if obj is None: - return obj - return type(obj)( - _deserialize(deserializer, entry, module) - for entry, deserializer in zip(obj, entry_deserializers) - ) - -def _deserialize_sequence( - deserializer: typing.Optional[typing.Callable], - module: typing.Optional[str], - obj, -): - if obj is None: - return obj - return type(obj)(_deserialize(deserializer, entry, module) for entry in obj) - -def _sorted_annotations(types: typing.List[typing.Any]) -> typing.List[typing.Any]: - return sorted( - types, - key=lambda x: hasattr(x, "__name__") and x.__name__.lower() in ("str", "float", "int", "bool"), - ) - -def _get_deserialize_callable_from_annotation( # pylint: disable=R0911, R0915, R0912 - annotation: typing.Any, - module: typing.Optional[str], - rf: typing.Optional["_RestField"] = None, -) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]: - if not annotation or annotation in [int, float]: - return None - - # is it a type alias? - if isinstance(annotation, str): - if module is not None: - annotation = _get_type_alias_type(module, annotation) - - # is it a forward ref / in quotes? - if isinstance(annotation, (str, typing.ForwardRef)): - try: - model_name = annotation.__forward_arg__ # type: ignore - except AttributeError: - model_name = annotation - if module is not None: - annotation = _get_model(module, model_name) - - try: - if module and _is_model(annotation): - if rf: - rf._is_model = True - - return functools.partial(_deserialize_model, annotation) # pyright: ignore - except Exception: - pass - - # is it a literal? - try: - if annotation.__origin__ is typing.Literal: # pyright: ignore - return None - except AttributeError: - pass - - # is it optional? - try: - if any(a for a in annotation.__args__ if a == type(None)): # pyright: ignore - if len(annotation.__args__) <= 2: # pyright: ignore - if_obj_deserializer = _get_deserialize_callable_from_annotation( - next(a for a in annotation.__args__ if a != type(None)), module, rf # pyright: ignore - ) - - return functools.partial(_deserialize_with_optional, if_obj_deserializer) - # the type is Optional[Union[...]], we need to remove the None type from the Union - annotation_copy = copy.copy(annotation) - annotation_copy.__args__ = [a for a in annotation_copy.__args__ if a != type(None)] # pyright: ignore - return _get_deserialize_callable_from_annotation(annotation_copy, module, rf) - except AttributeError: - pass - - # is it union? - if getattr(annotation, "__origin__", None) is typing.Union: - # initial ordering is we make `string` the last deserialization option, because it is often them most generic - deserializers = [ - _get_deserialize_callable_from_annotation(arg, module, rf) - for arg in _sorted_annotations(annotation.__args__) # pyright: ignore - ] - - return functools.partial(_deserialize_with_union, deserializers) - - try: - if annotation._name == "Dict": # pyright: ignore - value_deserializer = _get_deserialize_callable_from_annotation( - annotation.__args__[1], module, rf # pyright: ignore - ) - - - return functools.partial( - _deserialize_dict, - value_deserializer, - module, - ) - except (AttributeError, IndexError): - pass - try: - if annotation._name in ["List", "Set", "Tuple", "Sequence"]: # pyright: ignore - if len(annotation.__args__) > 1: # pyright: ignore - - - entry_deserializers = [ - _get_deserialize_callable_from_annotation(dt, module, rf) for dt in annotation.__args__ # pyright: ignore - ] - return functools.partial(_deserialize_multiple_sequence, entry_deserializers, module) - deserializer = _get_deserialize_callable_from_annotation( - annotation.__args__[0], module, rf # pyright: ignore - ) - - - - return functools.partial(_deserialize_sequence, deserializer, module) - except (TypeError, IndexError, AttributeError, SyntaxError): - pass - - def _deserialize_default( - deserializer, - obj, - ): - if obj is None: - return obj - try: - return _deserialize_with_callable(deserializer, obj) - except Exception: - pass - return obj - - if get_deserializer(annotation, rf): - return functools.partial(_deserialize_default, get_deserializer(annotation, rf)) - - return functools.partial(_deserialize_default, annotation) - - -def _deserialize_with_callable( - deserializer: typing.Optional[typing.Callable[[typing.Any], typing.Any]], - value: typing.Any, -): - try: - if value is None or isinstance(value, _Null): - return None - if deserializer is None: - return value - if isinstance(deserializer, CaseInsensitiveEnumMeta): - try: - return deserializer(value) - except ValueError: - # for unknown value, return raw value - return value - if isinstance(deserializer, type) and issubclass(deserializer, Model): - return deserializer._deserialize(value, []) - return typing.cast(typing.Callable[[typing.Any], typing.Any], deserializer)(value) - except Exception as e: - raise DeserializationError() from e - - -def _deserialize( - deserializer: typing.Any, - value: typing.Any, - module: typing.Optional[str] = None, - rf: typing.Optional["_RestField"] = None, - format: typing.Optional[str] = None, -) -> typing.Any: - if isinstance(value, PipelineResponse): - value = value.http_response.json() - if rf is None and format: - rf = _RestField(format=format) - if not isinstance(deserializer, functools.partial): - deserializer = _get_deserialize_callable_from_annotation(deserializer, module, rf) - return _deserialize_with_callable(deserializer, value) - - -class _RestField: - def __init__( - self, - *, - name: typing.Optional[str] = None, - type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - is_discriminator: bool = False, - visibility: typing.Optional[typing.List[str]] = None, - default: typing.Any = _UNSET, - format: typing.Optional[str] = None, - is_multipart_file_input: bool = False, - ): - self._type = type - self._rest_name_input = name - self._module: typing.Optional[str] = None - self._is_discriminator = is_discriminator - self._visibility = visibility - self._is_model = False - self._default = default - self._format = format - self._is_multipart_file_input = is_multipart_file_input - - @property - def _class_type(self) -> typing.Any: - return getattr(self._type, "args", [None])[0] - - @property - def _rest_name(self) -> str: - if self._rest_name_input is None: - raise ValueError("Rest name was never set") - return self._rest_name_input - - def __get__(self, obj: Model, type=None): # pylint: disable=redefined-builtin - # by this point, type and rest_name will have a value bc we default - # them in __new__ of the Model class - item = obj.get(self._rest_name) - if item is None: - return item - if self._is_model: - return item - return _deserialize(self._type, _serialize(item, self._format), rf=self) - - def __set__(self, obj: Model, value) -> None: - if value is None: - # we want to wipe out entries if users set attr to None - try: - obj.__delitem__(self._rest_name) - except KeyError: - pass - return - if self._is_model: - if not _is_model(value): - value = _deserialize(self._type, value) - obj.__setitem__(self._rest_name, value) - return - obj.__setitem__(self._rest_name, _serialize(value, self._format)) - - def _get_deserialize_callable_from_annotation( - self, annotation: typing.Any - ) -> typing.Optional[typing.Callable[[typing.Any], typing.Any]]: - return _get_deserialize_callable_from_annotation(annotation, self._module, self) - - -def rest_field( - *, - name: typing.Optional[str] = None, - type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin - visibility: typing.Optional[typing.List[str]] = None, - default: typing.Any = _UNSET, - format: typing.Optional[str] = None, - is_multipart_file_input: bool = False, -) -> typing.Any: - return _RestField(name=name, type=type, visibility=visibility, default=default, format=format, is_multipart_file_input=is_multipart_file_input) - - -def rest_discriminator( - *, - name: typing.Optional[str] = None, - type: typing.Optional[typing.Callable] = None, # pylint: disable=redefined-builtin -) -> typing.Any: - return _RestField(name=name, type=type, is_discriminator=True) diff --git a/packages/autorest.python/generator/pygen/codegen/templates/model_container.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/model_container.py.jinja2 deleted file mode 100644 index 4b824762f51..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/model_container.py.jinja2 +++ /dev/null @@ -1,13 +0,0 @@ -{% import 'operation_tools.jinja2' as op_tools %} -# coding=utf-8 -# pylint: disable=too-many-lines -{{ code_model.options['license_header'] }} - -{{ imports }} -{% for model in code_model.model_types %} -{% if model.base == "dpg" %} -{% include "model_dpg.py.jinja2" %} -{% elif model.base == "msrest" %} -{% include "model_msrest.py.jinja2" %} -{% endif %} -{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/model_dpg.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/model_dpg.py.jinja2 deleted file mode 100644 index 048de17212c..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/model_dpg.py.jinja2 +++ /dev/null @@ -1,90 +0,0 @@ -{# actual template starts here #} -{% import "macros.jinja2" as macros %} - - -{{ serializer.declare_model(model) }} - """{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n ") }} - {% if model.discriminated_subtypes %} - - {{ serializer.discriminator_docstring(model) | wordwrap(width=95, break_long_words=False, break_on_hyphens=False, wrapstring='\n ') }} - {% endif %} - {% if model.has_readonly_or_constant_property %} - - Readonly variables are only populated by the server, and will be ignored when sending a request. - {% endif %} - {% if (model.properties | selectattr('optional', "equalto", false) | first) is defined %} - - All required parameters must be populated in order to send to server. - {% endif %} - - {% if model.properties != None %} - {% for p in model.properties %} - {% for line in serializer.variable_documentation_string(p) %} - {% for doc_string in line.replace('\n', '\n ').split('\n') %} - {{ macros.wrap_model_string(doc_string, '\n ') -}} - {% endfor %} - {% endfor %} - {% endfor %} - {% endif %} - """ - - {% if model.is_polymorphic %} - __mapping__: Dict[str, _model_base.Model] = {} - {% endif %} - {% for p in serializer.get_properties_to_declare(model)%} - {{ serializer.declare_property(p) }} - {% set prop_description = p.description(is_operation_file=False).replace('"', '\\"') %} - {% if prop_description %} - """{{ macros.wrap_model_string(prop_description, '\n ', '\"\"\"') -}} - {% endif %} - {% endfor %} - - {% if code_model.options["models_mode"] == "dpg" and model.flattened_property %} - __flattened_items = ["{{ model.flattened_items|join('\", \"') }}"] - {% endif %} - - {% if not model.internal and serializer.init_line(model) %} - @overload - def __init__( - self, - {% for param_signature in serializer.init_line(model) %} - {{ param_signature }} - {% endfor %} - ): - ... - - @overload - def __init__(self, mapping: Mapping[str, Any]): - """ - :param mapping: raw JSON to initialize the model. - :type mapping: Mapping[str, Any] - """ - - {% endif %} - {% set initialize_properties = serializer.initialize_properties(model) %} - {% if not model.internal and serializer.init_line(model) or initialize_properties %} - def __init__(self, *args: Any, **kwargs: Any) -> None:{{ '# pylint: disable=useless-super-delegation' if not initialize_properties else '' }} - {% for line in serializer.super_call(model) %} - {{ line }} - {% endfor %} - {% for initialize_property in initialize_properties %} - {{ initialize_property }} - {% endfor %} - {% if code_model.options["models_mode"] == "dpg" and model.flattened_property %} - {% set flattened_property_attr = model.flattened_property.client_name %} - - def __getattr__(self, name: str) -> Any: - if name in self.__flattened_items: - if self.{{ flattened_property_attr }} is None: return None - return getattr(self.{{ flattened_property_attr }}, name) - raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'") - - def __setattr__(self, key: str, value: Any) -> None: - if key in self.__flattened_items: - if self.{{ flattened_property_attr }} is None: - self.{{ flattened_property_attr }} = self._attr_to_rest_field["{{ flattened_property_attr }}"]._class_type() - setattr(self.properties, key, value) - else: - super().__setattr__(key, value) - {% endif %} - {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/model_init.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/model_init.py.jinja2 deleted file mode 100644 index f4710680849..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/model_init.py.jinja2 +++ /dev/null @@ -1,28 +0,0 @@ -{% import 'keywords.jinja2' as keywords %} -# coding=utf-8 -{{ code_model.options['license_header'] }} -{% if schemas %} - - {% for schema in schemas %} -from .{{ code_model.models_filename }} import {{ schema }} - {% endfor %} -{% endif %} -{% if enums %} - -{% for enum in enums %} -from .{{ code_model.enums_filename }} import {{ enum }} -{% endfor %} -{% endif %} -{{ keywords.patch_imports() }} -__all__ = [ - {% for schema in schemas %} - '{{ schema }}', - {% endfor %} - {% if enums %} - {% for enum in enums %} - '{{ enum }}', - {% endfor %} -{% endif %} -] -{{ keywords.extend_all }} -_patch_sdk() diff --git a/packages/autorest.python/generator/pygen/codegen/templates/model_msrest.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/model_msrest.py.jinja2 deleted file mode 100644 index 15ccc8a2d90..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/model_msrest.py.jinja2 +++ /dev/null @@ -1,92 +0,0 @@ -{# actual template starts here #} -{% import "macros.jinja2" as macros %} -{% set initialize_properties = serializer.initialize_properties(model) %} -{% set exist_constant = (model.properties | selectattr('constant') | first) is defined %} - -{{ serializer.declare_model(model) }} - """{{ op_tools.wrap_string(model.description(is_operation_file=False), "\n ") }} - {% if model.discriminated_subtypes %} - - {{ serializer.discriminator_docstring(model) | wordwrap(width=95, break_long_words=False, break_on_hyphens=False, wrapstring='\n ') }} - {% endif %} - {% if model.has_readonly_or_constant_property %} - - Variables are only populated by the server, and will be ignored when sending a request. - {% endif %} - {% if (model.properties | selectattr('optional', "equalto", false) | first) is defined %} - - All required parameters must be populated in order to send to server. - {% endif %} - - {% if model.properties != None %} - {% for p in model.properties %} - {% for line in serializer.variable_documentation_string(p) %} - {% for doc_string in line.replace('\n', '\n ').split('\n') %} - {{ macros.wrap_model_string(doc_string, '\n ') -}} - {% endfor %} - {% endfor %} - {% endfor %} - {% endif %} - """ -{% if initialize_properties or exist_constant %} - {% if (model.properties | selectattr('validation') ) | first %} - - _validation = { - {% for p in model.properties | selectattr('validation')%} - '{{ p.client_name }}': {{ str(p.validation) }}, - {% endfor %} - } - {% endif %} - - _attribute_map = { - {% if model.properties %} - {% for p in model.properties %} - {{ serializer.declare_property(p) }} - {% endfor %} - {% endif %} - } - {% if model.discriminated_subtypes %} - - _subtype_map = { - '{{ model.discriminator.client_name }}': {{ str(model.discriminated_subtypes_name_mapping) }} - } - {% endif %} - {% if model.xml_map_content %} - _xml_map = { - {{ model.xml_map_content }} - } - {% endif %} - {% if exist_constant %} - - {% for p in model.properties | selectattr('constant')%} - {{ p.client_name }} = {{ p.type.get_declaration() }} - {% endfor %} - {% endif %} - - def __init__({{ model.init_pylint_disable }} - self, - {% for param_signature in serializer.init_line(model) %} - {{ param_signature }} - {% endfor %} - **kwargs: Any - ) -> None: - """ - {% if model.properties %} - {% for p in model.properties %} - {% if p.is_input %} - {% for line in serializer.input_documentation_string(p) %} - {% for doc_string in line.replace('\n', '\n ').split('\n') %} - {{ macros.wrap_model_string(doc_string, '\n ') -}} - {% endfor %} - {% endfor %} - {% endif %} - {% endfor %} - {% endif %} - """ - {% for line in serializer.super_call(model) %} - {{ line }} - {% endfor %} - {% for initialize_property in initialize_properties %} - {{ initialize_property }} - {% endfor %} -{% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/operation.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/operation.py.jinja2 deleted file mode 100644 index aec6199880a..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/operation.py.jinja2 +++ /dev/null @@ -1,21 +0,0 @@ -{% import 'keywords.jinja2' as keywords with context %} -{% import 'operation_tools.jinja2' as op_tools %} -{# actual template starts here #} -{% if operation.overloads and operation.include_documentation %} -{{ op_tools.generate_overloads(operation_serializer, operation) }} -{% endif %} -{{ operation_serializer.method_signature_and_response_type_annotation(operation) }} -{% if operation.include_documentation %} - {{ op_tools.description(operation, operation_serializer) | indent }}{% endif %} - {% if not operation.abstract %} - {% if operation.deprecated %} - warnings.warn('Method {{operation.name}} is deprecated', DeprecationWarning) - {% endif %} - {{ op_tools.serialize(operation_serializer.error_map(operation)) | indent }} - {% if operation_serializer.pop_kwargs_from_signature(operation) %} - {{ op_tools.serialize(operation_serializer.pop_kwargs_from_signature(operation)) | indent }} - {% endif %} - {{ op_tools.serialize(operation_serializer.call_request_builder(operation)) | indent }} - {{ op_tools.serialize(operation_serializer.make_pipeline_call(operation)) | indent }} - {{ op_tools.serialize(operation_serializer.handle_response(operation)) | indent }} - {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/operation_group.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/operation_group.py.jinja2 deleted file mode 100644 index 63e4385164b..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/operation_group.py.jinja2 +++ /dev/null @@ -1,75 +0,0 @@ -{% set base_class = ("(" + operation_group.base_class + ")") if operation_group.base_class else "" %} -{% macro check_abstract_methods() %} -{% if operation_group.has_abstract_operations %} - raise_if_not_implemented(self.__class__, [ - {% for operation in operation_group.operations if operation.abstract %} - '{{operation.name}}', - {% endfor %} - ]) -{% endif %} -{% endmacro %} -{% if operation_group.base_class %} -class {{ operation_group.class_name }}( {{ operation_group.pylint_disable }} - {{ operation_group.base_class }} -): -{% else %} -class {{ operation_group.class_name }}: {{ operation_group.pylint_disable }} -{% endif %} -{% if not operation_group.is_mixin %} - """ - .. warning:: - **DO NOT** instantiate this class directly. - - Instead, you should access the following operations through - :class:`{{ "~" + code_model.namespace + (".aio." if async_mode else ".") + operation_group.client.name }}`'s - :attr:`{{ operation_group.property_name }}` attribute. - """ - -{% if code_model.public_model_types and code_model.options["models_mode"] == "msrest" %} - models = _models - -{% endif %} - def __init__(self, *args, **kwargs){{ return_none_type_annotation }}: - input_args = list(args) - self._client = input_args.pop(0) if input_args else kwargs.pop("client") - self._config = input_args.pop(0) if input_args else kwargs.pop("config") - self._serialize = input_args.pop(0) if input_args else kwargs.pop("serializer") - self._deserialize = input_args.pop(0) if input_args else kwargs.pop("deserializer") - {% if code_model.options["multiapi"] %} - self._api_version = input_args.pop(0) if input_args else kwargs.pop("api_version") - {% endif %} - - {% for og in operation_group.operation_groups %} - self.{{ og.property_name }} = {{ og.class_name }}( - self._client, self._config, self._serialize, self._deserialize{{ ", self._api_version" if code_model.options["multiapi"] else "" }} - ) - {% endfor %} - -{{ check_abstract_methods() }} -{% elif operation_group.has_abstract_operations %} - - def __init__(self){{ return_none_type_annotation }}: -{{ check_abstract_methods() }} -{% endif %} -{% if operation_group.is_mixin and code_model.options["multiapi"] %} - def _api_version(self, op_name: str) -> str: # pylint: disable=unused-argument - try: - return self._config.api_version - except: # pylint: disable=bare-except - return "" -{% endif %} -{% for operation in operation_group.operations if not operation.abstract %} - -{% set request_builder = operation.request_builder %} -{% set operation_serializer = get_operation_serializer(operation) %} - {% if operation.operation_type == "lropaging" %} - {%- macro someop() %}{% include "lro_paging_operation.py.jinja2" %}{% endmacro %} - {% elif operation.operation_type == "lro" %} - {%- macro someop() %}{% include "lro_operation.py.jinja2" %}{% endmacro %} - {% elif operation.operation_type == "paging" %} - {% macro someop() %}{% include "paging_operation.py.jinja2" %}{% endmacro %} - {% else %} - {% macro someop() %}{% include "operation.py.jinja2" %}{% endmacro %} - {% endif %} - {{ someop()|indent }} -{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 deleted file mode 100644 index 453087bcc18..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 +++ /dev/null @@ -1,20 +0,0 @@ -{% import 'operation_tools.jinja2' as op_tools %} -{% set operations_description = "async operations" if async_mode else "operations" %} -{% set return_none_type_annotation = " -> None" if async_mode else "" %} -# pylint: disable=too-many-lines,too-many-statements -# coding=utf-8 -{{ code_model.options['license_header'] }} -{{ imports }} -{{ unset }} -{% if code_model.options["builders_visibility"] == "embedded" and not async_mode %} -{{ op_tools.declare_serializer(code_model) }} - {% for operation_group in operation_groups %} - {% for request_builder in get_request_builders(operation_group) %} - -{% include "request_builder.py.jinja2" %} - {% endfor %} - {% endfor %} -{% endif %} -{% for operation_group in operation_groups %} - {% include "operation_group.py.jinja2" %} -{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/operation_tools.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/operation_tools.jinja2 deleted file mode 100644 index 598da57e4d0..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/operation_tools.jinja2 +++ /dev/null @@ -1,74 +0,0 @@ -{% macro wrap_string(string, wrapstring, width=95) %}{{ string | replace("\\", "\\\\") | wordwrap(width=width, break_long_words=False, break_on_hyphens=False, wrapstring=wrapstring)}}{% endmacro %} - -{% macro description(builder, serializer) %} -{% set example_template = serializer.example_template(builder) %} - {% for description in serializer.description_and_summary(builder) %} - {% if description %} -{% set description = wrap_string(description, wrapstring='\n') %} - {% if serializer.line_too_long(example_template) and loop.first %} -# pylint: disable=line-too-long - {% endif %} -{{ '"""' + description if loop.first else description }} - {% else %} - - {% endif %} - {% endfor %} - {% for description in serializer.param_description_and_response_docstring(builder) %} - {% if description %} -{{ wrap_string(description, wrapstring='\n ') }} - {% else %} - - {% endif %} -{% endfor %} -{% if example_template %} - -Example: - .. code-block:: python - {% for template_line in example_template %} - {% if template_line %} - {% set wrap_amount = (template_line | length) - (template_line.lstrip() | length) + 10 %} - {{ wrap_string(template_line, wrapstring='\n' + " " * wrap_amount, width=(95 - wrap_amount)) }} - {% else %} - - {% endif %} - {% endfor %} -{% endif %} -""" -{% endmacro %} - -{% macro serialize(lines) %} -{% for line in lines %} - {% if line %} -{{ line }} - {% else %} - - {% endif %} -{% endfor %}{% endmacro %} - -{% macro serialize_with_wrap(lines, wrapstring) %} -{% for line in lines %} - {% if line %} -{{ wrap_string(line, wrapstring=wrapstring) }} - {% else %} - - {% endif %} -{% endfor %}{% endmacro %} - -{% macro declare_serializer(code_model) %} -{% if code_model.has_non_abstract_operations %} -_SERIALIZER = Serializer() - {% if not code_model.options["client_side_validation"] %} -_SERIALIZER.client_side_validation = False - {% endif %} -{% endif %} -{% endmacro %} - -{% macro generate_overloads(operation_serializer, operation) %} -{% for overload in operation.overloads %} -{{ operation_serializer.method_signature_and_response_type_annotation(overload) }} -{% if not operation.internal %} - {{ description(overload, operation_serializer) | indent }} -{% else %} - ... -{% endif %} -{% endfor %}{% endmacro %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 deleted file mode 100644 index bb38196f4ac..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 +++ /dev/null @@ -1,17 +0,0 @@ -{% import 'operation_tools.jinja2' as op_tools %} -{% import 'keywords.jinja2' as keywords %} -{# actual template starts here #} -# coding=utf-8 -{{ code_model.options['license_header'] }} - -{{ op_tools.serialize(operation_group_imports()) }} -{{ keywords.patch_imports() }} -__all__ = [ - {% for client in clients %} - {% for operation_group in client.operation_groups %} - '{{ operation_group.class_name }}', - {% endfor %} - {% endfor %} -] -{{ keywords.extend_all }} -_patch_sdk() diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 deleted file mode 100644 index bddf9a22c7b..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 +++ /dev/null @@ -1,6 +0,0 @@ -# Release History - -## 1.0.0b1 (1970-01-01) - -- Initial version - diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 deleted file mode 100644 index 3499e00ae53..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) {{ code_model.options["company_name"] }} Corporation. - -MIT License - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 deleted file mode 100644 index 454a9ad2717..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 +++ /dev/null @@ -1,8 +0,0 @@ -include *.md -include LICENSE -include {{ package_name.replace('-', '/') }}/py.typed -recursive-include tests *.py -recursive-include samples *.py *.md -{%- for init_name in init_names %} -include {{ init_name }} -{%- endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 deleted file mode 100644 index fbea5913604..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 +++ /dev/null @@ -1,107 +0,0 @@ -{% if code_model.is_azure_flavor %} -{% if package_mode == "mgmtplane" -%} -# Microsoft Azure SDK for Python - -This is the Microsoft {{package_pprint_name}} Client Library. -This package has been tested with Python 3.8+. -For a more complete view of Azure libraries, see the [azure sdk python release](https://aka.ms/azsdk/python/all). - -# Usage - -To learn how to use this package, see the [quickstart guide](https://aka.ms/azsdk/python/mgmt) - -For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure) -Code samples for this package can be found at [{{package_pprint_name}}](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com. -Additional code samples for different Azure services are available at [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples) - -# Provide Feedback - -If you encounter any bugs or have suggestions, please file an issue in the -[Issues](https://github.com/Azure/azure-sdk-for-python/issues) -section of the project. - - -![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2F{{package_name}}%2FREADME.png) -{% else %} -# {{ package_pprint_name }} client library for Python - - -## Getting started - -### Install the package - -```bash -python -m pip install {{ package_name }} -``` - -#### Prequisites - -- Python 3.8 or later is required to use this package. -- You need an [Azure subscription][azure_sub] to use this package. -- An existing {{ package_pprint_name }} instance. - -{%- if token_credential %} -#### Create with an Azure Active Directory Credential -To use an [Azure Active Directory (AAD) token credential][authenticate_with_token], -provide an instance of the desired credential type obtained from the -[azure-identity][azure_identity_credentials] library. - -To authenticate with AAD, you must first [pip][pip] install [`azure-identity`][azure_identity_pip] - -After setup, you can choose which type of [credential][azure_identity_credentials] from azure.identity to use. -As an example, [DefaultAzureCredential][default_azure_credential] can be used to authenticate the client: - -Set the values of the client ID, tenant ID, and client secret of the AAD application as environment variables: -`AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_CLIENT_SECRET` - -Use the returned token credential to authenticate the client: - -```python ->>> from {{ namespace }} import {{ client_name }} ->>> from azure.identity import DefaultAzureCredential ->>> client = {{ client_name }}(endpoint='', credential=DefaultAzureCredential()) -``` - -## Examples - -```python ->>> from {{ namespace }} import {{ client_name }} ->>> from azure.identity import DefaultAzureCredential ->>> from {{ code_model.core_library }}.exceptions import HttpResponseError - ->>> client = {{ client_name }}(endpoint='', credential=DefaultAzureCredential()) ->>> try: - - except HttpResponseError as e: - print('service responds error: {}'.format(e.response.json())) - -``` -{%- endif %} - -## Contributing - -This project welcomes contributions and suggestions. Most contributions require -you to agree to a Contributor License Agreement (CLA) declaring that you have -the right to, and actually do, grant us the rights to use your contribution. -For details, visit https://cla.microsoft.com. - -When you submit a pull request, a CLA-bot will automatically determine whether -you need to provide a CLA and decorate the PR appropriately (e.g., label, -comment). Simply follow the instructions provided by the bot. You will only -need to do this once across all repos using our CLA. - -This project has adopted the -[Microsoft Open Source Code of Conduct][code_of_conduct]. For more information, -see the Code of Conduct FAQ or contact opencode@microsoft.com with any -additional questions or comments. - - -[code_of_conduct]: https://opensource.microsoft.com/codeofconduct/ -[authenticate_with_token]: https://docs.microsoft.com/azure/cognitive-services/authentication?tabs=powershell#authenticate-with-an-authentication-token -[azure_identity_credentials]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/identity/azure-identity#credentials -[azure_identity_pip]: https://pypi.org/project/azure-identity/ -[default_azure_credential]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/identity/azure-identity#defaultazurecredential -[pip]: https://pypi.org/project/pip/ -[azure_sub]: https://azure.microsoft.com/free/ -{% endif %} -{% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 deleted file mode 100644 index a9782cabd58..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 +++ /dev/null @@ -1,9 +0,0 @@ --e ../../../tools/azure-sdk-tools -../../core/azure-core -{% if token_credential -%} -../../identity/azure-identity -{% endif -%} -{% if azure_arm -%} -../../core/azure-mgmt-core -{% endif -%} -aiohttp diff --git a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 deleted file mode 100644 index db3290b7521..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 +++ /dev/null @@ -1,110 +0,0 @@ -# coding=utf-8 -{{ license_header }} -# coding: utf-8 -{% if package_mode %} -import os -import re -{% endif -%} -from setuptools import setup, find_packages - -{% set package_name = package_name or code_model.clients[0].name %} - -PACKAGE_NAME = "{{ package_name|lower }}" -{% if package_mode -%} -PACKAGE_PPRINT_NAME = "{{ package_pprint_name }}" - -# a-b-c => a/b/c -package_folder_path = PACKAGE_NAME.replace("-", "/") - -# Version extraction inspired from 'requests' -with open(os.path.join(package_folder_path, "_version.py"), "r") as fd: - version = re.search( - r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE - ).group(1) - -if not version: - raise RuntimeError("Cannot find version information") -{% set description = "\"" + code_model.options["company_name"] + " {} Client Library for Python\".format(PACKAGE_PPRINT_NAME)" %} -{% set author_email = "azpysdkhelp@microsoft.com" %} -{% set url = "https://github.com/Azure/azure-sdk-for-python/tree/main/sdk" %} -{% else %} -version = "{{ package_version }}" -{% set description = "\"%s\""|format(package_name) %} -{% set long_description = code_model.description %} -{% set author_email = "" %} -{% set url = "" %} -{% endif -%} - -setup( - name=PACKAGE_NAME, - version=version, - description={{ description }}, - {% if package_mode %} - long_description=open("README.md", "r").read(), - long_description_content_type="text/markdown", - license="MIT License", - author="{{ code_model.options["company_name"] }} Corporation", - {% endif %} - {% if code_model.is_azure_flavor %} - author_email="{{ author_email }}", - url="{{ url }}", - keywords="azure, azure sdk", - {% endif %} - {% if package_mode %} - classifiers=[ - "Development Status :: {{ dev_status }}", - "Programming Language :: Python", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "License :: OSI Approved :: MIT License", - ], - zip_safe=False, - packages=find_packages( - exclude=[ - "tests", - {% if pkgutil_names %} - # Exclude packages that will be covered by PEP420 or nspkg - {% endif %} - {%- for pkgutil_name in pkgutil_names %} - "{{ pkgutil_name }}", - {%- endfor %} - ] - ), - include_package_data=True, - package_data={ - '{{ code_model.namespace }}': ['py.typed'], - }, - {% else %} - packages=find_packages(), - include_package_data=True, - {% endif %} - install_requires=[ - {% if code_model.is_legacy %} - "msrest>=0.7.1", - {% else %} - "isodate>=0.6.1", - {% endif %} - {% if azure_arm %} - "azure-mgmt-core>=1.3.2", - {% elif code_model.is_azure_flavor %} - "azure-core>=1.30.0", - {% else %} - "corehttp[requests]", - {% endif %} - {% if code_model.need_typing_extensions %} - "typing-extensions>=4.6.0", - {% endif %} - ], - {% if package_mode %} - python_requires=">=3.8", - {% else %} - long_description="""\ - {{ code_model.description }} - """ - {% endif %} -) diff --git a/packages/autorest.python/generator/pygen/codegen/templates/paging_operation.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/paging_operation.py.jinja2 deleted file mode 100644 index d6f0d6f14dc..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/paging_operation.py.jinja2 +++ /dev/null @@ -1,21 +0,0 @@ -{% import 'operation_tools.jinja2' as op_tools with context %} -{# actual template starts here #} -{% if operation.overloads and operation.include_documentation %} -{{ op_tools.generate_overloads(operation_serializer, operation) }} -{% endif %} -{{ operation_serializer.method_signature_and_response_type_annotation(operation) }} -{% if operation.include_documentation %} - {{ op_tools.description(operation, operation_serializer) | indent }}{% endif %} - {% if not operation.abstract %} - {% if operation.deprecated %} - warnings.warn('Method {{operation.name}} is deprecated', DeprecationWarning) - {% endif %} - {% if operation_serializer.pop_kwargs_from_signature(operation) %} - {{ op_tools.serialize(operation_serializer.pop_kwargs_from_signature(operation)) | indent }} - {% endif %} - {{ op_tools.serialize(operation_serializer.set_up_params_for_pager(operation)) | indent }} - - return {{ operation.get_pager(async_mode) }}( - get_next, extract_data - ) - {% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/patch.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/patch.py.jinja2 deleted file mode 100644 index 87c09548714..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/patch.py.jinja2 +++ /dev/null @@ -1,19 +0,0 @@ -# ------------------------------------ -# Copyright (c) {{ code_model.options["company_name"] }} Corporation. -# Licensed under the MIT License. -# ------------------------------------ -"""Customize generated code here. - -Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize -""" -{{ imports }} - -__all__: List[str] = [] # Add all objects you want publicly available to users at this package level - -def patch_sdk(): - """Do not remove from this file. - - `patch_sdk` is a last resort escape hatch that allows you to do customizations - you can't accomplish using the techniques described in - https://aka.ms/azsdk/python/dpcodegen/python/customize - """ diff --git a/packages/autorest.python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 deleted file mode 100644 index 5960c353a89..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 +++ /dev/null @@ -1 +0,0 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore \ No newline at end of file diff --git a/packages/autorest.python/generator/pygen/codegen/templates/request_builder.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/request_builder.py.jinja2 deleted file mode 100644 index f4810000e7b..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/request_builder.py.jinja2 +++ /dev/null @@ -1,28 +0,0 @@ -{% import 'keywords.jinja2' as keywords with context %} -{% import 'operation_tools.jinja2' as op_tools with context %} -{{ request_builder_serializer.method_signature_and_response_type_annotation(request_builder) }} -{% if code_model.options["builders_visibility"] == "public" %} - {{ op_tools.description(request_builder, request_builder_serializer) | indent }} -{% endif %} -{% if not request_builder.is_overload %} - {% if request_builder_serializer.pop_kwargs_from_signature(request_builder) %} - {{ op_tools.serialize(request_builder_serializer.pop_kwargs_from_signature(request_builder)) | indent }} - {%- endif -%} - {% if request_builder_serializer.declare_non_inputtable_constants(request_builder) %} - {{ op_tools.serialize(request_builder_serializer.declare_non_inputtable_constants(request_builder)) | indent }} - {% endif %} - # Construct URL - {{ request_builder_serializer.construct_url(request_builder) }} - {% if request_builder.parameters.path %} - {{ op_tools.serialize(request_builder_serializer.serialize_path(request_builder)) | indent }} - _url: str = _url.format(**path_format_arguments) # type: ignore - {% endif %} - - {% if request_builder.parameters.query %} - {{ op_tools.serialize(request_builder_serializer.serialize_query(request_builder)) | indent }} - {% endif %} - {% if request_builder.parameters.headers %} - {{ op_tools.serialize(request_builder_serializer.serialize_headers(request_builder)) | indent }} - {% endif %} - {{ op_tools.serialize(request_builder_serializer.create_http_request(request_builder)) | indent }} -{% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/request_builders.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/request_builders.py.jinja2 deleted file mode 100644 index 3c72ec2ac49..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/request_builders.py.jinja2 +++ /dev/null @@ -1,10 +0,0 @@ -{% import 'operation_tools.jinja2' as op_tools %} -# coding=utf-8 -{{ code_model.options['license_header'] }} -{{ imports }} - -{{ op_tools.declare_serializer(code_model) }} -{% for request_builder in request_builders %} - -{% include "request_builder.py.jinja2" %} -{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/rest_init.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/rest_init.py.jinja2 deleted file mode 100644 index 9833f3fdc47..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/rest_init.py.jinja2 +++ /dev/null @@ -1,12 +0,0 @@ -# coding=utf-8 -{{ code_model.options['license_header'] }} - -{% for request_builder in request_builders %} -from ._request_builders import {{ request_builder.name }} -{% endfor %} - -__all__ = [ - {% for request_builder in request_builders %} - '{{ request_builder.name }}', - {% endfor %} -] diff --git a/packages/autorest.python/generator/pygen/codegen/templates/sample.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/sample.py.jinja2 deleted file mode 100644 index 421a3a7c552..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/sample.py.jinja2 +++ /dev/null @@ -1,44 +0,0 @@ -# coding=utf-8 -{% set aad_token = "DefaultAzureCredential" %} -{% set azure_key = "AzureKeyCredential" %} -{{ code_model.options['license_header'] }} - -{{ imports }} -""" -# PREREQUISITES -{% if "credential" in client_params and aad_token in client_params["credential"] %} - pip install azure-identity -{% endif %} - pip install {{ (code_model.options["package_name"] or code_model.clients[0].name)|lower }} -# USAGE - python {{ file_name }} - {% if "credential" in client_params and aad_token in client_params["credential"] %} - - Before run the sample, please set the values of the client ID, tenant ID and client secret - of the AAD application as environment variables: AZURE_CLIENT_ID, AZURE_TENANT_ID, - AZURE_CLIENT_SECRET. For more info about how to get the value, please see: - https://docs.microsoft.com/azure/active-directory/develop/howto-create-service-principal-portal - {% elif "credential" in client_params and azure_key in client_params["credential"] %} - - Before run the sample, please set environment variables AZURE_KEY with real value - which can access your service - {% endif %} -""" -def main(): - client = {{ code_model.clients[0].name }}( - {% for key,value in client_params.items() %} - {{ key }}={{ value }}, - {% endfor %} - ) - - {{ return_var }}client{{ operation_group_name }}{{ operation_name }}( - {% for key, value in operation_params.items() %} - {{ key }}={{ value|indent(8) }}, - {% endfor %} - ){{ operation_result }} - -{% if origin_file %} -# x-ms-original-file: {{ origin_file }} -{% endif %} -if __name__ == "__main__": - main() diff --git a/packages/autorest.python/generator/pygen/codegen/templates/serialization.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/serialization.py.jinja2 deleted file mode 100644 index bb5dba9d174..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/serialization.py.jinja2 +++ /dev/null @@ -1,2004 +0,0 @@ -# -------------------------------------------------------------------------- -# -# Copyright (c) {{ code_model.options["company_name"] }} Corporation. All rights reserved. -# -# The MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the ""Software""), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -# -------------------------------------------------------------------------- - -# pylint: skip-file -# pyright: reportUnnecessaryTypeIgnoreComment=false - -from base64 import b64decode, b64encode -import calendar -import datetime -import decimal -import email -from enum import Enum -import json -import logging -import re -import sys -import codecs -from typing import ( - Dict, - Any, - cast, - Optional, - Union, - AnyStr, - IO, - Mapping, - Callable, - TypeVar, - MutableMapping, - Type, - List, - Mapping, -) - -try: - from urllib import quote # type: ignore -except ImportError: - from urllib.parse import quote -import xml.etree.ElementTree as ET - -import isodate # type: ignore - -from {{ code_model.core_library }}.exceptions import DeserializationError, SerializationError -from {{ code_model.core_library }}.serialization import NULL as CoreNull - -_BOM = codecs.BOM_UTF8.decode(encoding="utf-8") - -ModelType = TypeVar("ModelType", bound="Model") -JSON = MutableMapping[str, Any] - - -class RawDeserializer: - - # Accept "text" because we're open minded people... - JSON_REGEXP = re.compile(r"^(application|text)/([a-z+.]+\+)?json$") - - # Name used in context - CONTEXT_NAME = "deserialized_data" - - @classmethod - def deserialize_from_text(cls, data: Optional[Union[AnyStr, IO]], content_type: Optional[str] = None) -> Any: - """Decode data according to content-type. - - Accept a stream of data as well, but will be load at once in memory for now. - - If no content-type, will return the string version (not bytes, not stream) - - :param data: Input, could be bytes or stream (will be decoded with UTF8) or text - :type data: str or bytes or IO - :param str content_type: The content type. - """ - if hasattr(data, "read"): - # Assume a stream - data = cast(IO, data).read() - - if isinstance(data, bytes): - data_as_str = data.decode(encoding="utf-8-sig") - else: - # Explain to mypy the correct type. - data_as_str = cast(str, data) - - # Remove Byte Order Mark if present in string - data_as_str = data_as_str.lstrip(_BOM) - - if content_type is None: - return data - - if cls.JSON_REGEXP.match(content_type): - try: - return json.loads(data_as_str) - except ValueError as err: - raise DeserializationError("JSON is invalid: {}".format(err), err) - elif "xml" in (content_type or []): - try: - - try: - if isinstance(data, unicode): # type: ignore - # If I'm Python 2.7 and unicode XML will scream if I try a "fromstring" on unicode string - data_as_str = data_as_str.encode(encoding="utf-8") # type: ignore - except NameError: - pass - - return ET.fromstring(data_as_str) # nosec - except ET.ParseError as err: - # It might be because the server has an issue, and returned JSON with - # content-type XML.... - # So let's try a JSON load, and if it's still broken - # let's flow the initial exception - def _json_attemp(data): - try: - return True, json.loads(data) - except ValueError: - return False, None # Don't care about this one - - success, json_result = _json_attemp(data) - if success: - return json_result - # If i'm here, it's not JSON, it's not XML, let's scream - # and raise the last context in this block (the XML exception) - # The function hack is because Py2.7 messes up with exception - # context otherwise. - _LOGGER.critical("Wasn't XML not JSON, failing") - raise DeserializationError("XML is invalid") from err - raise DeserializationError("Cannot deserialize content-type: {}".format(content_type)) - - @classmethod - def deserialize_from_http_generics(cls, body_bytes: Optional[Union[AnyStr, IO]], headers: Mapping) -> Any: - """Deserialize from HTTP response. - - Use bytes and headers to NOT use any requests/aiohttp or whatever - specific implementation. - Headers will tested for "content-type" - """ - # Try to use content-type from headers if available - content_type = None - if "content-type" in headers: - content_type = headers["content-type"].split(";")[0].strip().lower() - # Ouch, this server did not declare what it sent... - # Let's guess it's JSON... - # Also, since Autorest was considering that an empty body was a valid JSON, - # need that test as well.... - else: - content_type = "application/json" - - if body_bytes: - return cls.deserialize_from_text(body_bytes, content_type) - return None - - -_LOGGER = logging.getLogger(__name__) - -try: - _long_type = long # type: ignore -except NameError: - _long_type = int - - -class UTC(datetime.tzinfo): - """Time Zone info for handling UTC""" - - def utcoffset(self, dt): - """UTF offset for UTC is 0.""" - return datetime.timedelta(0) - - def tzname(self, dt): - """Timestamp representation.""" - return "Z" - - def dst(self, dt): - """No daylight saving for UTC.""" - return datetime.timedelta(hours=1) - - -try: - from datetime import timezone as _FixedOffset # type: ignore -except ImportError: # Python 2.7 - - class _FixedOffset(datetime.tzinfo): # type: ignore - """Fixed offset in minutes east from UTC. - Copy/pasted from Python doc - :param datetime.timedelta offset: offset in timedelta format - """ - - def __init__(self, offset): - self.__offset = offset - - def utcoffset(self, dt): - return self.__offset - - def tzname(self, dt): - return str(self.__offset.total_seconds() / 3600) - - def __repr__(self): - return "".format(self.tzname(None)) - - def dst(self, dt): - return datetime.timedelta(0) - - def __getinitargs__(self): - return (self.__offset,) - - -try: - from datetime import timezone - - TZ_UTC = timezone.utc -except ImportError: - TZ_UTC = UTC() # type: ignore - -_FLATTEN = re.compile(r"(? None: - self.additional_properties: Optional[Dict[str, Any]] = {} - for k in kwargs: - if k not in self._attribute_map: - _LOGGER.warning("%s is not a known attribute of class %s and will be ignored", k, self.__class__) - elif k in self._validation and self._validation[k].get("readonly", False): - _LOGGER.warning("Readonly attribute %s will be ignored in class %s", k, self.__class__) - else: - setattr(self, k, kwargs[k]) - - def __eq__(self, other: Any) -> bool: - """Compare objects by comparing all attributes.""" - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - return False - - def __ne__(self, other: Any) -> bool: - """Compare objects by comparing all attributes.""" - return not self.__eq__(other) - - def __str__(self) -> str: - return str(self.__dict__) - - @classmethod - def enable_additional_properties_sending(cls) -> None: - cls._attribute_map["additional_properties"] = {"key": "", "type": "{object}"} - - @classmethod - def is_xml_model(cls) -> bool: - try: - cls._xml_map # type: ignore - except AttributeError: - return False - return True - - @classmethod - def _create_xml_node(cls): - """Create XML node.""" - try: - xml_map = cls._xml_map # type: ignore - except AttributeError: - xml_map = {} - - return _create_xml_node(xml_map.get("name", cls.__name__), xml_map.get("prefix", None), xml_map.get("ns", None)) - - def serialize(self, keep_readonly: bool = False, **kwargs: Any) -> JSON: - """Return the JSON that would be sent to server from this model. - - This is an alias to `as_dict(full_restapi_key_transformer, keep_readonly=False)`. - - If you want XML serialization, you can pass the kwargs is_xml=True. - - :param bool keep_readonly: If you want to serialize the readonly attributes - :returns: A dict JSON compatible object - :rtype: dict - """ - serializer = Serializer(self._infer_class_models()) - return serializer._serialize(self, keep_readonly=keep_readonly, **kwargs) # type: ignore - - def as_dict( - self, - keep_readonly: bool = True, - key_transformer: Callable[ - [str, Dict[str, Any], Any], Any - ] = attribute_transformer, - **kwargs: Any - ) -> JSON: - """Return a dict that can be serialized using json.dump. - - Advanced usage might optionally use a callback as parameter: - - .. code::python - - def my_key_transformer(key, attr_desc, value): - return key - - Key is the attribute name used in Python. Attr_desc - is a dict of metadata. Currently contains 'type' with the - msrest type and 'key' with the RestAPI encoded key. - Value is the current value in this object. - - The string returned will be used to serialize the key. - If the return type is a list, this is considered hierarchical - result dict. - - See the three examples in this file: - - - attribute_transformer - - full_restapi_key_transformer - - last_restapi_key_transformer - - If you want XML serialization, you can pass the kwargs is_xml=True. - - :param function key_transformer: A key transformer function. - :returns: A dict JSON compatible object - :rtype: dict - """ - serializer = Serializer(self._infer_class_models()) - return serializer._serialize(self, key_transformer=key_transformer, keep_readonly=keep_readonly, **kwargs) # type: ignore - - @classmethod - def _infer_class_models(cls): - try: - str_models = cls.__module__.rsplit(".", 1)[0] - models = sys.modules[str_models] - client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)} - if cls.__name__ not in client_models: - raise ValueError("Not Autorest generated code") - except Exception: - # Assume it's not Autorest generated (tests?). Add ourselves as dependencies. - client_models = {cls.__name__: cls} - return client_models - - @classmethod - def deserialize(cls: Type[ModelType], data: Any, content_type: Optional[str] = None) -> ModelType: - """Parse a str using the RestAPI syntax and return a model. - - :param str data: A str using RestAPI structure. JSON by default. - :param str content_type: JSON by default, set application/xml if XML. - :returns: An instance of this model - :raises: DeserializationError if something went wrong - """ - deserializer = Deserializer(cls._infer_class_models()) - return deserializer(cls.__name__, data, content_type=content_type) # type: ignore - - @classmethod - def from_dict( - cls: Type[ModelType], - data: Any, - key_extractors: Optional[Callable[[str, Dict[str, Any], Any], Any]] = None, - content_type: Optional[str] = None, - ) -> ModelType: - """Parse a dict using given key extractor return a model. - - By default consider key - extractors (rest_key_case_insensitive_extractor, attribute_key_case_insensitive_extractor - and last_rest_key_case_insensitive_extractor) - - :param dict data: A dict using RestAPI structure - :param str content_type: JSON by default, set application/xml if XML. - :returns: An instance of this model - :raises: DeserializationError if something went wrong - """ - deserializer = Deserializer(cls._infer_class_models()) - deserializer.key_extractors = ( # type: ignore - [ # type: ignore - attribute_key_case_insensitive_extractor, - rest_key_case_insensitive_extractor, - last_rest_key_case_insensitive_extractor, - ] - if key_extractors is None - else key_extractors - ) - return deserializer(cls.__name__, data, content_type=content_type) # type: ignore - - @classmethod - def _flatten_subtype(cls, key, objects): - if "_subtype_map" not in cls.__dict__: - return {} - result = dict(cls._subtype_map[key]) - for valuetype in cls._subtype_map[key].values(): - result.update(objects[valuetype]._flatten_subtype(key, objects)) - return result - - @classmethod - def _classify(cls, response, objects): - """Check the class _subtype_map for any child classes. - We want to ignore any inherited _subtype_maps. - Remove the polymorphic key from the initial data. - """ - for subtype_key in cls.__dict__.get("_subtype_map", {}).keys(): - subtype_value = None - - if not isinstance(response, ET.Element): - rest_api_response_key = cls._get_rest_key_parts(subtype_key)[-1] - subtype_value = response.pop(rest_api_response_key, None) or response.pop(subtype_key, None) - else: - subtype_value = xml_key_extractor(subtype_key, cls._attribute_map[subtype_key], response) - if subtype_value: - # Try to match base class. Can be class name only - # (bug to fix in Autorest to support x-ms-discriminator-name) - if cls.__name__ == subtype_value: - return cls - flatten_mapping_type = cls._flatten_subtype(subtype_key, objects) - try: - return objects[flatten_mapping_type[subtype_value]] # type: ignore - except KeyError: - _LOGGER.warning( - "Subtype value %s has no mapping, use base class %s.", - subtype_value, - cls.__name__, - ) - break - else: - _LOGGER.warning("Discriminator %s is absent or null, use base class %s.", subtype_key, cls.__name__) - break - return cls - - @classmethod - def _get_rest_key_parts(cls, attr_key): - """Get the RestAPI key of this attr, split it and decode part - :param str attr_key: Attribute key must be in attribute_map. - :returns: A list of RestAPI part - :rtype: list - """ - rest_split_key = _FLATTEN.split(cls._attribute_map[attr_key]["key"]) - return [_decode_attribute_map_key(key_part) for key_part in rest_split_key] - - -def _decode_attribute_map_key(key): - """This decode a key in an _attribute_map to the actual key we want to look at - inside the received data. - - :param str key: A key string from the generated code - """ - return key.replace("\\.", ".") - - -class Serializer(object): - """Request object model serializer.""" - - basic_types = {str: "str", int: "int", bool: "bool", float: "float"} - - _xml_basic_types_serializers = {"bool": lambda x: str(x).lower()} - days = {0: "Mon", 1: "Tue", 2: "Wed", 3: "Thu", 4: "Fri", 5: "Sat", 6: "Sun"} - months = { - 1: "Jan", - 2: "Feb", - 3: "Mar", - 4: "Apr", - 5: "May", - 6: "Jun", - 7: "Jul", - 8: "Aug", - 9: "Sep", - 10: "Oct", - 11: "Nov", - 12: "Dec", - } - validation = { - "min_length": lambda x, y: len(x) < y, - "max_length": lambda x, y: len(x) > y, - "minimum": lambda x, y: x < y, - "maximum": lambda x, y: x > y, - "minimum_ex": lambda x, y: x <= y, - "maximum_ex": lambda x, y: x >= y, - "min_items": lambda x, y: len(x) < y, - "max_items": lambda x, y: len(x) > y, - "pattern": lambda x, y: not re.match(y, x, re.UNICODE), - "unique": lambda x, y: len(x) != len(set(x)), - "multiple": lambda x, y: x % y != 0, - } - - def __init__(self, classes: Optional[Mapping[str, type]]=None): - self.serialize_type = { - "iso-8601": Serializer.serialize_iso, - "rfc-1123": Serializer.serialize_rfc, - "unix-time": Serializer.serialize_unix, - "duration": Serializer.serialize_duration, - "date": Serializer.serialize_date, - "time": Serializer.serialize_time, - "decimal": Serializer.serialize_decimal, - "long": Serializer.serialize_long, - "bytearray": Serializer.serialize_bytearray, - "base64": Serializer.serialize_base64, - "object": self.serialize_object, - "[]": self.serialize_iter, - "{}": self.serialize_dict, - } - self.dependencies: Dict[str, type] = dict(classes) if classes else {} - self.key_transformer = full_restapi_key_transformer - self.client_side_validation = True - - def _serialize(self, target_obj, data_type=None, **kwargs): - """Serialize data into a string according to type. - - :param target_obj: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str, dict - :raises: SerializationError if serialization fails. - """ - key_transformer = kwargs.get("key_transformer", self.key_transformer) - keep_readonly = kwargs.get("keep_readonly", False) - if target_obj is None: - return None - - attr_name = None - class_name = target_obj.__class__.__name__ - - if data_type: - return self.serialize_data(target_obj, data_type, **kwargs) - - if not hasattr(target_obj, "_attribute_map"): - data_type = type(target_obj).__name__ - if data_type in self.basic_types.values(): - return self.serialize_data(target_obj, data_type, **kwargs) - - # Force "is_xml" kwargs if we detect a XML model - try: - is_xml_model_serialization = kwargs["is_xml"] - except KeyError: - is_xml_model_serialization = kwargs.setdefault("is_xml", target_obj.is_xml_model()) - - serialized = {} - if is_xml_model_serialization: - serialized = target_obj._create_xml_node() - try: - attributes = target_obj._attribute_map - for attr, attr_desc in attributes.items(): - attr_name = attr - if not keep_readonly and target_obj._validation.get(attr_name, {}).get("readonly", False): - continue - - if attr_name == "additional_properties" and attr_desc["key"] == "": - if target_obj.additional_properties is not None: - serialized.update(target_obj.additional_properties) - continue - try: - - orig_attr = getattr(target_obj, attr) - if is_xml_model_serialization: - pass # Don't provide "transformer" for XML for now. Keep "orig_attr" - else: # JSON - keys, orig_attr = key_transformer(attr, attr_desc.copy(), orig_attr) - keys = keys if isinstance(keys, list) else [keys] - - kwargs["serialization_ctxt"] = attr_desc - new_attr = self.serialize_data(orig_attr, attr_desc["type"], **kwargs) - - if is_xml_model_serialization: - xml_desc = attr_desc.get("xml", {}) - xml_name = xml_desc.get("name", attr_desc["key"]) - xml_prefix = xml_desc.get("prefix", None) - xml_ns = xml_desc.get("ns", None) - if xml_desc.get("attr", False): - if xml_ns: - ET.register_namespace(xml_prefix, xml_ns) - xml_name = "{% raw %}{{{}}}{}{% endraw %}".format(xml_ns, xml_name) - serialized.set(xml_name, new_attr) # type: ignore - continue - if xml_desc.get("text", False): - serialized.text = new_attr # type: ignore - continue - if isinstance(new_attr, list): - serialized.extend(new_attr) # type: ignore - elif isinstance(new_attr, ET.Element): - # If the down XML has no XML/Name, we MUST replace the tag with the local tag. But keeping the namespaces. - if "name" not in getattr(orig_attr, "_xml_map", {}): - splitted_tag = new_attr.tag.split("}") - if len(splitted_tag) == 2: # Namespace - new_attr.tag = "}".join([splitted_tag[0], xml_name]) - else: - new_attr.tag = xml_name - serialized.append(new_attr) # type: ignore - else: # That's a basic type - # Integrate namespace if necessary - local_node = _create_xml_node(xml_name, xml_prefix, xml_ns) - local_node.text = str(new_attr) - serialized.append(local_node) # type: ignore - else: # JSON - for k in reversed(keys): # type: ignore - new_attr = {k: new_attr} - - _new_attr = new_attr - _serialized = serialized - for k in keys: # type: ignore - if k not in _serialized: - _serialized.update(_new_attr) # type: ignore - _new_attr = _new_attr[k] # type: ignore - _serialized = _serialized[k] - except ValueError as err: - if isinstance(err, SerializationError): - raise - - except (AttributeError, KeyError, TypeError) as err: - msg = "Attribute {} in object {} cannot be serialized.\n{}".format(attr_name, class_name, str(target_obj)) - raise SerializationError(msg) from err - else: - return serialized - - def body(self, data, data_type, **kwargs): - """Serialize data intended for a request body. - - :param data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: dict - :raises: SerializationError if serialization fails. - :raises: ValueError if data is None - """ - - # Just in case this is a dict - internal_data_type_str = data_type.strip("[]{}") - internal_data_type = self.dependencies.get(internal_data_type_str, None) - try: - is_xml_model_serialization = kwargs["is_xml"] - except KeyError: - if internal_data_type and issubclass(internal_data_type, Model): - is_xml_model_serialization = kwargs.setdefault("is_xml", internal_data_type.is_xml_model()) - else: - is_xml_model_serialization = False - if internal_data_type and not isinstance(internal_data_type, Enum): - try: - deserializer = Deserializer(self.dependencies) - # Since it's on serialization, it's almost sure that format is not JSON REST - # We're not able to deal with additional properties for now. - deserializer.additional_properties_detection = False - if is_xml_model_serialization: - deserializer.key_extractors = [ # type: ignore - attribute_key_case_insensitive_extractor, - ] - else: - deserializer.key_extractors = [ - rest_key_case_insensitive_extractor, - attribute_key_case_insensitive_extractor, - last_rest_key_case_insensitive_extractor, - ] - data = deserializer._deserialize(data_type, data) - except DeserializationError as err: - raise SerializationError("Unable to build a model: " + str(err)) from err - - return self._serialize(data, data_type, **kwargs) - - def url(self, name, data, data_type, **kwargs): - """Serialize data intended for a URL path. - - :param data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str - :raises: TypeError if serialization fails. - :raises: ValueError if data is None - """ - try: - output = self.serialize_data(data, data_type, **kwargs) - if data_type == "bool": - output = json.dumps(output) - - if kwargs.get("skip_quote") is True: - output = str(output) - output = output.replace("{", quote("{")).replace("}", quote("}")) - else: - output = quote(str(output), safe="") - except SerializationError: - raise TypeError("{} must be type {}.".format(name, data_type)) - else: - return output - - def query(self, name, data, data_type, **kwargs): - """Serialize data intended for a URL query. - - :param data: The data to be serialized. - :param str data_type: The type to be serialized from. - :keyword bool skip_quote: Whether to skip quote the serialized result. - Defaults to False. - :rtype: str, list - :raises: TypeError if serialization fails. - :raises: ValueError if data is None - """ - try: - # Treat the list aside, since we don't want to encode the div separator - if data_type.startswith("["): - internal_data_type = data_type[1:-1] - do_quote = not kwargs.get('skip_quote', False) - return self.serialize_iter(data, internal_data_type, do_quote=do_quote, **kwargs) - - # Not a list, regular serialization - output = self.serialize_data(data, data_type, **kwargs) - if data_type == "bool": - output = json.dumps(output) - if kwargs.get("skip_quote") is True: - output = str(output) - else: - output = quote(str(output), safe="") - except SerializationError: - raise TypeError("{} must be type {}.".format(name, data_type)) - else: - return str(output) - - def header(self, name, data, data_type, **kwargs): - """Serialize data intended for a request header. - - :param data: The data to be serialized. - :param str data_type: The type to be serialized from. - :rtype: str - :raises: TypeError if serialization fails. - :raises: ValueError if data is None - """ - try: - if data_type in ["[str]"]: - data = ["" if d is None else d for d in data] - - output = self.serialize_data(data, data_type, **kwargs) - if data_type == "bool": - output = json.dumps(output) - except SerializationError: - raise TypeError("{} must be type {}.".format(name, data_type)) - else: - return str(output) - - def serialize_data(self, data, data_type, **kwargs): - """Serialize generic data according to supplied data type. - - :param data: The data to be serialized. - :param str data_type: The type to be serialized from. - :param bool required: Whether it's essential that the data not be - empty or None - :raises: AttributeError if required data is None. - :raises: ValueError if data is None - :raises: SerializationError if serialization fails. - """ - if data is None: - raise ValueError("No value for given attribute") - - try: - if data is CoreNull: - return None - if data_type in self.basic_types.values(): - return self.serialize_basic(data, data_type, **kwargs) - - elif data_type in self.serialize_type: - return self.serialize_type[data_type](data, **kwargs) - - # If dependencies is empty, try with current data class - # It has to be a subclass of Enum anyway - enum_type = self.dependencies.get(data_type, data.__class__) - if issubclass(enum_type, Enum): - return Serializer.serialize_enum(data, enum_obj=enum_type) - - iter_type = data_type[0] + data_type[-1] - if iter_type in self.serialize_type: - return self.serialize_type[iter_type](data, data_type[1:-1], **kwargs) - - except (ValueError, TypeError) as err: - msg = "Unable to serialize value: {!r} as type: {!r}." - raise SerializationError(msg.format(data, data_type)) from err - else: - return self._serialize(data, **kwargs) - - @classmethod - def _get_custom_serializers(cls, data_type, **kwargs): - custom_serializer = kwargs.get("basic_types_serializers", {}).get(data_type) - if custom_serializer: - return custom_serializer - if kwargs.get("is_xml", False): - return cls._xml_basic_types_serializers.get(data_type) - - @classmethod - def serialize_basic(cls, data, data_type, **kwargs): - """Serialize basic builting data type. - Serializes objects to str, int, float or bool. - - Possible kwargs: - - basic_types_serializers dict[str, callable] : If set, use the callable as serializer - - is_xml bool : If set, use xml_basic_types_serializers - - :param data: Object to be serialized. - :param str data_type: Type of object in the iterable. - """ - custom_serializer = cls._get_custom_serializers(data_type, **kwargs) - if custom_serializer: - return custom_serializer(data) - if data_type == "str": - return cls.serialize_unicode(data) - return eval(data_type)(data) # nosec - - @classmethod - def serialize_unicode(cls, data): - """Special handling for serializing unicode strings in Py2. - Encode to UTF-8 if unicode, otherwise handle as a str. - - :param data: Object to be serialized. - :rtype: str - """ - try: # If I received an enum, return its value - return data.value - except AttributeError: - pass - - try: - if isinstance(data, unicode): # type: ignore - # Don't change it, JSON and XML ElementTree are totally able - # to serialize correctly u'' strings - return data - except NameError: - return str(data) - else: - return str(data) - - def serialize_iter(self, data, iter_type, div=None, **kwargs): - """Serialize iterable. - - Supported kwargs: - - serialization_ctxt dict : The current entry of _attribute_map, or same format. - serialization_ctxt['type'] should be same as data_type. - - is_xml bool : If set, serialize as XML - - :param list attr: Object to be serialized. - :param str iter_type: Type of object in the iterable. - :param bool required: Whether the objects in the iterable must - not be None or empty. - :param str div: If set, this str will be used to combine the elements - in the iterable into a combined string. Default is 'None'. - :keyword bool do_quote: Whether to quote the serialized result of each iterable element. - Defaults to False. - :rtype: list, str - """ - if isinstance(data, str): - raise SerializationError("Refuse str type as a valid iter type.") - - serialization_ctxt = kwargs.get("serialization_ctxt", {}) - is_xml = kwargs.get("is_xml", False) - - serialized = [] - for d in data: - try: - serialized.append(self.serialize_data(d, iter_type, **kwargs)) - except ValueError as err: - if isinstance(err, SerializationError): - raise - serialized.append(None) - - if kwargs.get('do_quote', False): - serialized = [ - '' if s is None else quote(str(s), safe='') - for s - in serialized - ] - - if div: - serialized = ["" if s is None else str(s) for s in serialized] - serialized = div.join(serialized) - - if "xml" in serialization_ctxt or is_xml: - # XML serialization is more complicated - xml_desc = serialization_ctxt.get("xml", {}) - xml_name = xml_desc.get("name") - if not xml_name: - xml_name = serialization_ctxt["key"] - - # Create a wrap node if necessary (use the fact that Element and list have "append") - is_wrapped = xml_desc.get("wrapped", False) - node_name = xml_desc.get("itemsName", xml_name) - if is_wrapped: - final_result = _create_xml_node(xml_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) - else: - final_result = [] - # All list elements to "local_node" - for el in serialized: - if isinstance(el, ET.Element): - el_node = el - else: - el_node = _create_xml_node(node_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) - if el is not None: # Otherwise it writes "None" :-p - el_node.text = str(el) - final_result.append(el_node) - return final_result - return serialized - - def serialize_dict(self, attr, dict_type, **kwargs): - """Serialize a dictionary of objects. - - :param dict attr: Object to be serialized. - :param str dict_type: Type of object in the dictionary. - :param bool required: Whether the objects in the dictionary must - not be None or empty. - :rtype: dict - """ - serialization_ctxt = kwargs.get("serialization_ctxt", {}) - serialized = {} - for key, value in attr.items(): - try: - serialized[self.serialize_unicode(key)] = self.serialize_data(value, dict_type, **kwargs) - except ValueError as err: - if isinstance(err, SerializationError): - raise - serialized[self.serialize_unicode(key)] = None - - if "xml" in serialization_ctxt: - # XML serialization is more complicated - xml_desc = serialization_ctxt["xml"] - xml_name = xml_desc["name"] - - final_result = _create_xml_node(xml_name, xml_desc.get("prefix", None), xml_desc.get("ns", None)) - for key, value in serialized.items(): - ET.SubElement(final_result, key).text = value - return final_result - - return serialized - - def serialize_object(self, attr, **kwargs): - """Serialize a generic object. - This will be handled as a dictionary. If object passed in is not - a basic type (str, int, float, dict, list) it will simply be - cast to str. - - :param dict attr: Object to be serialized. - :rtype: dict or str - """ - if attr is None: - return None - if isinstance(attr, ET.Element): - return attr - obj_type = type(attr) - if obj_type in self.basic_types: - return self.serialize_basic(attr, self.basic_types[obj_type], **kwargs) - if obj_type is _long_type: - return self.serialize_long(attr) - if obj_type is str: - return self.serialize_unicode(attr) - if obj_type is datetime.datetime: - return self.serialize_iso(attr) - if obj_type is datetime.date: - return self.serialize_date(attr) - if obj_type is datetime.time: - return self.serialize_time(attr) - if obj_type is datetime.timedelta: - return self.serialize_duration(attr) - if obj_type is decimal.Decimal: - return self.serialize_decimal(attr) - - # If it's a model or I know this dependency, serialize as a Model - elif obj_type in self.dependencies.values() or isinstance(attr, Model): - return self._serialize(attr) - - if obj_type == dict: - serialized = {} - for key, value in attr.items(): - try: - serialized[self.serialize_unicode(key)] = self.serialize_object(value, **kwargs) - except ValueError: - serialized[self.serialize_unicode(key)] = None - return serialized - - if obj_type == list: - serialized = [] - for obj in attr: - try: - serialized.append(self.serialize_object(obj, **kwargs)) - except ValueError: - pass - return serialized - return str(attr) - - @staticmethod - def serialize_enum(attr, enum_obj=None): - try: - result = attr.value - except AttributeError: - result = attr - try: - enum_obj(result) # type: ignore - return result - except ValueError: - for enum_value in enum_obj: # type: ignore - if enum_value.value.lower() == str(attr).lower(): - return enum_value.value - error = "{!r} is not valid value for enum {!r}" - raise SerializationError(error.format(attr, enum_obj)) - - @staticmethod - def serialize_bytearray(attr, **kwargs): - """Serialize bytearray into base-64 string. - - :param attr: Object to be serialized. - :rtype: str - """ - return b64encode(attr).decode() - - @staticmethod - def serialize_base64(attr, **kwargs): - """Serialize str into base-64 string. - - :param attr: Object to be serialized. - :rtype: str - """ - encoded = b64encode(attr).decode("ascii") - return encoded.strip("=").replace("+", "-").replace("/", "_") - - @staticmethod - def serialize_decimal(attr, **kwargs): - """Serialize Decimal object to float. - - :param attr: Object to be serialized. - :rtype: float - """ - return float(attr) - - @staticmethod - def serialize_long(attr, **kwargs): - """Serialize long (Py2) or int (Py3). - - :param attr: Object to be serialized. - :rtype: int/long - """ - return _long_type(attr) - - @staticmethod - def serialize_date(attr, **kwargs): - """Serialize Date object into ISO-8601 formatted string. - - :param Date attr: Object to be serialized. - :rtype: str - """ - if isinstance(attr, str): - attr = isodate.parse_date(attr) - t = "{:04}-{:02}-{:02}".format(attr.year, attr.month, attr.day) - return t - - @staticmethod - def serialize_time(attr, **kwargs): - """Serialize Time object into ISO-8601 formatted string. - - :param datetime.time attr: Object to be serialized. - :rtype: str - """ - if isinstance(attr, str): - attr = isodate.parse_time(attr) - t = "{:02}:{:02}:{:02}".format(attr.hour, attr.minute, attr.second) - if attr.microsecond: - t += ".{:02}".format(attr.microsecond) - return t - - @staticmethod - def serialize_duration(attr, **kwargs): - """Serialize TimeDelta object into ISO-8601 formatted string. - - :param TimeDelta attr: Object to be serialized. - :rtype: str - """ - if isinstance(attr, str): - attr = isodate.parse_duration(attr) - return isodate.duration_isoformat(attr) - - @staticmethod - def serialize_rfc(attr, **kwargs): - """Serialize Datetime object into RFC-1123 formatted string. - - :param Datetime attr: Object to be serialized. - :rtype: str - :raises: TypeError if format invalid. - """ - try: - if not attr.tzinfo: - _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") - utc = attr.utctimetuple() - except AttributeError: - raise TypeError("RFC1123 object must be valid Datetime object.") - - return "{}, {:02} {} {:04} {:02}:{:02}:{:02} GMT".format( - Serializer.days[utc.tm_wday], - utc.tm_mday, - Serializer.months[utc.tm_mon], - utc.tm_year, - utc.tm_hour, - utc.tm_min, - utc.tm_sec, - ) - - @staticmethod - def serialize_iso(attr, **kwargs): - """Serialize Datetime object into ISO-8601 formatted string. - - :param Datetime attr: Object to be serialized. - :rtype: str - :raises: SerializationError if format invalid. - """ - if isinstance(attr, str): - attr = isodate.parse_datetime(attr) - try: - if not attr.tzinfo: - _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") - utc = attr.utctimetuple() - if utc.tm_year > 9999 or utc.tm_year < 1: - raise OverflowError("Hit max or min date") - - microseconds = str(attr.microsecond).rjust(6, "0").rstrip("0").ljust(3, "0") - if microseconds: - microseconds = "." + microseconds - date = "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}".format( - utc.tm_year, utc.tm_mon, utc.tm_mday, utc.tm_hour, utc.tm_min, utc.tm_sec - ) - return date + microseconds + "Z" - except (ValueError, OverflowError) as err: - msg = "Unable to serialize datetime object." - raise SerializationError(msg) from err - except AttributeError as err: - msg = "ISO-8601 object must be valid Datetime object." - raise TypeError(msg) from err - - @staticmethod - def serialize_unix(attr, **kwargs): - """Serialize Datetime object into IntTime format. - This is represented as seconds. - - :param Datetime attr: Object to be serialized. - :rtype: int - :raises: SerializationError if format invalid - """ - if isinstance(attr, int): - return attr - try: - if not attr.tzinfo: - _LOGGER.warning("Datetime with no tzinfo will be considered UTC.") - return int(calendar.timegm(attr.utctimetuple())) - except AttributeError: - raise TypeError("Unix time object must be valid Datetime object.") - - -def rest_key_extractor(attr, attr_desc, data): - key = attr_desc["key"] - working_data = data - - while "." in key: - # Need the cast, as for some reasons "split" is typed as list[str | Any] - dict_keys = cast(List[str], _FLATTEN.split(key)) - if len(dict_keys) == 1: - key = _decode_attribute_map_key(dict_keys[0]) - break - working_key = _decode_attribute_map_key(dict_keys[0]) - working_data = working_data.get(working_key, data) - if working_data is None: - # If at any point while following flatten JSON path see None, it means - # that all properties under are None as well - return None - key = ".".join(dict_keys[1:]) - - return working_data.get(key) - - -def rest_key_case_insensitive_extractor(attr, attr_desc, data): - key = attr_desc["key"] - working_data = data - - while "." in key: - dict_keys = _FLATTEN.split(key) - if len(dict_keys) == 1: - key = _decode_attribute_map_key(dict_keys[0]) - break - working_key = _decode_attribute_map_key(dict_keys[0]) - working_data = attribute_key_case_insensitive_extractor(working_key, None, working_data) - if working_data is None: - # If at any point while following flatten JSON path see None, it means - # that all properties under are None as well - return None - key = ".".join(dict_keys[1:]) - - if working_data: - return attribute_key_case_insensitive_extractor(key, None, working_data) - - -def last_rest_key_extractor(attr, attr_desc, data): - """Extract the attribute in "data" based on the last part of the JSON path key.""" - key = attr_desc["key"] - dict_keys = _FLATTEN.split(key) - return attribute_key_extractor(dict_keys[-1], None, data) - - -def last_rest_key_case_insensitive_extractor(attr, attr_desc, data): - """Extract the attribute in "data" based on the last part of the JSON path key. - - This is the case insensitive version of "last_rest_key_extractor" - """ - key = attr_desc["key"] - dict_keys = _FLATTEN.split(key) - return attribute_key_case_insensitive_extractor(dict_keys[-1], None, data) - - -def attribute_key_extractor(attr, _, data): - return data.get(attr) - - -def attribute_key_case_insensitive_extractor(attr, _, data): - found_key = None - lower_attr = attr.lower() - for key in data: - if lower_attr == key.lower(): - found_key = key - break - - return data.get(found_key) - - -def _extract_name_from_internal_type(internal_type): - """Given an internal type XML description, extract correct XML name with namespace. - - :param dict internal_type: An model type - :rtype: tuple - :returns: A tuple XML name + namespace dict - """ - internal_type_xml_map = getattr(internal_type, "_xml_map", {}) - xml_name = internal_type_xml_map.get("name", internal_type.__name__) - xml_ns = internal_type_xml_map.get("ns", None) - if xml_ns: - xml_name = "{% raw %}{{{}}}{}{% endraw %}".format(xml_ns, xml_name) - return xml_name - - -def xml_key_extractor(attr, attr_desc, data): - if isinstance(data, dict): - return None - - # Test if this model is XML ready first - if not isinstance(data, ET.Element): - return None - - xml_desc = attr_desc.get("xml", {}) - xml_name = xml_desc.get("name", attr_desc["key"]) - - # Look for a children - is_iter_type = attr_desc["type"].startswith("[") - is_wrapped = xml_desc.get("wrapped", False) - internal_type = attr_desc.get("internalType", None) - internal_type_xml_map = getattr(internal_type, "_xml_map", {}) - - # Integrate namespace if necessary - xml_ns = xml_desc.get("ns", internal_type_xml_map.get("ns", None)) - if xml_ns: - xml_name = "{% raw %}{{{}}}{}{% endraw %}".format(xml_ns, xml_name) - - # If it's an attribute, that's simple - if xml_desc.get("attr", False): - return data.get(xml_name) - - # If it's x-ms-text, that's simple too - if xml_desc.get("text", False): - return data.text - - # Scenario where I take the local name: - # - Wrapped node - # - Internal type is an enum (considered basic types) - # - Internal type has no XML/Name node - if is_wrapped or (internal_type and (issubclass(internal_type, Enum) or "name" not in internal_type_xml_map)): - children = data.findall(xml_name) - # If internal type has a local name and it's not a list, I use that name - elif not is_iter_type and internal_type and "name" in internal_type_xml_map: - xml_name = _extract_name_from_internal_type(internal_type) - children = data.findall(xml_name) - # That's an array - else: - if internal_type: # Complex type, ignore itemsName and use the complex type name - items_name = _extract_name_from_internal_type(internal_type) - else: - items_name = xml_desc.get("itemsName", xml_name) - children = data.findall(items_name) - - if len(children) == 0: - if is_iter_type: - if is_wrapped: - return None # is_wrapped no node, we want None - else: - return [] # not wrapped, assume empty list - return None # Assume it's not there, maybe an optional node. - - # If is_iter_type and not wrapped, return all found children - if is_iter_type: - if not is_wrapped: - return children - else: # Iter and wrapped, should have found one node only (the wrap one) - if len(children) != 1: - raise DeserializationError( - "Tried to deserialize an array not wrapped, and found several nodes '{}'. Maybe you should declare this array as wrapped?".format( - xml_name - ) - ) - return list(children[0]) # Might be empty list and that's ok. - - # Here it's not a itertype, we should have found one element only or empty - if len(children) > 1: - raise DeserializationError("Find several XML '{}' where it was not expected".format(xml_name)) - return children[0] - - -class Deserializer(object): - """Response object model deserializer. - - :param dict classes: Class type dictionary for deserializing complex types. - :ivar list key_extractors: Ordered list of extractors to be used by this deserializer. - """ - - basic_types = {str: "str", int: "int", bool: "bool", float: "float"} - - valid_date = re.compile(r"\d{4}[-]\d{2}[-]\d{2}T\d{2}:\d{2}:\d{2}" r"\.?\d*Z?[-+]?[\d{2}]?:?[\d{2}]?") - - def __init__(self, classes: Optional[Mapping[str, type]]=None): - self.deserialize_type = { - "iso-8601": Deserializer.deserialize_iso, - "rfc-1123": Deserializer.deserialize_rfc, - "unix-time": Deserializer.deserialize_unix, - "duration": Deserializer.deserialize_duration, - "date": Deserializer.deserialize_date, - "time": Deserializer.deserialize_time, - "decimal": Deserializer.deserialize_decimal, - "long": Deserializer.deserialize_long, - "bytearray": Deserializer.deserialize_bytearray, - "base64": Deserializer.deserialize_base64, - "object": self.deserialize_object, - "[]": self.deserialize_iter, - "{}": self.deserialize_dict, - } - self.deserialize_expected_types = { - "duration": (isodate.Duration, datetime.timedelta), - "iso-8601": (datetime.datetime), - } - self.dependencies: Dict[str, type] = dict(classes) if classes else {} - self.key_extractors = [rest_key_extractor, xml_key_extractor] - # Additional properties only works if the "rest_key_extractor" is used to - # extract the keys. Making it to work whatever the key extractor is too much - # complicated, with no real scenario for now. - # So adding a flag to disable additional properties detection. This flag should be - # used if your expect the deserialization to NOT come from a JSON REST syntax. - # Otherwise, result are unexpected - self.additional_properties_detection = True - - def __call__(self, target_obj, response_data, content_type=None): - """Call the deserializer to process a REST response. - - :param str target_obj: Target data type to deserialize to. - :param requests.Response response_data: REST response object. - :param str content_type: Swagger "produces" if available. - :raises: DeserializationError if deserialization fails. - :return: Deserialized object. - """ - data = self._unpack_content(response_data, content_type) - return self._deserialize(target_obj, data) - - def _deserialize(self, target_obj, data): - """Call the deserializer on a model. - - Data needs to be already deserialized as JSON or XML ElementTree - - :param str target_obj: Target data type to deserialize to. - :param object data: Object to deserialize. - :raises: DeserializationError if deserialization fails. - :return: Deserialized object. - """ - # This is already a model, go recursive just in case - if hasattr(data, "_attribute_map"): - constants = [name for name, config in getattr(data, "_validation", {}).items() if config.get("constant")] - try: - for attr, mapconfig in data._attribute_map.items(): - if attr in constants: - continue - value = getattr(data, attr) - if value is None: - continue - local_type = mapconfig["type"] - internal_data_type = local_type.strip("[]{}") - if internal_data_type not in self.dependencies or isinstance(internal_data_type, Enum): - continue - setattr(data, attr, self._deserialize(local_type, value)) - return data - except AttributeError: - return - - response, class_name = self._classify_target(target_obj, data) - - if isinstance(response, str): - return self.deserialize_data(data, response) - elif isinstance(response, type) and issubclass(response, Enum): - return self.deserialize_enum(data, response) - - if data is None or data is CoreNull: - return data - try: - attributes = response._attribute_map # type: ignore - d_attrs = {} - for attr, attr_desc in attributes.items(): - # Check empty string. If it's not empty, someone has a real "additionalProperties"... - if attr == "additional_properties" and attr_desc["key"] == "": - continue - raw_value = None - # Enhance attr_desc with some dynamic data - attr_desc = attr_desc.copy() # Do a copy, do not change the real one - internal_data_type = attr_desc["type"].strip("[]{}") - if internal_data_type in self.dependencies: - attr_desc["internalType"] = self.dependencies[internal_data_type] - - for key_extractor in self.key_extractors: - found_value = key_extractor(attr, attr_desc, data) - if found_value is not None: - if raw_value is not None and raw_value != found_value: - msg = ( - "Ignoring extracted value '%s' from %s for key '%s'" - " (duplicate extraction, follow extractors order)" - ) - _LOGGER.warning(msg, found_value, key_extractor, attr) - continue - raw_value = found_value - - value = self.deserialize_data(raw_value, attr_desc["type"]) - d_attrs[attr] = value - except (AttributeError, TypeError, KeyError) as err: - msg = "Unable to deserialize to object: " + class_name # type: ignore - raise DeserializationError(msg) from err - else: - additional_properties = self._build_additional_properties(attributes, data) - return self._instantiate_model(response, d_attrs, additional_properties) - - def _build_additional_properties(self, attribute_map, data): - if not self.additional_properties_detection: - return None - if "additional_properties" in attribute_map and attribute_map.get("additional_properties", {}).get("key") != "": - # Check empty string. If it's not empty, someone has a real "additionalProperties" - return None - if isinstance(data, ET.Element): - data = {el.tag: el.text for el in data} - - known_keys = { - _decode_attribute_map_key(_FLATTEN.split(desc["key"])[0]) - for desc in attribute_map.values() - if desc["key"] != "" - } - present_keys = set(data.keys()) - missing_keys = present_keys - known_keys - return {key: data[key] for key in missing_keys} - - def _classify_target(self, target, data): - """Check to see whether the deserialization target object can - be classified into a subclass. - Once classification has been determined, initialize object. - - :param str target: The target object type to deserialize to. - :param str/dict data: The response data to deserialize. - """ - if target is None: - return None, None - - if isinstance(target, str): - try: - target = self.dependencies[target] - except KeyError: - return target, target - - try: - target = target._classify(data, self.dependencies) # type: ignore - except AttributeError: - pass # Target is not a Model, no classify - return target, target.__class__.__name__ # type: ignore - - def failsafe_deserialize(self, target_obj, data, content_type=None): - """Ignores any errors encountered in deserialization, - and falls back to not deserializing the object. Recommended - for use in error deserialization, as we want to return the - HttpResponseError to users, and not have them deal with - a deserialization error. - - :param str target_obj: The target object type to deserialize to. - :param str/dict data: The response data to deserialize. - :param str content_type: Swagger "produces" if available. - """ - try: - return self(target_obj, data, content_type=content_type) - except: - _LOGGER.debug( - "Ran into a deserialization error. Ignoring since this is failsafe deserialization", exc_info=True - ) - return None - - @staticmethod - def _unpack_content(raw_data, content_type=None): - """Extract the correct structure for deserialization. - - If raw_data is a PipelineResponse, try to extract the result of RawDeserializer. - if we can't, raise. Your Pipeline should have a RawDeserializer. - - If not a pipeline response and raw_data is bytes or string, use content-type - to decode it. If no content-type, try JSON. - - If raw_data is something else, bypass all logic and return it directly. - - :param raw_data: Data to be processed. - :param content_type: How to parse if raw_data is a string/bytes. - :raises JSONDecodeError: If JSON is requested and parsing is impossible. - :raises UnicodeDecodeError: If bytes is not UTF8 - """ - # Assume this is enough to detect a Pipeline Response without importing it - context = getattr(raw_data, "context", {}) - if context: - if RawDeserializer.CONTEXT_NAME in context: - return context[RawDeserializer.CONTEXT_NAME] - raise ValueError("This pipeline didn't have the RawDeserializer policy; can't deserialize") - - # Assume this is enough to recognize universal_http.ClientResponse without importing it - if hasattr(raw_data, "body"): - return RawDeserializer.deserialize_from_http_generics(raw_data.text(), raw_data.headers) - - # Assume this enough to recognize requests.Response without importing it. - if hasattr(raw_data, "_content_consumed"): - return RawDeserializer.deserialize_from_http_generics(raw_data.text, raw_data.headers) - - if isinstance(raw_data, (str, bytes)) or hasattr(raw_data, "read"): - return RawDeserializer.deserialize_from_text(raw_data, content_type) # type: ignore - return raw_data - - def _instantiate_model(self, response, attrs, additional_properties=None): - """Instantiate a response model passing in deserialized args. - - :param response: The response model class. - :param d_attrs: The deserialized response attributes. - """ - if callable(response): - subtype = getattr(response, "_subtype_map", {}) - try: - readonly = [k for k, v in response._validation.items() if v.get("readonly")] - const = [k for k, v in response._validation.items() if v.get("constant")] - kwargs = {k: v for k, v in attrs.items() if k not in subtype and k not in readonly + const} - response_obj = response(**kwargs) - for attr in readonly: - setattr(response_obj, attr, attrs.get(attr)) - if additional_properties: - response_obj.additional_properties = additional_properties - return response_obj - except TypeError as err: - msg = "Unable to deserialize {} into model {}. ".format(kwargs, response) # type: ignore - raise DeserializationError(msg + str(err)) - else: - try: - for attr, value in attrs.items(): - setattr(response, attr, value) - return response - except Exception as exp: - msg = "Unable to populate response model. " - msg += "Type: {}, Error: {}".format(type(response), exp) - raise DeserializationError(msg) - - def deserialize_data(self, data, data_type): - """Process data for deserialization according to data type. - - :param str data: The response string to be deserialized. - :param str data_type: The type to deserialize to. - :raises: DeserializationError if deserialization fails. - :return: Deserialized object. - """ - if data is None: - return data - - try: - if not data_type: - return data - if data_type in self.basic_types.values(): - return self.deserialize_basic(data, data_type) - if data_type in self.deserialize_type: - if isinstance(data, self.deserialize_expected_types.get(data_type, tuple())): - return data - - is_a_text_parsing_type = lambda x: x not in ["object", "[]", r"{}"] - if isinstance(data, ET.Element) and is_a_text_parsing_type(data_type) and not data.text: - return None - data_val = self.deserialize_type[data_type](data) - return data_val - - iter_type = data_type[0] + data_type[-1] - if iter_type in self.deserialize_type: - return self.deserialize_type[iter_type](data, data_type[1:-1]) - - obj_type = self.dependencies[data_type] - if issubclass(obj_type, Enum): - if isinstance(data, ET.Element): - data = data.text - return self.deserialize_enum(data, obj_type) - - except (ValueError, TypeError, AttributeError) as err: - msg = "Unable to deserialize response data." - msg += " Data: {}, {}".format(data, data_type) - raise DeserializationError(msg) from err - else: - return self._deserialize(obj_type, data) - - def deserialize_iter(self, attr, iter_type): - """Deserialize an iterable. - - :param list attr: Iterable to be deserialized. - :param str iter_type: The type of object in the iterable. - :rtype: list - """ - if attr is None: - return None - if isinstance(attr, ET.Element): # If I receive an element here, get the children - attr = list(attr) - if not isinstance(attr, (list, set)): - raise DeserializationError("Cannot deserialize as [{}] an object of type {}".format(iter_type, type(attr))) - return [self.deserialize_data(a, iter_type) for a in attr] - - def deserialize_dict(self, attr, dict_type): - """Deserialize a dictionary. - - :param dict/list attr: Dictionary to be deserialized. Also accepts - a list of key, value pairs. - :param str dict_type: The object type of the items in the dictionary. - :rtype: dict - """ - if isinstance(attr, list): - return {x["key"]: self.deserialize_data(x["value"], dict_type) for x in attr} - - if isinstance(attr, ET.Element): - # Transform value into {"Key": "value"} - attr = {el.tag: el.text for el in attr} - return {k: self.deserialize_data(v, dict_type) for k, v in attr.items()} - - def deserialize_object(self, attr, **kwargs): - """Deserialize a generic object. - This will be handled as a dictionary. - - :param dict attr: Dictionary to be deserialized. - :rtype: dict - :raises: TypeError if non-builtin datatype encountered. - """ - if attr is None: - return None - if isinstance(attr, ET.Element): - # Do no recurse on XML, just return the tree as-is - return attr - if isinstance(attr, str): - return self.deserialize_basic(attr, "str") - obj_type = type(attr) - if obj_type in self.basic_types: - return self.deserialize_basic(attr, self.basic_types[obj_type]) - if obj_type is _long_type: - return self.deserialize_long(attr) - - if obj_type == dict: - deserialized = {} - for key, value in attr.items(): - try: - deserialized[key] = self.deserialize_object(value, **kwargs) - except ValueError: - deserialized[key] = None - return deserialized - - if obj_type == list: - deserialized = [] - for obj in attr: - try: - deserialized.append(self.deserialize_object(obj, **kwargs)) - except ValueError: - pass - return deserialized - - else: - error = "Cannot deserialize generic object with type: " - raise TypeError(error + str(obj_type)) - - def deserialize_basic(self, attr, data_type): - """Deserialize basic builtin data type from string. - Will attempt to convert to str, int, float and bool. - This function will also accept '1', '0', 'true' and 'false' as - valid bool values. - - :param str attr: response string to be deserialized. - :param str data_type: deserialization data type. - :rtype: str, int, float or bool - :raises: TypeError if string format is not valid. - """ - # If we're here, data is supposed to be a basic type. - # If it's still an XML node, take the text - if isinstance(attr, ET.Element): - attr = attr.text - if not attr: - if data_type == "str": - # None or '', node is empty string. - return "" - else: - # None or '', node with a strong type is None. - # Don't try to model "empty bool" or "empty int" - return None - - if data_type == "bool": - if attr in [True, False, 1, 0]: - return bool(attr) - elif isinstance(attr, str): - if attr.lower() in ["true", "1"]: - return True - elif attr.lower() in ["false", "0"]: - return False - raise TypeError("Invalid boolean value: {}".format(attr)) - - if data_type == "str": - return self.deserialize_unicode(attr) - return eval(data_type)(attr) # nosec - - @staticmethod - def deserialize_unicode(data): - """Preserve unicode objects in Python 2, otherwise return data - as a string. - - :param str data: response string to be deserialized. - :rtype: str or unicode - """ - # We might be here because we have an enum modeled as string, - # and we try to deserialize a partial dict with enum inside - if isinstance(data, Enum): - return data - - # Consider this is real string - try: - if isinstance(data, unicode): # type: ignore - return data - except NameError: - return str(data) - else: - return str(data) - - @staticmethod - def deserialize_enum(data, enum_obj): - """Deserialize string into enum object. - - If the string is not a valid enum value it will be returned as-is - and a warning will be logged. - - :param str data: Response string to be deserialized. If this value is - None or invalid it will be returned as-is. - :param Enum enum_obj: Enum object to deserialize to. - :rtype: Enum - """ - if isinstance(data, enum_obj) or data is None: - return data - if isinstance(data, Enum): - data = data.value - if isinstance(data, int): - # Workaround. We might consider remove it in the future. - try: - return list(enum_obj.__members__.values())[data] - except IndexError: - error = "{!r} is not a valid index for enum {!r}" - raise DeserializationError(error.format(data, enum_obj)) - try: - return enum_obj(str(data)) - except ValueError: - for enum_value in enum_obj: - if enum_value.value.lower() == str(data).lower(): - return enum_value - # We don't fail anymore for unknown value, we deserialize as a string - _LOGGER.warning("Deserializer is not able to find %s as valid enum in %s", data, enum_obj) - return Deserializer.deserialize_unicode(data) - - @staticmethod - def deserialize_bytearray(attr): - """Deserialize string into bytearray. - - :param str attr: response string to be deserialized. - :rtype: bytearray - :raises: TypeError if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - return bytearray(b64decode(attr)) # type: ignore - - @staticmethod - def deserialize_base64(attr): - """Deserialize base64 encoded string into string. - - :param str attr: response string to be deserialized. - :rtype: bytearray - :raises: TypeError if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - padding = "=" * (3 - (len(attr) + 3) % 4) # type: ignore - attr = attr + padding # type: ignore - encoded = attr.replace("-", "+").replace("_", "/") - return b64decode(encoded) - - @staticmethod - def deserialize_decimal(attr): - """Deserialize string into Decimal object. - - :param str attr: response string to be deserialized. - :rtype: Decimal - :raises: DeserializationError if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - return decimal.Decimal(str(attr)) # type: ignore - except decimal.DecimalException as err: - msg = "Invalid decimal {}".format(attr) - raise DeserializationError(msg) from err - - @staticmethod - def deserialize_long(attr): - """Deserialize string into long (Py2) or int (Py3). - - :param str attr: response string to be deserialized. - :rtype: long or int - :raises: ValueError if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - return _long_type(attr) # type: ignore - - @staticmethod - def deserialize_duration(attr): - """Deserialize ISO-8601 formatted string into TimeDelta object. - - :param str attr: response string to be deserialized. - :rtype: TimeDelta - :raises: DeserializationError if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - duration = isodate.parse_duration(attr) - except (ValueError, OverflowError, AttributeError) as err: - msg = "Cannot deserialize duration object." - raise DeserializationError(msg) from err - else: - return duration - - @staticmethod - def deserialize_date(attr): - """Deserialize ISO-8601 formatted string into Date object. - - :param str attr: response string to be deserialized. - :rtype: Date - :raises: DeserializationError if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - if re.search(r"[^\W\d_]", attr, re.I + re.U): # type: ignore - raise DeserializationError("Date must have only digits and -. Received: %s" % attr) - # This must NOT use defaultmonth/defaultday. Using None ensure this raises an exception. - return isodate.parse_date(attr, defaultmonth=0, defaultday=0) - - @staticmethod - def deserialize_time(attr): - """Deserialize ISO-8601 formatted string into time object. - - :param str attr: response string to be deserialized. - :rtype: datetime.time - :raises: DeserializationError if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - if re.search(r"[^\W\d_]", attr, re.I + re.U): # type: ignore - raise DeserializationError("Date must have only digits and -. Received: %s" % attr) - return isodate.parse_time(attr) - - @staticmethod - def deserialize_rfc(attr): - """Deserialize RFC-1123 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: Datetime - :raises: DeserializationError if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - parsed_date = email.utils.parsedate_tz(attr) # type: ignore - date_obj = datetime.datetime( - *parsed_date[:6], tzinfo=_FixedOffset(datetime.timedelta(minutes=(parsed_date[9] or 0) / 60)) - ) - if not date_obj.tzinfo: - date_obj = date_obj.astimezone(tz=TZ_UTC) - except ValueError as err: - msg = "Cannot deserialize to rfc datetime object." - raise DeserializationError(msg) from err - else: - return date_obj - - @staticmethod - def deserialize_iso(attr): - """Deserialize ISO-8601 formatted string into Datetime object. - - :param str attr: response string to be deserialized. - :rtype: Datetime - :raises: DeserializationError if string format invalid. - """ - if isinstance(attr, ET.Element): - attr = attr.text - try: - attr = attr.upper() # type: ignore - match = Deserializer.valid_date.match(attr) - if not match: - raise ValueError("Invalid datetime string: " + attr) - - check_decimal = attr.split(".") - if len(check_decimal) > 1: - decimal_str = "" - for digit in check_decimal[1]: - if digit.isdigit(): - decimal_str += digit - else: - break - if len(decimal_str) > 6: - attr = attr.replace(decimal_str, decimal_str[0:6]) - - date_obj = isodate.parse_datetime(attr) - test_utc = date_obj.utctimetuple() - if test_utc.tm_year > 9999 or test_utc.tm_year < 1: - raise OverflowError("Hit max or min date") - except (ValueError, OverflowError, AttributeError) as err: - msg = "Cannot deserialize datetime object." - raise DeserializationError(msg) from err - else: - return date_obj - - @staticmethod - def deserialize_unix(attr): - """Serialize Datetime object into IntTime format. - This is represented as seconds. - - :param int attr: Object to be serialized. - :rtype: Datetime - :raises: DeserializationError if format invalid - """ - if isinstance(attr, ET.Element): - attr = int(attr.text) # type: ignore - try: - attr = int(attr) - date_obj = datetime.datetime.fromtimestamp(attr, TZ_UTC) - except ValueError as err: - msg = "Cannot deserialize to unix datetime object." - raise DeserializationError(msg) from err - else: - return date_obj diff --git a/packages/autorest.python/generator/pygen/codegen/templates/test.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/test.py.jinja2 deleted file mode 100644 index 7664fa914c6..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/test.py.jinja2 +++ /dev/null @@ -1,26 +0,0 @@ -{% set prefix_lower = test.prefix|lower %} -{% set async = "async " if test.is_async else "" %} -{% set async_suffix = "_async" if test.is_async else "" %} -# coding=utf-8 -{{ code_model.options['license_header'] }} -import pytest -{{ imports }} - - -@pytest.mark.skip("you may need to update the auto-generated test case before run it") -class {{ test.test_class_name }}({{ test.base_test_class_name }}): -{% for testcase in test.testcases %} - @{{ test.preparer_name }}() - @recorded_by_proxy{{ async_suffix }} - {{ async }}def test_{{ testcase.operation.name }}(self, {{ prefix_lower }}_endpoint): - client = self.{{ test.create_client_name }}(endpoint={{ prefix_lower }}_endpoint) - {{testcase.response }}client{{ testcase.operation_group_prefix }}.{{ testcase.operation.name }}( - {% for key, value in testcase.params.items() %} - {{ key }}={{ value|indent(12) }}, - {% endfor %} - ){{ testcase.operation_suffix }} - {{ testcase.extra_operation }} - # please add some check logic here by yourself - # ... - -{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/testpreparer.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/testpreparer.py.jinja2 deleted file mode 100644 index b3b15f37276..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/testpreparer.py.jinja2 +++ /dev/null @@ -1,26 +0,0 @@ -# coding=utf-8 -{{ code_model.options['license_header'] }} -{{ imports }} - -{% for test_name in test_names %} -{% set extra_async = ", is_async=True" if test_name.is_async else ""%} -{% set prefix_lower = test_name.prefix|lower %} -class {{ test_name.base_test_class_name }}(AzureRecordedTestCase): - - def {{ test_name.create_client_name }}(self, endpoint): - credential = self.get_credential({{ test_name.client_name }}{{ extra_async }}) - return self.create_client_from_credential( - {{ test_name.client_name }}, - credential=credential, - endpoint=endpoint, - ) - -{% if not test_name.is_async %} -{{ test_name.preparer_name }} = functools.partial( - PowerShellPreparer, - "{{ prefix_lower }}", - {{ prefix_lower }}_endpoint="https://fake_{{ prefix_lower }}_endpoint.com" -) -{% endif %} - -{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/types.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/types.py.jinja2 deleted file mode 100644 index 2dc930871fc..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/types.py.jinja2 +++ /dev/null @@ -1,8 +0,0 @@ -# coding=utf-8 -# pylint: disable=too-many-lines -{{ code_model.options['license_header'] }} - -{{ imports }} -{% for nu in code_model.named_unions %} -{{nu.name}} = {{nu.type_definition()}} -{% endfor %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/validation.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/validation.py.jinja2 deleted file mode 100644 index ebc4b243881..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/validation.py.jinja2 +++ /dev/null @@ -1,38 +0,0 @@ -{{ code_model.options['license_header'] }} -import functools - -def api_version_validation(**kwargs): - params_added_on = kwargs.pop("params_added_on", {}) - method_added_on = kwargs.pop("method_added_on", "") - - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - # this assumes the client has an _api_version attribute - client = args[0] - client_api_version = client._config.api_version # pylint: disable=protected-access - except AttributeError: - return func(*args, **kwargs) - - if method_added_on > client_api_version: - raise ValueError( - f"'{func.__name__}' is not available in API version " - f"{client_api_version}. Pass service API version {method_added_on} or newer to your client." - ) - - unsupported = { - parameter: api_version - for api_version, parameters in params_added_on.items() - for parameter in parameters - if parameter in kwargs and api_version > client_api_version - } - if unsupported: - raise ValueError("".join([ - f"'{param}' is not available in API version {client_api_version}. " - f"Use service API version {version} or newer.\n" - for param, version in unsupported.items() - ])) - return func(*args, **kwargs) - return wrapper - return decorator diff --git a/packages/autorest.python/generator/pygen/codegen/templates/vendor.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/vendor.py.jinja2 deleted file mode 100644 index bc15b14dd56..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/vendor.py.jinja2 +++ /dev/null @@ -1,98 +0,0 @@ -{% import 'keywords.jinja2' as keywords with context %} -{{ code_model.options['license_header'] }} - -{{ imports }} - -{% if code_model.need_mixin_abc %} - {% for client in clients | selectattr("has_mixin") %} -{% set pylint_disable = "# pylint: disable=name-too-long" if (client.name | length) + ("MixinABC" | length) > 40 else "" %} -class {{ client.name }}MixinABC( {{ pylint_disable }} - ABC -): - """DO NOT use this class. It is for internal typing use only.""" - _client: "{{ keywords.async_class }}PipelineClient" - _config: {{ client.name }}Configuration - _serialize: "Serializer" - _deserialize: "Deserializer" - {% endfor %} -{% endif %} -{% if code_model.has_abstract_operations %} - -def raise_if_not_implemented(cls, abstract_methods): - not_implemented = [f for f in abstract_methods if not callable(getattr(cls, f, None))] - if not_implemented: - raise NotImplementedError("The following methods on operation group '{}' are not implemented: '{}'." - " Please refer to https://aka.ms/azsdk/python/dpcodegen/python/customize to learn how to customize.".format( - cls.__name__, '\', \''.join(not_implemented)) - ) -{% endif %} - -{% if code_model.has_etag %} -def quote_etag(etag: Optional[str]) -> Optional[str]: - if not etag or etag == "*": - return etag - if etag.startswith("W/"): - return etag - if etag.startswith('"') and etag.endswith('"'): - return etag - if etag.startswith("'") and etag.endswith("'"): - return etag - return '"' + etag + '"' - - -def prep_if_match(etag: Optional[str], match_condition: Optional[MatchConditions]) -> Optional[str]: - if match_condition == MatchConditions.IfNotModified: - if_match = quote_etag(etag) if etag else None - return if_match - if match_condition == MatchConditions.IfPresent: - return "*" - return None - - -def prep_if_none_match(etag: Optional[str], match_condition: Optional[MatchConditions]) -> Optional[str]: - if match_condition == MatchConditions.IfModified: - if_none_match = quote_etag(etag) if etag else None - return if_none_match - if match_condition == MatchConditions.IfMissing: - return "*" - return None -{% endif %} -{% if code_model.has_form_data and code_model.options["models_mode"] == "dpg" and not async_mode %} -# file-like tuple could be `(filename, IO (or bytes))` or `(filename, IO (or bytes), content_type)` -FileContent = Union[str, bytes, IO[str], IO[bytes]] - -FileType = Union[ - # file (or bytes) - FileContent, - # (filename, file (or bytes)) - Tuple[Optional[str], FileContent], - # (filename, file (or bytes), content_type) - Tuple[Optional[str], FileContent, Optional[str]], -] - -FilesType = Union[Mapping[str, FileType], Sequence[Tuple[str, FileType]]] - -def serialize_multipart_data_entry(data_entry: Any) -> Any: - if isinstance(data_entry, (list, tuple, dict, Model)): - return json.dumps(data_entry, cls=SdkJSONEncoder, exclude_readonly=True) - return data_entry - -def prepare_multipart_form_data( - body: Mapping[str, Any], multipart_fields: List[str], data_fields: List[str] -) -> Tuple[List[FileType], Dict[str, Any]]: - files: List[FileType] = [] - data: Dict[str, Any] = {} - for multipart_field in multipart_fields: - multipart_entry = body.get(multipart_field) - if isinstance(multipart_entry, list): - files.extend([(multipart_field, e) for e in multipart_entry ]) - elif multipart_entry: - files.append((multipart_field, multipart_entry)) - - for data_field in data_fields: - data_entry = body.get(data_field) - if data_entry: - data[data_field] = serialize_multipart_data_entry(data_entry) - - return files, data -{% endif %} diff --git a/packages/autorest.python/generator/pygen/codegen/templates/version.py.jinja2 b/packages/autorest.python/generator/pygen/codegen/templates/version.py.jinja2 deleted file mode 100644 index 2ea06fccbe3..00000000000 --- a/packages/autorest.python/generator/pygen/codegen/templates/version.py.jinja2 +++ /dev/null @@ -1,4 +0,0 @@ -# coding=utf-8 -{{ code_model.options['license_header'] }} - -VERSION = "{{ code_model.options['package_version'] }}" diff --git a/packages/autorest.python/generator/pygen/m2r/__init__.py b/packages/autorest.python/generator/pygen/m2r/__init__.py deleted file mode 100644 index d6e78871787..00000000000 --- a/packages/autorest.python/generator/pygen/m2r/__init__.py +++ /dev/null @@ -1,65 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -"""An MD to RST plugin. -""" -import logging -from typing import Any, Dict, Set, Union - -import m2r2 - -from .. import YamlUpdatePlugin -from ..utils import parse_args - - -_LOGGER = logging.getLogger(__name__) - - -class GeneratorRenderer(m2r2.RestRenderer): - """Redefine the concept of inline HTML in the renderer, we don't want to define a new format - in the description/summary. - """ - - def inline_html(self, html: str) -> str: - """Do not render inline HTML with a role definition.""" - return f":code:`{html}`" - - -class M2R(YamlUpdatePlugin): # pylint: disable=abstract-method - """A plugin to convert any description and summary from MD to RST.""" - - def update_yaml(self, yaml_data: Dict[str, Any]) -> None: - """Convert in place the YAML str.""" - self._convert_docstring_no_cycles(yaml_data, set()) - - def _convert_docstring_no_cycles(self, yaml_data: Union[Dict[str, Any], str], node_list: Set[int]) -> None: - """Walk the YAML tree to convert MD to RST.""" - if id(yaml_data) in node_list: - return - node_list.add(id(yaml_data)) - - if isinstance(yaml_data, list): - for elt in yaml_data: - self._convert_docstring_no_cycles(elt, node_list) - elif isinstance(yaml_data, dict): - for key, value in yaml_data.items(): - if key in ["description", "summary"]: - yaml_data[key] = self.convert_to_rst(value) - continue - self._convert_docstring_no_cycles(value, node_list) - - @staticmethod - def convert_to_rst(string_to_convert: str) -> str: - """Convert that string from MD to RST.""" - try: - return m2r2.convert(string_to_convert, renderer=GeneratorRenderer()).strip() - except Exception: # pylint: disable=broad-except - return string_to_convert - - -if __name__ == "__main__": - # CADL pipeline will call this - args, unknown_args = parse_args() - M2R(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process() diff --git a/packages/autorest.python/generator/pygen/postprocess/__init__.py b/packages/autorest.python/generator/pygen/postprocess/__init__.py deleted file mode 100644 index 771177261cb..00000000000 --- a/packages/autorest.python/generator/pygen/postprocess/__init__.py +++ /dev/null @@ -1,183 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Tuple, Any, Dict -from pathlib import Path -import os -import shutil -from venv import EnvBuilder -import black -from black.report import NothingChanged -from .venvtools import ExtendedEnvBuilder, python_run - -from .. import Plugin - -_BLACK_MODE = black.Mode() # pyright: ignore [reportPrivateImportUsage] -_BLACK_MODE.line_length = 120 - - -def format_file(file: Path, file_content: str) -> str: - if not file.suffix == ".py": - return file_content - try: - file_content = black.format_file_contents(file_content, fast=True, mode=_BLACK_MODE) - except NothingChanged: - pass - return file_content - - -class PostProcessPlugin(Plugin): # pylint: disable=abstract-method - def __init__(self, **kwargs: Any): - super().__init__(**kwargs) - output_folder_uri = self.options["outputFolderUri"] - if output_folder_uri.startswith("file:"): - output_folder_uri = output_folder_uri[5:] - if os.name == "nt" and output_folder_uri.startswith("///"): - output_folder_uri = output_folder_uri[3:] - self.output_folder = Path(output_folder_uri) # path to where the setup.py is - self.setup_venv() - - # set up the venv - # base folder is where the code starts, i.e. where we - self.base_folder, self.namespace = self.get_namespace(self.output_folder, "") - - def setup_venv(self): - venv_path = self.output_folder / Path(".temp_folder") / Path("temp_venv") - - if venv_path.exists(): - env_builder = EnvBuilder(with_pip=True) - self.venv_context = env_builder.ensure_directories(venv_path) - else: - env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True) - env_builder.create(venv_path) - self.venv_context = env_builder.context - python_run( - self.venv_context, - "pip", - ["install", "-e", str(self.output_folder)], - directory=self.output_folder, - ) - - def get_namespace(self, dir: Path, namespace: str) -> Tuple[Path, str]: - try: - init_file = next(d for d in dir.iterdir() if d.name == "__init__.py") - # we don't care about pkgutil inits, we skip over them - file_content = self.read_file(init_file.relative_to(self.output_folder)) - if "pkgutil" not in file_content: - return dir, namespace - except StopIteration: - pass - - try: - # first, see if we can get a folder that has the same name as the current output folder - start = self.output_folder.stem.split("-")[0] - next_dir = next(d for d in dir.iterdir() if d.is_dir() and d.name == start) - except StopIteration: - invalid_start_chars = [".", "_"] - invalid_dirs = [ - "swagger", - "out", - "tests", - "samples", - ] - - next_dir = next( - d - for d in dir.iterdir() - if d.is_dir() - and not str(d).endswith("egg-info") - and d.name[0] not in invalid_start_chars - and d.name not in invalid_dirs - ) - - namespace = f"{namespace}.{next_dir.name}" if namespace else next_dir.name - return self.get_namespace(next_dir, namespace) - - def process(self) -> bool: - folders = [f for f in self.base_folder.glob("**/*") if f.is_dir() and not f.stem.startswith("__")] - # will always have the root - self.fix_imports_in_init( - generated_file_name="_client", - folder_path=self.base_folder, - namespace=self.namespace, - ) - try: - aio_folder = next(f for f in folders if f.stem == "aio") - self.fix_imports_in_init( - generated_file_name="_client", - folder_path=aio_folder, - namespace=f"{self.namespace}.aio", - ) - except StopIteration: - pass - - try: - models_folder = next(f for f in folders if f.stem == "models") - self.fix_imports_in_init( - generated_file_name="_models", - folder_path=models_folder, - namespace=f"{self.namespace}.models", - ) - except StopIteration: - pass - operations_folders = [f for f in folders if f.stem in ["operations", "_operations"]] - for operations_folder in operations_folders: - sub_namespace = ".".join(str(operations_folder.relative_to(self.base_folder)).split(os.sep)) - self.fix_imports_in_init( - generated_file_name="_operations", - folder_path=operations_folder, - namespace=f"{self.namespace}.{sub_namespace}", - ) - shutil.rmtree(f"{str(self.output_folder)}/.temp_folder") - return True - - def fix_imports_in_init(self, generated_file_name: str, folder_path: Path, namespace: str) -> None: - customized_objects_str = python_run( - self.venv_context, - command=[namespace, str(self.output_folder)], - module="get_all", - ) - - if not customized_objects_str: - return - customized_objects = {k: None for k in customized_objects_str.split(",")}.keys() # filter out duplicates - file = (folder_path / "__init__.py").relative_to(self.output_folder) - file_content = self.read_file(file).replace("\r\n", "\n") - added_objs = [] - for obj in customized_objects: - if f" import {obj}\n" in file_content: - # means we're overriding a generated model - file_content = file_content.replace( - f"from .{generated_file_name} import {obj}\n", - f"from ._patch import {obj}\n", - ) - else: - added_objs.append(obj) - file_content = file_content.replace( - "try:\n from ._patch import __all__ as _patch_all\n " - "from ._patch import * # pylint: disable=unused-wildcard-import" - "\nexcept ImportError:\n _patch_all = []", - "", - ) - file_content = file_content.replace("from ._patch import __all__ as _patch_all", "") - file_content = file_content.replace( - "from ._patch import * # pylint: disable=unused-wildcard-import\n", - "", - ) - file_content = file_content.replace("__all__.extend([p for p in _patch_all if p not in __all__])", "") - if added_objs: - # add import - patch_sdk_import = "from ._patch import patch_sdk as _patch_sdk" - imports = "\n".join([f"from ._patch import {obj}" for obj in added_objs]) - if imports: - replacement = f"{imports}\n{patch_sdk_import}" - else: - replacement = patch_sdk_import - file_content = file_content.replace(patch_sdk_import, replacement) - # add to __all__ - added_objs_all = "\n".join([f' "{obj}",' for obj in added_objs]) + "\n" - file_content = file_content.replace("__all__ = [", f"__all__ = [\n{added_objs_all}", 1) - formatted_file = format_file(file, file_content) - self.write_file(file, formatted_file) diff --git a/packages/autorest.python/generator/pygen/postprocess/get_all.py b/packages/autorest.python/generator/pygen/postprocess/get_all.py deleted file mode 100644 index 4206e36a9c4..00000000000 --- a/packages/autorest.python/generator/pygen/postprocess/get_all.py +++ /dev/null @@ -1,19 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import sys -import importlib - - -def main(namespace): - sdk = importlib.import_module(namespace) - return sdk._patch.__all__ # pylint: disable=protected-access - - -if __name__ == "__main__": - patched = ",".join(main(sys.argv[1])) - output_folder = sys.argv[2] - with open(f"{output_folder}/.temp_folder/patched.txt", "w", encoding="utf-8-sig") as f: - f.write(patched) diff --git a/packages/autorest.python/generator/pygen/postprocess/venvtools.py b/packages/autorest.python/generator/pygen/postprocess/venvtools.py deleted file mode 100644 index 482cb9ee0ff..00000000000 --- a/packages/autorest.python/generator/pygen/postprocess/venvtools.py +++ /dev/null @@ -1,77 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Optional -import subprocess -import venv -import sys -from pathlib import Path - - -_ROOT_DIR = Path(__file__).parent - - -class ExtendedEnvBuilder(venv.EnvBuilder): - """An extended env builder which saves the context, to have access - easily to bin path and such. - """ - - def __init__(self, *args, **kwargs): - self.context = None - if sys.version_info < (3, 9, 0): - # Not supported on Python 3.8, and we don't need it - kwargs.pop("upgrade_deps", None) - super().__init__(*args, **kwargs) - - def ensure_directories(self, env_dir): - self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir) - return self.context - - -def create( - env_dir, - system_site_packages=False, - clear=False, - symlinks=False, - with_pip=False, - prompt=None, - upgrade_deps=False, -): - """Create a virtual environment in a directory.""" - builder = ExtendedEnvBuilder( - system_site_packages=system_site_packages, - clear=clear, - symlinks=symlinks, - with_pip=with_pip, - prompt=prompt, - upgrade_deps=upgrade_deps, - ) - builder.create(env_dir) - return builder.context - - -def python_run( # pylint: disable=inconsistent-return-statements - venv_context, module, command, directory=_ROOT_DIR -) -> Optional[str]: - try: - cmd_line = [ - venv_context.env_exe, - "-m", - module, - ] + command - print("Executing: {}".format(" ".join(cmd_line))) - subprocess.run( - cmd_line, - cwd=directory, - check=True, - stdout=False, - ) - if module == "get_all": - with open(f"{command[1]}/.temp_folder/patched.txt", "r", encoding="utf-8-sig") as f: - return f.read() - except subprocess.CalledProcessError as err: - print(err) - sys.exit(1) - return None diff --git a/packages/autorest.python/generator/pygen/preprocess/__init__.py b/packages/autorest.python/generator/pygen/preprocess/__init__.py deleted file mode 100644 index d823a910872..00000000000 --- a/packages/autorest.python/generator/pygen/preprocess/__init__.py +++ /dev/null @@ -1,483 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -"""The preprocessing autorest plugin. -""" -import copy -from typing import Callable, Dict, Any, List, Optional - -from ..utils import to_snake_case -from .helpers import ( - add_redefined_builtin_info, - pad_builtin_namespaces, - pad_special_chars, -) -from .python_mappings import CADL_RESERVED_WORDS, RESERVED_WORDS, PadType - -from .. import YamlUpdatePlugin -from ..utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES - - -def update_overload_section( - overload: Dict[str, Any], - yaml_data: Dict[str, Any], - section: str, -): - try: - for overload_s, original_s in zip(overload[section], yaml_data[section]): - if overload_s.get("type"): - overload_s["type"] = original_s["type"] - if overload_s.get("headers"): - for overload_h, original_h in zip(overload_s["headers"], original_s["headers"]): - if overload_h.get("type"): - overload_h["type"] = original_h["type"] - except KeyError as exc: - raise ValueError(overload["name"]) from exc - - -def add_overload(yaml_data: Dict[str, Any], body_type: Dict[str, Any], for_flatten_params=False): - overload = copy.deepcopy(yaml_data) - overload["isOverload"] = True - overload["bodyParameter"]["type"] = body_type - overload["bodyParameter"]["defaultToUnsetSentinel"] = False - overload["overloads"] = [] - if yaml_data.get("initialOperation"): - overload["initialOperation"] = yaml_data["initialOperation"] - - if for_flatten_params: - overload["bodyParameter"]["flattened"] = True - else: - overload["parameters"] = [p for p in overload["parameters"] if not p.get("inFlattenedBody")] - # for yaml sync, we need to make sure all of the responses, parameters, and exceptions' types have the same yaml id - for overload_p, original_p in zip(overload["parameters"], yaml_data["parameters"]): - overload_p["type"] = original_p["type"] - update_overload_section(overload, yaml_data, "responses") - update_overload_section(overload, yaml_data, "exceptions") - - # update content type to be an overloads content type - content_type_param = next(p for p in overload["parameters"] if p["wireName"].lower() == "content-type") - content_type_param["inOverload"] = True - content_type_param["inDocstring"] = True - body_type_description = get_body_type_for_description(overload["bodyParameter"]) - content_type_param["description"] = ( - f"Body Parameter content-type. Content type parameter for {body_type_description} body." - ) - content_types = yaml_data["bodyParameter"]["contentTypes"] - if body_type["type"] == "binary" and len(content_types) > 1: - content_types = "'" + "', '".join(content_types) + "'" - content_type_param["description"] += f" Known values are: {content_types}." - overload["bodyParameter"]["inOverload"] = True - for parameter in overload["parameters"]: - parameter["inOverload"] = True - parameter["defaultToUnsetSentinel"] = False - return overload - - -def add_overloads_for_body_param(yaml_data: Dict[str, Any]) -> None: - """If we added a body parameter type, add overloads for that type""" - body_parameter = yaml_data["bodyParameter"] - if not ( - body_parameter["type"]["type"] == "combined" - and len(yaml_data["bodyParameter"]["type"]["types"]) > len(yaml_data["overloads"]) - ): - return - for body_type in body_parameter["type"]["types"]: - if any(o for o in yaml_data["overloads"] if id(o["bodyParameter"]["type"]) == id(body_type)): - continue - yaml_data["overloads"].append(add_overload(yaml_data, body_type)) - if body_type.get("type") == "model" and body_type.get("base") == "json": - yaml_data["overloads"].append(add_overload(yaml_data, body_type, for_flatten_params=True)) - content_type_param = next(p for p in yaml_data["parameters"] if p["wireName"].lower() == "content-type") - content_type_param["inOverload"] = False - content_type_param["inOverriden"] = True - content_type_param["inDocstring"] = True - content_type_param["clientDefaultValue"] = ( - None # make it none bc it will be overriden, we depend on default of overloads - ) - content_type_param["optional"] = True - - -def update_description(description: Optional[str], default_description: str = "") -> str: - if not description: - description = default_description - description.rstrip(" ") - if description and description[-1] != ".": - description += "." - return description - - -def update_operation_group_class_name(prefix: str, class_name: str) -> str: - if class_name == "": - return prefix + "OperationsMixin" - if class_name == "Operations": - return "Operations" - return class_name + "Operations" - - -def update_paging_response(yaml_data: Dict[str, Any]) -> None: - yaml_data["discriminator"] = "paging" - - -HEADERS_HIDE_IN_METHOD = ( - "repeatability-request-id", - "repeatability-first-sent", - "x-ms-client-request-id", - "client-request-id", - "return-client-request-id", -) -HEADERS_CONVERT_IN_METHOD = { - "if-match": { - "clientName": "etag", - "wireName": "etag", - "description": "check if resource is changed. Set None to skip checking etag.", - }, - "if-none-match": { - "clientName": "match_condition", - "wireName": "match-condition", - "description": "The match condition to use upon the etag.", - "type": { - "type": "sdkcore", - "name": "MatchConditions", - }, - }, -} - - -def get_wire_name_lower(parameter: Dict[str, Any]) -> str: - return (parameter.get("wireName") or "").lower() - - -def headers_convert(yaml_data: Dict[str, Any], replace_data: Any) -> None: - if isinstance(replace_data, dict): - for k, v in replace_data.items(): - yaml_data[k] = v - - -def has_json_content_type(yaml_data: Dict[str, Any]) -> bool: - return any(ct for ct in yaml_data.get("contentTypes", []) if JSON_REGEXP.match(ct)) - - -def has_multi_part_content_type(yaml_data: Dict[str, Any]) -> bool: - return any(ct for ct in yaml_data.get("contentTypes", []) if ct == "multipart/form-data") - - -class PreProcessPlugin(YamlUpdatePlugin): # pylint: disable=abstract-method - """Add Python naming information.""" - - @property - def azure_arm(self) -> bool: - return self.options.get("azure-arm", False) - - @property - def version_tolerant(self) -> bool: - return self.options.get("version-tolerant", True) - - @property - def models_mode(self) -> Optional[str]: - return self.options.get("models-mode", "dpg" if self.is_cadl else None) - - @property - def is_cadl(self) -> bool: - return self.options.get("cadl_file", False) - - def add_body_param_type( - self, - code_model: Dict[str, Any], - body_parameter: Dict[str, Any], - ): - # only add overload for special content type - if ( # pylint: disable=too-many-boolean-expressions - body_parameter - and body_parameter["type"]["type"] in ("model", "dict", "list") - and ( - has_json_content_type(body_parameter) or (self.is_cadl and has_multi_part_content_type(body_parameter)) - ) - and not body_parameter["type"].get("xmlMetadata") - and not any(t for t in ["flattened", "groupedBy"] if body_parameter.get(t)) - ): - origin_type = body_parameter["type"]["type"] - is_dpg_model = body_parameter["type"].get("base") == "dpg" - body_parameter["type"] = { - "type": "combined", - "types": [body_parameter["type"]], - } - # don't add binary overload for multipart content type - if not (self.is_cadl and has_multi_part_content_type(body_parameter)): - body_parameter["type"]["types"].append(KNOWN_TYPES["binary"]) - - if origin_type == "model" and is_dpg_model and self.models_mode == "dpg": - body_parameter["type"]["types"].insert(1, KNOWN_TYPES["any-object"]) - code_model["types"].append(body_parameter["type"]) - - def pad_reserved_words(self, name: str, pad_type: PadType): - # we want to pad hidden variables as well - if not name: - # we'll pass in empty operation groups sometime etc. - return name - - if self.is_cadl: - reserved_words = {k: (v + CADL_RESERVED_WORDS.get(k, [])) for k, v in RESERVED_WORDS.items()} - else: - reserved_words = RESERVED_WORDS - name = pad_special_chars(name) - name_prefix = "_" if name[0] == "_" else "" - name = name[1:] if name[0] == "_" else name - if name.lower() in reserved_words[pad_type]: - return name_prefix + name + pad_type - return name_prefix + name - - def update_types(self, yaml_data: List[Dict[str, Any]]) -> None: - for type in yaml_data: - for property in type.get("properties", []): - property["description"] = update_description(property.get("description", "")) - property["clientName"] = self.pad_reserved_words(property["clientName"].lower(), PadType.PROPERTY) - add_redefined_builtin_info(property["clientName"], property) - if type.get("name"): - name = self.pad_reserved_words(type["name"], PadType.MODEL) - type["name"] = name[0].upper() + name[1:] - type["description"] = update_description(type.get("description", ""), type["name"]) - type["snakeCaseName"] = to_snake_case(type["name"]) - if type.get("values"): - # we're enums - for value in type["values"]: - padded_name = self.pad_reserved_words(value["name"].lower(), PadType.ENUM).upper() - if padded_name[0] in "0123456789": - padded_name = "ENUM_" + padded_name - value["name"] = padded_name - - # add type for reference - for v in HEADERS_CONVERT_IN_METHOD.values(): - if isinstance(v, dict) and "type" in v: - yaml_data.append(v["type"]) - - def update_client(self, yaml_data: Dict[str, Any]) -> None: - yaml_data["description"] = update_description(yaml_data["description"], default_description=yaml_data["name"]) - yaml_data["legacyFilename"] = to_snake_case(yaml_data["name"].replace(" ", "_")) - parameters = yaml_data["parameters"] - for parameter in parameters: - self.update_parameter(parameter) - if parameter["clientName"] == "credential": - policy = parameter["type"].get("policy") - if policy and policy["type"] == "BearerTokenCredentialPolicy" and self.azure_arm: - policy["type"] = "ARMChallengeAuthenticationPolicy" - policy["credentialScopes"] = ["https://management.azure.com/.default"] - if ( - (not self.version_tolerant or self.azure_arm) - and parameters - and parameters[-1]["clientName"] == "credential" - ): - # we need to move credential to the front in mgmt mode for backcompat reasons - yaml_data["parameters"] = [parameters[-1]] + parameters[:-1] - prop_name = yaml_data["name"] - if prop_name.endswith("Client"): - prop_name = prop_name[: len(prop_name) - len("Client")] - yaml_data["builderPadName"] = to_snake_case(prop_name) - for og in yaml_data["operationGroups"]: - for o in og["operations"]: - property_if_match = None - property_if_none_match = None - for p in o["parameters"]: - wire_name_lower = get_wire_name_lower(p) - if p["location"] == "header" and wire_name_lower == "client-request-id": - yaml_data["requestIdHeaderName"] = wire_name_lower - if self.version_tolerant and p["location"] == "header": - if wire_name_lower == "if-match": - property_if_match = p - elif wire_name_lower == "if-none-match": - property_if_none_match = p - # pylint: disable=line-too-long - # some service(e.g. https://github.com/Azure/azure-rest-api-specs/blob/main/specification/cosmos-db/data-plane/Microsoft.Tables/preview/2019-02-02/table.json) - # only has one, so we need to add "if-none-match" or "if-match" if it's missing - if not property_if_match and property_if_none_match: - property_if_match = property_if_none_match.copy() - property_if_match["wireName"] = "if-match" - if not property_if_none_match and property_if_match: - property_if_none_match = property_if_match.copy() - property_if_none_match["wireName"] = "if-none-match" - - if property_if_match and property_if_none_match: - # arrange if-match and if-none-match to the end of parameters - o["parameters"] = [ - item - for item in o["parameters"] - if get_wire_name_lower(item) not in ("if-match", "if-none-match") - ] + [property_if_match, property_if_none_match] - - o["hasEtag"] = True - yaml_data["hasEtag"] = True - - def get_operation_updater(self, yaml_data: Dict[str, Any]) -> Callable[[Dict[str, Any], Dict[str, Any]], None]: - if yaml_data["discriminator"] == "lropaging": - return self.update_lro_paging_operation - if yaml_data["discriminator"] == "lro": - return self.update_lro_operation - if yaml_data["discriminator"] == "paging": - return self.update_paging_operation - return self.update_operation - - def update_parameter(self, yaml_data: Dict[str, Any]) -> None: - yaml_data["description"] = update_description(yaml_data.get("description", "")) - if not (yaml_data["location"] == "header" and yaml_data["clientName"] in ("content_type", "accept")): - yaml_data["clientName"] = self.pad_reserved_words(yaml_data["clientName"].lower(), PadType.PARAMETER) - if yaml_data.get("propertyToParameterName"): - # need to create a new one with padded keys and values - yaml_data["propertyToParameterName"] = { - self.pad_reserved_words(prop, PadType.PROPERTY): self.pad_reserved_words( - param_name, PadType.PARAMETER - ).lower() - for prop, param_name in yaml_data["propertyToParameterName"].items() - } - wire_name_lower = (yaml_data.get("wireName") or "").lower() - if yaml_data["location"] == "header" and ( - wire_name_lower in HEADERS_HIDE_IN_METHOD or yaml_data.get("clientDefaultValue") == "multipart/form-data" - ): - yaml_data["hideInMethod"] = True - if self.version_tolerant and yaml_data["location"] == "header" and wire_name_lower in HEADERS_CONVERT_IN_METHOD: - headers_convert(yaml_data, HEADERS_CONVERT_IN_METHOD[wire_name_lower]) - if wire_name_lower in ["$host", "content-type", "accept"] and yaml_data["type"]["type"] == "constant": - yaml_data["clientDefaultValue"] = yaml_data["type"]["value"] - - def update_operation( - self, - code_model: Dict[str, Any], - yaml_data: Dict[str, Any], - *, - is_overload: bool = False, - ) -> None: - yaml_data["groupName"] = self.pad_reserved_words(yaml_data["groupName"], PadType.OPERATION_GROUP) - yaml_data["groupName"] = to_snake_case(yaml_data["groupName"]) - yaml_data["name"] = yaml_data["name"].lower() - yaml_data["name"] = self.pad_reserved_words(yaml_data["name"], PadType.METHOD) - yaml_data["description"] = update_description(yaml_data["description"], yaml_data["name"]) - yaml_data["summary"] = update_description(yaml_data.get("summary", "")) - body_parameter = yaml_data.get("bodyParameter") - for parameter in yaml_data["parameters"]: - self.update_parameter(parameter) - if yaml_data.get("bodyParameter"): - self.update_parameter(yaml_data["bodyParameter"]) - for entry in yaml_data["bodyParameter"].get("entries", []): - self.update_parameter(entry) - for overload in yaml_data.get("overloads", []): - self.update_operation(code_model, overload, is_overload=True) - for response in yaml_data.get("responses", []): - response["discriminator"] = "operation" - if body_parameter and not is_overload: - # if we have a JSON body, we add a binary overload - self.add_body_param_type(code_model, body_parameter) - add_overloads_for_body_param(yaml_data) - - def _update_lro_operation_helper(self, yaml_data: Dict[str, Any]) -> None: - for response in yaml_data.get("responses", []): - response["discriminator"] = "lro" - response["pollerSync"] = response.get("pollerSync") or "azure.core.polling.LROPoller" - response["pollerAsync"] = response.get("pollerAsync") or "azure.core.polling.AsyncLROPoller" - if not response.get("pollingMethodSync"): - response["pollingMethodSync"] = ( - "azure.mgmt.core.polling.arm_polling.ARMPolling" - if self.azure_arm - else "azure.core.polling.base_polling.LROBasePolling" - ) - if not response.get("pollingMethodAsync"): - response["pollingMethodAsync"] = ( - "azure.mgmt.core.polling.async_arm_polling.AsyncARMPolling" - if self.azure_arm - else "azure.core.polling.async_base_polling.AsyncLROBasePolling" - ) - - def update_lro_paging_operation( - self, - code_model: Dict[str, Any], - yaml_data: Dict[str, Any], - is_overload: bool = False, - item_type: Optional[Dict[str, Any]] = None, - ) -> None: - self.update_lro_operation(code_model, yaml_data, is_overload=is_overload) - self.update_paging_operation(code_model, yaml_data, is_overload=is_overload, item_type=item_type) - yaml_data["discriminator"] = "lropaging" - for response in yaml_data.get("responses", []): - response["discriminator"] = "lropaging" - for overload in yaml_data.get("overloads", []): - self.update_lro_paging_operation( - code_model, - overload, - is_overload=True, - item_type=yaml_data["responses"][0]["itemType"], - ) - - def update_lro_operation( - self, - code_model: Dict[str, Any], - yaml_data: Dict[str, Any], - is_overload: bool = False, - ) -> None: - self.update_operation(code_model, yaml_data, is_overload=is_overload) - self.update_operation(code_model, yaml_data["initialOperation"], is_overload=is_overload) - self._update_lro_operation_helper(yaml_data) - for overload in yaml_data.get("overloads", []): - self._update_lro_operation_helper(overload) - self.update_operation(code_model, overload["initialOperation"], is_overload=True) - - def update_paging_operation( - self, - code_model: Dict[str, Any], - yaml_data: Dict[str, Any], - is_overload: bool = False, - item_type: Optional[Dict[str, Any]] = None, - ) -> None: - self.update_operation(code_model, yaml_data, is_overload=is_overload) - item_type = item_type or yaml_data["itemType"]["elementType"] - if yaml_data.get("nextOperation"): - yaml_data["nextOperation"]["groupName"] = self.pad_reserved_words( - yaml_data["nextOperation"]["groupName"], PadType.OPERATION_GROUP - ) - yaml_data["nextOperation"]["groupName"] = to_snake_case(yaml_data["nextOperation"]["groupName"]) - for response in yaml_data["nextOperation"].get("responses", []): - update_paging_response(response) - response["itemType"] = item_type - for response in yaml_data.get("responses", []): - update_paging_response(response) - response["itemType"] = item_type - for overload in yaml_data.get("overloads", []): - self.update_paging_operation(code_model, overload, is_overload=True, item_type=item_type) - - def update_operation_groups(self, code_model: Dict[str, Any], client: Dict[str, Any]) -> None: - operation_groups_yaml_data = client["operationGroups"] - for operation_group in operation_groups_yaml_data: - operation_group["identifyName"] = self.pad_reserved_words( - operation_group.get("name", operation_group["propertyName"]), - PadType.OPERATION_GROUP, - ) - operation_group["identifyName"] = to_snake_case(operation_group["identifyName"]) - operation_group["propertyName"] = self.pad_reserved_words( - operation_group["propertyName"], PadType.OPERATION_GROUP - ) - operation_group["propertyName"] = to_snake_case(operation_group["propertyName"]) - operation_group["className"] = update_operation_group_class_name( - client["name"], operation_group["className"] - ) - for operation in operation_group["operations"]: - self.get_operation_updater(operation)(code_model, operation) - - if operation_group.get("operationGroups"): - self.update_operation_groups(code_model, operation_group) - - def update_yaml(self, yaml_data: Dict[str, Any]) -> None: - """Convert in place the YAML str.""" - self.update_types(yaml_data["types"]) - for client in yaml_data["clients"]: - self.update_client(client) - self.update_operation_groups(yaml_data, client) - for clients in yaml_data["subnamespaceToClients"].values(): - for client in clients: - self.update_client(client) - self.update_operation_groups(yaml_data, client) - if yaml_data.get("namespace"): - yaml_data["namespace"] = pad_builtin_namespaces(yaml_data["namespace"]) - - -if __name__ == "__main__": - # CADL pipeline will call this - args, unknown_args = parse_args() - PreProcessPlugin(output_folder=args.output_folder, cadl_file=args.cadl_file, **unknown_args).process() diff --git a/packages/autorest.python/generator/pygen/preprocess/helpers.py b/packages/autorest.python/generator/pygen/preprocess/helpers.py deleted file mode 100644 index cb9a664a776..00000000000 --- a/packages/autorest.python/generator/pygen/preprocess/helpers.py +++ /dev/null @@ -1,27 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import re -from typing import Any, Dict -from .python_mappings import ( - REDEFINED_BUILTINS, - BUILTIN_PACKAGES, -) - - -def add_redefined_builtin_info(name: str, yaml_data: Dict[str, Any]) -> None: - if name in REDEFINED_BUILTINS: - yaml_data["pylintDisable"] = "redefined-builtin" - - -def pad_builtin_namespaces(namespace: str) -> str: - items = namespace.split(".") - if items[0] in BUILTIN_PACKAGES: - items[0] = items[0] + "_" - return ".".join(items) - - -def pad_special_chars(name: str) -> str: - return re.sub(r"[^A-z0-9_]", "_", name) diff --git a/packages/autorest.python/generator/pygen/preprocess/python_mappings.py b/packages/autorest.python/generator/pygen/preprocess/python_mappings.py deleted file mode 100644 index 4edebd7120b..00000000000 --- a/packages/autorest.python/generator/pygen/preprocess/python_mappings.py +++ /dev/null @@ -1,222 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from enum import Enum - -basic_latin_chars = { - " ": "Space", - "!": "ExclamationMark", - '"': "QuotationMark", - "#": "NumberSign", - "$": "DollarSign", - "%": "PercentSign", - "&": "Ampersand", - "'": "Apostrophe", - "(": "LeftParenthesis", - ")": "RightParenthesis", - "*": "Asterisk", - "+": "PlusSign", - ",": "Comma", - "-": "HyphenMinus", - ".": "FullStop", - "/": "Slash", - "0": "Zero", - "1": "One", - "2": "Two", - "3": "Three", - "4": "Four", - "5": "Five", - "6": "Six", - "7": "Seven", - "8": "Eight", - "9": "Nine", - ":": "Colon", - ";": "Semicolon", - "<": "LessThanSign", - "=": "EqualSign", - ">": "GreaterThanSign", - "?": "QuestionMark", - "@": "AtSign", - "[": "LeftSquareBracket", - "\\": "Backslash", - "]": "RightSquareBracket", - "^": "CircumflexAccent", - "`": "GraveAccent", - "{": "LeftCurlyBracket", - "|": "VerticalBar", - "}": "RightCurlyBracket", - "~": "Tilde", -} - - -class PadType(str, Enum): - MODEL = "Model" - METHOD = "_method" - PARAMETER = "_parameter" - ENUM = "_enum" - PROPERTY = "_property" - OPERATION_GROUP = "Operations" - - -_always_reserved = [ - "and", - "as", - "assert", - "break", - "class", - "continue", - "def", - "del", - "elif", - "else", - "except", - "exec", - "finally", - "for", - "from", - "global", - "if", - "import", - "in", - "is", - "lambda", - "not", - "or", - "pass", - "raise", - "return", - "try", - "while", - "with", - "yield", - "async", - "await", - "int", -] - -RESERVED_MODEL_PROPERTIES = [ - "keys", - "items", - "values", - "popitem", - "clear", - "update", - "setdefault", - "pop", - "get", -] - -RESERVED_WORDS = { - PadType.METHOD: [*_always_reserved], - PadType.PARAMETER: [ - "self", - # these are kwargs we've reserved for our generated operations - "content_type", - "accept", - "cls", - "polling", - "continuation_token", # for LRO calls - # these are transport kwargs - # https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/CLIENT_LIBRARY_DEVELOPER.md#transport - "connection_timeout", - "connection_verify", - "connection_cert", - "connection_data_block_size", - "use_env_settings", - # the following aren't in the readme, but Xiang said these are also transport kwargs - "read_timeout", - "proxies", - "cookies", - # these are policy kwargs - # https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/core/azure-core/CLIENT_LIBRARY_DEVELOPER.md#available-policies - "base_headers", - "headers", - "request_id", - "auto_request_id", - "base_user_agent", - "user_agent", - "user_agent_overwrite", - "user_agent_use_env", - "user_agent", - "sdk_moniker", - "logging_enable", - "logger", - "response_encoding", - "proxies", - "raw_request_hook", - "raw_response_hook", - "network_span_namer", - "tracing_attributes", - "permit_redirects", - "redirect_max", - "redirect_remove_headers", - "redirect_on_status_codes", - "permit_redirects", - "redirect_max", - "redirect_remove_headers", - "redirect_on_status_codes", - "retry_total", - "retry_connect", - "retry_read", - "retry_status", - "retry_backoff_factor", - "retry_backoff_max", - "retry_mode", - "retry_on_status_codes", - "retry_total", - "retry_connect", - "retry_read", - "retry_status", - "retry_backoff_factor", - "retry_backoff_max", - "retry_mode", - "retry_on_status_codes", - *_always_reserved, - ], - PadType.MODEL: [*_always_reserved], - PadType.PROPERTY: ["self", *_always_reserved], - PadType.ENUM: ["mro", *_always_reserved], - PadType.OPERATION_GROUP: [*_always_reserved], -} - -CADL_RESERVED_WORDS = { - PadType.PARAMETER: ["stream"], - PadType.PROPERTY: RESERVED_MODEL_PROPERTIES, -} - -REDEFINED_BUILTINS = [ # we don't pad, but we need to do lint ignores - "id", - "min", - "max", - "filter", - "property", -] - -BUILTIN_PACKAGES = [ - "array", - "atexit", - "binascii", - "builtins", - "cmath", - "errno", - "faulthandler", - "fcntl", - "gc", - "grp", - "itertools", - "marshal", - "math", - "posix", - "pwd", - "pyexpat", - "select", - "spwd", - "sys", - "syslog", - "time", - "unicodedata", - "xxsubtype", - "zlib", -] diff --git a/packages/autorest.python/generator/pygen/utils.py b/packages/autorest.python/generator/pygen/utils.py deleted file mode 100644 index 95bf493bde8..00000000000 --- a/packages/autorest.python/generator/pygen/utils.py +++ /dev/null @@ -1,149 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from typing import Any, Dict, Tuple, List -import re -import argparse - - -def update_enum_value(name: str, value: Any, description: str, enum_type: Dict[str, Any]) -> Dict[str, Any]: - return { - "name": name, - "type": "enumvalue", - "value": value, - "description": description, - "enumType": enum_type, - "valueType": enum_type["valueType"], - } - - -def to_snake_case(name: str) -> str: - def replace_upper_characters(m) -> str: - match_str = m.group().lower() - if m.start() > 0 and name[m.start() - 1] == "_": - # we are good if a '_' already exists - return match_str - # the first letter should not have _ - prefix = "_" if m.start() > 0 else "" - - # we will add an extra _ if there are multiple upper case chars together - next_non_upper_case_char_location = m.start() + len(match_str) - if ( - len(match_str) > 2 - and len(name) - next_non_upper_case_char_location > 1 - and name[next_non_upper_case_char_location].isalpha() - ): - return prefix + match_str[: len(match_str) - 1] + "_" + match_str[len(match_str) - 1] - - return prefix + match_str - - result = re.sub("[A-Z]+", replace_upper_characters, name) - return result.replace(" ", "_").replace("__", "_").replace("-", "") - - -def parse_args( - need_cadl_file: bool = True, -) -> Tuple[argparse.Namespace, Dict[str, Any]]: - parser = argparse.ArgumentParser( - description="Run mypy against target folder. Add a local custom plugin to the path prior to execution. " - ) - parser.add_argument( - "--output-folder", - dest="output_folder", - help="Output folder for generated SDK", - required=True, - ) - parser.add_argument( - "--cadl-file", - dest="cadl_file", - help="Serialized cadl file", - required=need_cadl_file, - ) - parser.add_argument( - "--debug", - dest="debug", - help="Debug mode", - required=False, - action="store", - ) - args, unknown_args = parser.parse_known_args() - - def _get_value(value: Any) -> Any: - if value == "true": - return True - if value == "false": - return False - try: - return int(value) - except ValueError: - pass - return value - - unknown_args_ret = { - ua.strip("--").split("=", maxsplit=1)[0]: _get_value( # pylint: disable=bad-str-strip-call - ua.strip("--").split("=", maxsplit=1)[1] # pylint: disable=bad-str-strip-call - ) - for ua in unknown_args - } - return args, unknown_args_ret - - -def get_body_type_for_description(body_parameter: Dict[str, Any]) -> str: - if body_parameter["type"]["type"] == "binary": - return "binary" - if body_parameter["type"]["type"] == "string": - return "string" - return "JSON" - - -# used if we want to get a string / binary type etc -KNOWN_TYPES: Dict[str, Dict[str, Any]] = { - "string": {"type": "string"}, - "binary": {"type": "binary"}, - "anydict": {"type": "dict", "elementType": {"type": "any"}}, - "any-object": {"type": "any-object"}, -} - -JSON_REGEXP = re.compile(r"^(application|text)/(.+\+)?json$") - - -def build_policies( - is_arm: bool, - async_mode: bool, - *, - is_azure_flavor: bool = False, - tracing: bool = True, -) -> List[str]: - if is_azure_flavor: - # for Azure - async_prefix = "Async" if async_mode else "" - policies = [ - "policies.RequestIdPolicy(**kwargs)", - "self._config.headers_policy", - "self._config.user_agent_policy", - "self._config.proxy_policy", - "policies.ContentDecodePolicy(**kwargs)", - (f"{async_prefix}ARMAutoResourceProviderRegistrationPolicy()" if is_arm else None), - "self._config.redirect_policy", - "self._config.retry_policy", - "self._config.authentication_policy", - "self._config.custom_hook_policy", - "self._config.logging_policy", - "policies.DistributedTracingPolicy(**kwargs)" if tracing else None, - "policies.SensitiveHeaderCleanupPolicy(**kwargs) if self._config.redirect_policy else None", - "self._config.http_logging_policy", - ] - else: - # for non-Azure - policies = [ - "self._config.headers_policy", - "self._config.user_agent_policy", - "self._config.proxy_policy", - "policies.ContentDecodePolicy(**kwargs)", - "self._config.retry_policy", - "self._config.authentication_policy", - "self._config.logging_policy", - ] - return [p for p in policies if p] diff --git a/packages/autorest.python/generator/requirements.txt b/packages/autorest.python/generator/requirements.txt deleted file mode 100644 index bebbbb5a645..00000000000 --- a/packages/autorest.python/generator/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -black==24.4.0 -click==8.1.3 -docutils==0.19 -Jinja2==3.1.4 -m2r2==0.3.3 -MarkupSafe==2.1.2 -mistune==0.8.4 -pathspec==0.11.1 -platformdirs==3.2.0 -PyYAML==6.0.1 -tomli==2.0.1 -setuptools==69.2.0 diff --git a/packages/autorest.python/generator/setup.py b/packages/autorest.python/generator/setup.py deleted file mode 100644 index b19d4f2692e..00000000000 --- a/packages/autorest.python/generator/setup.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python - -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - - -import os -import re - -from setuptools import setup, find_packages - - -# Version extraction inspired from 'requests' -with open(os.path.join("pygen", "_version.py"), "r") as fd: - version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) - -if not version: - raise RuntimeError("Cannot find version information") - -setup( - name="pygen", - version=version, - description="Core Library for Python Generation", - long_description=open("README.md", "r").read(), - long_description_content_type="text/markdown", - license="MIT License", - author="Microsoft Corporation", - author_email="azpysdkhelp@microsoft.com", - url="https://github.com/Azure/autorest.python/packages/core", - classifiers=[ - "Development Status :: 4 - Beta", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "License :: OSI Approved :: MIT License", - ], - packages=find_packages( - exclude=[ - "test", - ] - ), - install_requires=[ - "Jinja2 >= 2.11", # I need "include" and auto-context + blank line are not indented by default - "pyyaml", - "m2r2", - "black", - ], -) diff --git a/packages/typespec-python/scripts/prepare.py b/packages/typespec-python/scripts/prepare.py index 6ba7db0d5b0..61c519852e5 100644 --- a/packages/typespec-python/scripts/prepare.py +++ b/packages/typespec-python/scripts/prepare.py @@ -6,6 +6,7 @@ # license information. # -------------------------------------------------------------------------- import sys +import os import argparse if not sys.version_info >= (3, 8, 0): diff --git a/packages/typespec-python/scripts/run_tsp.py b/packages/typespec-python/scripts/run_tsp.py index 8f68f260cbb..e7a73943b9f 100644 --- a/packages/typespec-python/scripts/run_tsp.py +++ b/packages/typespec-python/scripts/run_tsp.py @@ -34,7 +34,7 @@ breakpoint() # pylint: disable=undefined-variable # run m2r - python_run(venv_context, "m2r", command=sys.argv[1:]) - python_run(venv_context, "preprocess.__init__", command=sys.argv[1:]) - python_run(venv_context, "codegen.__init__", command=sys.argv[1:]) - python_run(venv_context, "black", command=sys.argv[1:]) + python_run(venv_context, "pygen.m2r", command=sys.argv[1:]) + python_run(venv_context, "pygen.preprocess.__init__", command=sys.argv[1:]) + python_run(venv_context, "pygen.codegen.__init__", command=sys.argv[1:]) + python_run(venv_context, "pygen.black", command=sys.argv[1:]) diff --git a/packages/typespec-python/scripts/venvtools.py b/packages/typespec-python/scripts/venvtools.py index 690f09e174e..112eb9ca95a 100644 --- a/packages/typespec-python/scripts/venvtools.py +++ b/packages/typespec-python/scripts/venvtools.py @@ -73,7 +73,7 @@ def python_run(venv_context, module, command=None, *, additional_dir="."): print("Executing: {}".format(" ".join(cmd_line))) subprocess.run( cmd_line, - cwd=_ROOT_DIR / "generator" / "pygen" / additional_dir, + cwd=_ROOT_DIR / "generator" / additional_dir, check=True, ) except subprocess.CalledProcessError as err: From a6ff383600a67ea87abd878be418782cd0aa46db Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 6 Jun 2024 18:25:49 -0400 Subject: [PATCH 67/75] move venv into top level of package --- .../autorest.python/scripts/copy-generator.js | 5 ++++ packages/autorest.python/scripts/install.py | 23 +++++++++++-------- packages/autorest.python/scripts/start.py | 2 +- packages/typespec-python/scripts/install.py | 2 +- packages/typespec-python/scripts/prepare.py | 2 +- packages/typespec-python/scripts/run_tsp.py | 2 +- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/packages/autorest.python/scripts/copy-generator.js b/packages/autorest.python/scripts/copy-generator.js index 9a3c1f8dbf8..50bb5bb3e91 100644 --- a/packages/autorest.python/scripts/copy-generator.js +++ b/packages/autorest.python/scripts/copy-generator.js @@ -6,5 +6,10 @@ const url = require('url'); const sourceDir = path.join(__dirname, "..", "node_modules", "@azure-tools", "typespec-python", "generator"); const destDir = path.join(__dirname, "..", "generator"); +// Delete the destination directory if it exists +if (fs.existsSync(destDir)) { + fs.removeSync(destDir); +} + // Copy the source directory to the destination directory fs.copySync(sourceDir, destDir); diff --git a/packages/autorest.python/scripts/install.py b/packages/autorest.python/scripts/install.py index 2610b9227f3..415ac9455d6 100644 --- a/packages/autorest.python/scripts/install.py +++ b/packages/autorest.python/scripts/install.py @@ -24,22 +24,25 @@ # Now we have pip and Py >= 3.8, go to work from pathlib import Path -import shutil -from venvtools import python_run +from venvtools import ExtendedEnvBuilder, python_run _ROOT_DIR = Path(__file__).parent.parent def main(): - venv_path = _ROOT_DIR / "generator" / "venv" - assert venv_path.exists() # Otherwise install was not done - - env_builder = venv.EnvBuilder(with_pip=True) - venv_context = env_builder.ensure_directories(venv_path) - # install autorest.python specific stuff into it - python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) - python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) + venv_path = _ROOT_DIR / "venv" + if venv_path.exists(): + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + else: + env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True) + env_builder.create(venv_path) + venv_context = env_builder.context + + python_run(venv_context, "pip", ["install", "-U", "pip"]) + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/generator/requirements.txt"]) + python_run(venv_context, "pip", ["install", "-e", f"{_ROOT_DIR}/generator"]) if __name__ == "__main__": diff --git a/packages/autorest.python/scripts/start.py b/packages/autorest.python/scripts/start.py index c9321edd01a..714fc50155f 100644 --- a/packages/autorest.python/scripts/start.py +++ b/packages/autorest.python/scripts/start.py @@ -19,7 +19,7 @@ def main(): - venv_path = _ROOT_DIR / "generator" / "venv" + venv_path = _ROOT_DIR / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/packages/typespec-python/scripts/install.py b/packages/typespec-python/scripts/install.py index 80138a01803..415ac9455d6 100644 --- a/packages/typespec-python/scripts/install.py +++ b/packages/typespec-python/scripts/install.py @@ -31,7 +31,7 @@ def main(): - venv_path = _ROOT_DIR / "generator" / "venv" + venv_path = _ROOT_DIR / "venv" if venv_path.exists(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) diff --git a/packages/typespec-python/scripts/prepare.py b/packages/typespec-python/scripts/prepare.py index 61c519852e5..f01a5256676 100644 --- a/packages/typespec-python/scripts/prepare.py +++ b/packages/typespec-python/scripts/prepare.py @@ -21,7 +21,7 @@ def main(): - venv_path = _ROOT_DIR / "generator" / "venv" + venv_path = _ROOT_DIR / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done diff --git a/packages/typespec-python/scripts/run_tsp.py b/packages/typespec-python/scripts/run_tsp.py index e7a73943b9f..243512cd1ee 100644 --- a/packages/typespec-python/scripts/run_tsp.py +++ b/packages/typespec-python/scripts/run_tsp.py @@ -14,7 +14,7 @@ _LOGGER = logging.getLogger(__name__) if __name__ == "__main__": - venv_path = _ROOT_DIR / "generator" / "venv" + venv_path = _ROOT_DIR / "venv" venv_prexists = venv_path.exists() assert venv_prexists # Otherwise install was not done From 1d757bbba991a220c6a68040948d50f0f02d8004 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Thu, 6 Jun 2024 18:41:44 -0400 Subject: [PATCH 68/75] get autorest.python running --- packages/autorest.python/dev_requirements.txt | 4 +- packages/autorest.python/package.json | 1 + packages/autorest.python/requirements.txt | 13 ++++++- packages/autorest.python/scripts/install.py | 2 +- packages/autorest.python/scripts/prepare.py | 38 +++++++++++++++++++ 5 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 packages/autorest.python/scripts/prepare.py diff --git a/packages/autorest.python/dev_requirements.txt b/packages/autorest.python/dev_requirements.txt index 9859a339b00..d458756b7d8 100644 --- a/packages/autorest.python/dev_requirements.txt +++ b/packages/autorest.python/dev_requirements.txt @@ -1,5 +1,5 @@ -e . --r ../../../eng/requirements.txt --r ../../../eng/dev_requirements.txt +-r ../../eng/requirements.txt +-r ../../eng/dev_requirements.txt ptvsd==4.3.2 types-PyYAML==6.0.12.8 diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index a24e324cac6..1d1807ebf4d 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -4,6 +4,7 @@ "description": "The Python extension for generators in AutoRest.", "scripts": { "start": "node ./scripts/run-python3.js ./scripts/start.py", + "prepare": "node ./scripts/run-python3.js ./scripts/prepare.py", "copy-generator": "node ./scripts/copy-generator.js", "install": "npm run copy-generator && node ./scripts/run-python3.js ./scripts/install.py", "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index edfb2c92452..3f977ca26b8 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -1,2 +1,13 @@ +black==24.4.0 +click==8.1.3 +docutils==0.19 +Jinja2==3.1.4 +m2r2==0.3.3 +MarkupSafe==2.1.2 +mistune==0.8.4 +pathspec==0.11.1 +platformdirs==3.2.0 +PyYAML==6.0.1 +tomli==2.0.1 +setuptools==69.2.0 json-rpc==1.14.0 --e ./generator diff --git a/packages/autorest.python/scripts/install.py b/packages/autorest.python/scripts/install.py index 415ac9455d6..7b687f1171d 100644 --- a/packages/autorest.python/scripts/install.py +++ b/packages/autorest.python/scripts/install.py @@ -41,7 +41,7 @@ def main(): venv_context = env_builder.context python_run(venv_context, "pip", ["install", "-U", "pip"]) - python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/generator/requirements.txt"]) + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/requirements.txt"]) python_run(venv_context, "pip", ["install", "-e", f"{_ROOT_DIR}/generator"]) diff --git a/packages/autorest.python/scripts/prepare.py b/packages/autorest.python/scripts/prepare.py new file mode 100644 index 00000000000..eb89f7555b2 --- /dev/null +++ b/packages/autorest.python/scripts/prepare.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys +import os +import argparse + +if not sys.version_info >= (3, 8, 0): + raise Exception("Autorest for Python extension requires Python 3.8 at least") + +from pathlib import Path +import venv + +from venvtools import python_run + +_ROOT_DIR = Path(__file__).parent.parent + + +def main(): + venv_path = _ROOT_DIR / "venv" + venv_prexists = venv_path.exists() + + assert venv_prexists # Otherwise install was not done + + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + try: + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/dev_requirements.txt"]) + except FileNotFoundError as e: + raise ValueError(e.filename) + + +if __name__ == "__main__": + main() From 36bfb77fb2b918c1918dc71c1a4b28bd88e128da Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 7 Jun 2024 15:14:19 -0400 Subject: [PATCH 69/75] add generator to dev reqs --- packages/autorest.python/dev_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/autorest.python/dev_requirements.txt b/packages/autorest.python/dev_requirements.txt index d458756b7d8..0ac67bacc8a 100644 --- a/packages/autorest.python/dev_requirements.txt +++ b/packages/autorest.python/dev_requirements.txt @@ -1,4 +1,5 @@ -e . +-e ./generator -r ../../eng/requirements.txt -r ../../eng/dev_requirements.txt ptvsd==4.3.2 From 38682534e69720068c0d30abb6989e44b2fb7aec Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 7 Jun 2024 15:29:25 -0400 Subject: [PATCH 70/75] fix venvtools path --- packages/typespec-python/scripts/run_tsp.py | 8 ++++---- packages/typespec-python/scripts/venvtools.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/typespec-python/scripts/run_tsp.py b/packages/typespec-python/scripts/run_tsp.py index 243512cd1ee..0cb34ee9277 100644 --- a/packages/typespec-python/scripts/run_tsp.py +++ b/packages/typespec-python/scripts/run_tsp.py @@ -34,7 +34,7 @@ breakpoint() # pylint: disable=undefined-variable # run m2r - python_run(venv_context, "pygen.m2r", command=sys.argv[1:]) - python_run(venv_context, "pygen.preprocess.__init__", command=sys.argv[1:]) - python_run(venv_context, "pygen.codegen.__init__", command=sys.argv[1:]) - python_run(venv_context, "pygen.black", command=sys.argv[1:]) + python_run(venv_context, "generator.pygen.m2r", command=sys.argv[1:]) + python_run(venv_context, "generator.pygen.preprocess.__init__", command=sys.argv[1:]) + python_run(venv_context, "generator.pygen.codegen.__init__", command=sys.argv[1:]) + python_run(venv_context, "generator.pygen.black", command=sys.argv[1:]) diff --git a/packages/typespec-python/scripts/venvtools.py b/packages/typespec-python/scripts/venvtools.py index 112eb9ca95a..944ff96e36b 100644 --- a/packages/typespec-python/scripts/venvtools.py +++ b/packages/typespec-python/scripts/venvtools.py @@ -73,7 +73,7 @@ def python_run(venv_context, module, command=None, *, additional_dir="."): print("Executing: {}".format(" ".join(cmd_line))) subprocess.run( cmd_line, - cwd=_ROOT_DIR / "generator" / additional_dir, + cwd=_ROOT_DIR / additional_dir, check=True, ) except subprocess.CalledProcessError as err: From 47ed0b2e7383e08b908226f75b705632d16b56e8 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 7 Jun 2024 15:33:26 -0400 Subject: [PATCH 71/75] update dev reqs --- packages/typespec-python/generator/dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/typespec-python/generator/dev_requirements.txt b/packages/typespec-python/generator/dev_requirements.txt index 9859a339b00..867580b72a4 100644 --- a/packages/typespec-python/generator/dev_requirements.txt +++ b/packages/typespec-python/generator/dev_requirements.txt @@ -1,4 +1,4 @@ --e . +-e ./generator -r ../../../eng/requirements.txt -r ../../../eng/dev_requirements.txt ptvsd==4.3.2 From ad8123b6001c8704b2f2c704f7aeade5d1ffd6e9 Mon Sep 17 00:00:00 2001 From: iscai-msft Date: Fri, 7 Jun 2024 16:01:45 -0400 Subject: [PATCH 72/75] add generator to published files --- packages/autorest.python/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 5b770ab5ed1..1f23ed058b3 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -40,6 +40,7 @@ "autorest/**/*.jinja2", "scripts/", "setup.py", - "requirements.txt" + "requirements.txt", + "generator/" ] } From 9c3499e3b80e81b4f4de7f8fdd144c9334aed602 Mon Sep 17 00:00:00 2001 From: tadelesh Date: Wed, 3 Jul 2024 11:06:09 +0800 Subject: [PATCH 73/75] update script --- packages/autorest.python/package.json | 4 ++-- packages/autorest.python/scripts/copy-generator.js | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index e2d9daf09a0..692e55271ba 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -5,8 +5,8 @@ "scripts": { "start": "node ./scripts/run-python3.js ./scripts/start.py", "prepare": "node ./scripts/run-python3.js ./scripts/prepare.py", - "copy-generator": "node ./scripts/copy-generator.js", - "install": "npm run copy-generator && node ./scripts/run-python3.js ./scripts/install.py", + "build": "node ./scripts/copy-generator.js", + "install": "node ./scripts/run-python3.js ./scripts/install.py", "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" }, "main": "index.js", diff --git a/packages/autorest.python/scripts/copy-generator.js b/packages/autorest.python/scripts/copy-generator.js index 50bb5bb3e91..d2e72b2147c 100644 --- a/packages/autorest.python/scripts/copy-generator.js +++ b/packages/autorest.python/scripts/copy-generator.js @@ -1,6 +1,5 @@ const fs = require('fs-extra'); const path = require('path'); -const url = require('url'); // Define the source and destination directories const sourceDir = path.join(__dirname, "..", "node_modules", "@azure-tools", "typespec-python", "generator"); From 75ef50fbb64dc73f2bda309e8b033b32b03fae7f Mon Sep 17 00:00:00 2001 From: tadelesh Date: Wed, 3 Jul 2024 13:41:07 +0800 Subject: [PATCH 74/75] update --- packages/autorest.python/package.json | 7 +-- .../autorest.python/scripts/copy-generator.js | 52 +++++++++++++++---- .../typespec-python/src/external-process.ts | 30 +---------- packages/typespec-python/src/index.ts | 1 + pnpm-lock.yaml | 11 ++-- 5 files changed, 50 insertions(+), 51 deletions(-) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 692e55271ba..316478f537e 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -5,8 +5,7 @@ "scripts": { "start": "node ./scripts/run-python3.js ./scripts/start.py", "prepare": "node ./scripts/run-python3.js ./scripts/prepare.py", - "build": "node ./scripts/copy-generator.js", - "install": "node ./scripts/run-python3.js ./scripts/install.py", + "install": "node ./scripts/copy-generator.js && node ./scripts/run-python3.js ./scripts/install.py", "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" }, "main": "index.js", @@ -28,7 +27,6 @@ "dependencies": { "@autorest/system-requirements": "~1.0.2", "fs-extra": "~11.2.0", - "semver": "~7.6.2", "@azure-tools/typespec-python": "workspace:^" }, "devDependencies": { @@ -40,7 +38,6 @@ "autorest/**/*.jinja2", "scripts/", "setup.py", - "requirements.txt", - "generator/" + "requirements.txt" ] } diff --git a/packages/autorest.python/scripts/copy-generator.js b/packages/autorest.python/scripts/copy-generator.js index d2e72b2147c..09f7e89a19b 100644 --- a/packages/autorest.python/scripts/copy-generator.js +++ b/packages/autorest.python/scripts/copy-generator.js @@ -1,14 +1,46 @@ -const fs = require('fs-extra'); -const path = require('path'); +const { realpath, stat, readFile } = require("fs/promises"); +const fs = require("fs-extra"); +const path = require("path"); -// Define the source and destination directories -const sourceDir = path.join(__dirname, "..", "node_modules", "@azure-tools", "typespec-python", "generator"); -const destDir = path.join(__dirname, "..", "generator"); +async function resolveAndCopy() { + let resolved; -// Delete the destination directory if it exists -if (fs.existsSync(destDir)) { - fs.removeSync(destDir); + // Resolve @azure-tools/typespec-python module path + try { + const module = await import("@azure-tools/typespec-python"); + + const host = { + realpath, + readFile: async (path) => await readFile(path, "utf-8"), + stat, + }; + + resolved = await module.resolveModule(host, "@azure-tools/typespec-python", { + baseDir: process.cwd(), + }); + if (resolved.type !== "module") { + throw new Error( + `Error resolving "@azure-tools/typespec-python", expected to find a node module but found a file: "${resolved.path}".` + ); + } + } catch (err) { + throw new Error( + `Error resolving "@azure-tools/typespec-python", could not find the module: "${err}".` + ); + } + + // Define the source and destination directories + const sourceDir = path.join(resolved.path, "generator"); + const destDir = path.join(__dirname, "..", "generator"); + + // Delete the destination directory if it exists + if (fs.existsSync(destDir)) { + fs.removeSync(destDir); + } + + // Copy the source directory to the destination directory + fs.copySync(sourceDir, destDir); } -// Copy the source directory to the destination directory -fs.copySync(sourceDir, destDir); + +resolveAndCopy().catch(console.error); diff --git a/packages/typespec-python/src/external-process.ts b/packages/typespec-python/src/external-process.ts index 74eb017c1cf..bb2719aed0f 100644 --- a/packages/typespec-python/src/external-process.ts +++ b/packages/typespec-python/src/external-process.ts @@ -1,4 +1,4 @@ -import { CompilerHost, joinPaths, Program, resolveModule, ResolveModuleHost } from "@typespec/compiler"; +import { joinPaths } from "@typespec/compiler"; import { ChildProcess, spawn, SpawnOptions } from "child_process"; import { randomUUID } from "crypto"; import { mkdir, writeFile } from "fs/promises"; @@ -50,31 +50,3 @@ export async function execAsync( }); }); } - -/** - * Resolve root of module. - * @param program Cadl Program - * @param name Name of the module (e.g. "@autorest/python") - * @param baseDir Base directory to start looking from. Use `dirname(fileURLToPath(import.meta.url))` for current dir. - * @returns Path to the module root if found. - */ -export async function resolveModuleRoot(program: Program, name: string, baseDir: string): Promise { - const moduleHost = getResolveModuleHost(program.host); - - const resolved = await resolveModule(moduleHost, name, { - baseDir, - }); - - return resolved.path; -} - -function getResolveModuleHost(host: CompilerHost): ResolveModuleHost { - return { - realpath: host.realpath, - stat: host.stat, - readFile: async (path) => { - const file = await host.readFile(path); - return file.text; - }, - }; -} diff --git a/packages/typespec-python/src/index.ts b/packages/typespec-python/src/index.ts index 627847c619c..802df77e1e6 100644 --- a/packages/typespec-python/src/index.ts +++ b/packages/typespec-python/src/index.ts @@ -1,2 +1,3 @@ export * from "./emitter.js"; export { $lib } from "./lib.js"; +export { resolveModule } from "@typespec/compiler"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index faf4559ed5d..b395db6d924 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,9 +1,5 @@ lockfileVersion: '6.0' -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - importers: .: @@ -56,9 +52,6 @@ importers: fs-extra: specifier: ~11.2.0 version: 11.2.0 - semver: - specifier: ~7.6.2 - version: 7.6.2 devDependencies: '@microsoft.azure/autorest.testserver': specifier: ^3.3.46 @@ -6601,3 +6594,7 @@ packages: /zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} dev: true + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false From 9730829818c96fc275f35d87c635b110a4bb5b3b Mon Sep 17 00:00:00 2001 From: tadelesh Date: Wed, 3 Jul 2024 15:25:54 +0800 Subject: [PATCH 75/75] update --- packages/autorest.python/package.json | 10 ++-- .../autorest.python/scripts/copy-generator.js | 49 +++++-------------- packages/typespec-python/src/index.ts | 1 - pnpm-lock.yaml | 6 +-- 4 files changed, 20 insertions(+), 46 deletions(-) diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 316478f537e..1ddb7fbe15f 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -5,6 +5,7 @@ "scripts": { "start": "node ./scripts/run-python3.js ./scripts/start.py", "prepare": "node ./scripts/run-python3.js ./scripts/prepare.py", + "build": "node ./scripts/copy-generator.js --force", "install": "node ./scripts/copy-generator.js && node ./scripts/run-python3.js ./scripts/install.py", "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" }, @@ -26,18 +27,19 @@ "homepage": "https://github.com/Azure/autorest.python/blob/main/README.md", "dependencies": { "@autorest/system-requirements": "~1.0.2", - "fs-extra": "~11.2.0", - "@azure-tools/typespec-python": "workspace:^" + "fs-extra": "~11.2.0" }, "devDependencies": { "@microsoft.azure/autorest.testserver": "^3.3.46", - "typescript": "~5.1.3" + "typescript": "~5.1.3", + "@azure-tools/typespec-python": "workspace:^" }, "files": [ "autorest/**/*.py", "autorest/**/*.jinja2", "scripts/", "setup.py", - "requirements.txt" + "requirements.txt", + "generator/" ] } diff --git a/packages/autorest.python/scripts/copy-generator.js b/packages/autorest.python/scripts/copy-generator.js index 09f7e89a19b..c85af504e10 100644 --- a/packages/autorest.python/scripts/copy-generator.js +++ b/packages/autorest.python/scripts/copy-generator.js @@ -1,46 +1,19 @@ -const { realpath, stat, readFile } = require("fs/promises"); const fs = require("fs-extra"); const path = require("path"); -async function resolveAndCopy() { - let resolved; +const force = process.argv[2] === "--force" ? true : false; - // Resolve @azure-tools/typespec-python module path - try { - const module = await import("@azure-tools/typespec-python"); +const typespecModulePath = path.join(__dirname, "..", "node_modules", "@azure-tools", "typespec-python"); - const host = { - realpath, - readFile: async (path) => await readFile(path, "utf-8"), - stat, - }; +// Define the source and destination directories +const sourceDir = path.join(typespecModulePath, "generator"); +const destDir = path.join(__dirname, "..", "generator"); - resolved = await module.resolveModule(host, "@azure-tools/typespec-python", { - baseDir: process.cwd(), - }); - if (resolved.type !== "module") { - throw new Error( - `Error resolving "@azure-tools/typespec-python", expected to find a node module but found a file: "${resolved.path}".` - ); - } - } catch (err) { - throw new Error( - `Error resolving "@azure-tools/typespec-python", could not find the module: "${err}".` - ); - } - - // Define the source and destination directories - const sourceDir = path.join(resolved.path, "generator"); - const destDir = path.join(__dirname, "..", "generator"); - - // Delete the destination directory if it exists - if (fs.existsSync(destDir)) { - fs.removeSync(destDir); - } - - // Copy the source directory to the destination directory - fs.copySync(sourceDir, destDir); +// Delete the destination directory if it exists +if (fs.existsSync(destDir)) { + if (force) fs.removeSync(destDir); + else process.exit(0); } - -resolveAndCopy().catch(console.error); +// Copy the source directory to the destination directory +fs.copySync(sourceDir, destDir); diff --git a/packages/typespec-python/src/index.ts b/packages/typespec-python/src/index.ts index 802df77e1e6..627847c619c 100644 --- a/packages/typespec-python/src/index.ts +++ b/packages/typespec-python/src/index.ts @@ -1,3 +1,2 @@ export * from "./emitter.js"; export { $lib } from "./lib.js"; -export { resolveModule } from "@typespec/compiler"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b395db6d924..b57afdba416 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -46,13 +46,13 @@ importers: '@autorest/system-requirements': specifier: ~1.0.2 version: 1.0.2 - '@azure-tools/typespec-python': - specifier: workspace:^ - version: link:../typespec-python fs-extra: specifier: ~11.2.0 version: 11.2.0 devDependencies: + '@azure-tools/typespec-python': + specifier: workspace:^ + version: link:../typespec-python '@microsoft.azure/autorest.testserver': specifier: ^3.3.46 version: 3.3.46