diff --git a/.chronus/changes/link_emitters-2024-4-13-14-39-2.md b/.chronus/changes/link_emitters-2024-4-13-14-39-2.md new file mode 100644 index 00000000000..91280ecdf58 --- /dev/null +++ b/.chronus/changes/link_emitters-2024-4-13-14-39-2.md @@ -0,0 +1,8 @@ +--- +changeKind: feature +packages: + - "@autorest/python" + - "@azure-tools/typespec-python" +--- + +add package pygen that both autorest.python and typespec-python will rely on diff --git a/.gitignore b/.gitignore index 5fbc2245061..2bbf0a3b855 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,4 @@ node_modules/ # Generated test folders test/services/*/_generated +**/autorest.python/generator diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml index 28de9e9bb5a..5cca332de3c 100644 --- a/eng/pipelines/ci-template.yml +++ b/eng/pipelines/ci-template.yml @@ -69,10 +69,18 @@ steps: displayName: Build project workingDirectory: $(Build.SourcesDirectory)/autorest.python/ + - script: pip list + displayName: List installed packages + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pip install -r dev_requirements.txt displayName: Pip install dev requirements workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pip list + displayName: List installed packages + workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} + - script: pylint ${{parameters.pythonFolderName}} displayName: Pylint workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}} diff --git a/packages/autorest.python/autorest/__init__.py b/packages/autorest.python/autorest/__init__.py index 1fa6e0b83c8..e5682d9aad0 100644 --- a/packages/autorest.python/autorest/__init__.py +++ b/packages/autorest.python/autorest/__init__.py @@ -5,59 +5,18 @@ # -------------------------------------------------------------------------- import logging from pathlib import Path -import json -from abc import ABC, abstractmethod +from abc import abstractmethod from typing import Any, Dict, Union, List import yaml +from pygen import ReaderAndWriter, Plugin, YamlUpdatePlugin from .jsonrpc import AutorestAPI -from ._version import VERSION -__version__ = VERSION _LOGGER = logging.getLogger(__name__) -class ReaderAndWriter: - def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None: - self.output_folder = Path(output_folder) - self._list_file: List[str] = [] - try: - with open( - Path(self.output_folder) / Path("..") / Path("python.json"), - "r", - encoding="utf-8-sig", - ) as fd: - python_json = json.load(fd) - except Exception: # pylint: disable=broad-except - python_json = {} - self.options = kwargs - if python_json: - _LOGGER.warning("Loading python.json file. This behavior will be depreacted") - self.options.update(python_json) - - def read_file(self, path: Union[str, Path]) -> str: - """Directly reading from disk""" - # make path relative to output folder - try: - with open(self.output_folder / Path(path), "r", encoding="utf-8-sig") as fd: - return fd.read() - except FileNotFoundError: - return "" - - def write_file(self, filename: Union[str, Path], file_content: str) -> None: - """Directly writing to disk""" - file_folder = Path(filename).parent - if not Path.is_dir(self.output_folder / file_folder): - Path.mkdir(self.output_folder / file_folder, parents=True) - with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd: - fd.write(file_content) - - def list_file(self) -> List[str]: - return [str(f) for f in self.output_folder.glob("**/*") if f.is_file()] - - class ReaderAndWriterAutorest(ReaderAndWriter): def __init__(self, *, output_folder: Union[str, Path], autorestapi: AutorestAPI) -> None: super().__init__(output_folder=output_folder) @@ -73,23 +32,6 @@ def list_file(self) -> List[str]: return self._autorestapi.list_inputs() -class Plugin(ReaderAndWriter, ABC): - """A base class for autorest plugin. - - :param autorestapi: An autorest API instance - """ - - @abstractmethod - def process(self) -> bool: - """The plugin process. - - :rtype: bool - :returns: True if everything's ok, False optherwise - :raises Exception: Could raise any exception, stacktrace will be sent to autorest API - """ - raise NotImplementedError() - - class PluginAutorest(Plugin, ReaderAndWriterAutorest): """For our Autorest plugins, we want to take autorest api as input as options, then pass it to the Plugin""" @@ -102,39 +44,6 @@ def get_options(self) -> Dict[str, Any]: """Get the options bag using the AutorestAPI that we send to the parent plugin""" -class YamlUpdatePlugin(Plugin): - """A plugin that update the YAML as input.""" - - def get_yaml(self) -> Dict[str, Any]: - # cadl file doesn't have to be relative to output folder - with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd: - return yaml.safe_load(fd.read()) - - def write_yaml(self, yaml_string: str) -> None: - with open(self.options["cadl_file"], "w", encoding="utf-8-sig") as fd: - fd.write(yaml_string) - - def process(self) -> bool: - # List the input file, should be only one - yaml_data = self.get_yaml() - - self.update_yaml(yaml_data) - - yaml_string = yaml.safe_dump(yaml_data) - - self.write_yaml(yaml_string) - return True - - @abstractmethod - def update_yaml(self, yaml_data: Dict[str, Any]) -> None: - """The code-model-v4-no-tags yaml model tree. - - :rtype: updated yaml - :raises Exception: Could raise any exception, stacktrace will be sent to autorest API - """ - raise NotImplementedError() - - class YamlUpdatePluginAutorest(YamlUpdatePlugin, PluginAutorest): # pylint: disable=abstract-method def get_yaml(self) -> Dict[str, Any]: return yaml.safe_load(self.read_file("code-model-v4-no-tags.yaml")) diff --git a/packages/autorest.python/autorest/black.py b/packages/autorest.python/autorest/black.py new file mode 100644 index 00000000000..c77cfcf0513 --- /dev/null +++ b/packages/autorest.python/autorest/black.py @@ -0,0 +1,13 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Dict, Any +from pygen.black import BlackScriptPlugin +from . import PluginAutorest + + +class BlackScriptPluginAutorest(BlackScriptPlugin, PluginAutorest): + def get_options(self) -> Dict[str, Any]: + return {"output_folder": self._autorestapi.get_value("outputFolderUri")} diff --git a/packages/autorest.python/autorest/codegen.py b/packages/autorest.python/autorest/codegen.py new file mode 100644 index 00000000000..3240f7e7d49 --- /dev/null +++ b/packages/autorest.python/autorest/codegen.py @@ -0,0 +1,117 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from typing import Dict, Any, Union +from pathlib import Path +import yaml + +from pygen.codegen import CodeGenerator +from pygen.codegen.models import CodeModel +from pygen.codegen.serializers import JinjaSerializer + +from . import ReaderAndWriterAutorest +from .jsonrpc import AutorestAPI +from . import PluginAutorest + + +_LOGGER = logging.getLogger(__name__) + + +class JinjaSerializerAutorest(JinjaSerializer, ReaderAndWriterAutorest): + def __init__( + self, + autorestapi: AutorestAPI, + code_model: CodeModel, + *, + output_folder: Union[str, Path], + **kwargs: Any, + ) -> None: + super().__init__( # type: ignore + autorestapi=autorestapi, + code_model=code_model, + output_folder=output_folder, + **kwargs, + ) + + +class CodeGeneratorAutorest(CodeGenerator, PluginAutorest): + def get_options(self) -> Dict[str, Any]: + if self._autorestapi.get_boolean_value("python3-only") is False: + _LOGGER.warning("You have passed in --python3-only=False. We have force overriden this to True.") + if self._autorestapi.get_boolean_value("add-python3-operation-files"): + _LOGGER.warning( + "You have passed in --add-python3-operation-files. " + "This flag no longer has an effect bc all SDKs are now Python3 only." + ) + if self._autorestapi.get_boolean_value("reformat-next-link"): + _LOGGER.warning( + "You have passed in --reformat-next-link. We have force overriden " + "this to False because we no longer reformat initial query parameters into next " + "calls unless explicitly defined in the service definition." + ) + options = { + "azure-arm": self._autorestapi.get_boolean_value("azure-arm"), + "header-text": self._autorestapi.get_value("header-text"), + "low-level-client": self._autorestapi.get_boolean_value("low-level-client", False), + "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant", True), + "show-operations": self._autorestapi.get_boolean_value("show-operations"), + "python3-only": self._autorestapi.get_boolean_value("python3-only"), + "head-as-boolean": self._autorestapi.get_boolean_value("head-as-boolean", False), + "keep-version-file": self._autorestapi.get_boolean_value("keep-version-file"), + "no-async": self._autorestapi.get_boolean_value("no-async"), + "no-namespace-folders": self._autorestapi.get_boolean_value("no-namespace-folders"), + "basic-setup-py": self._autorestapi.get_boolean_value("basic-setup-py"), + "package-name": self._autorestapi.get_value("package-name"), + "package-version": self._autorestapi.get_value("package-version"), + "client-side-validation": self._autorestapi.get_boolean_value("client-side-validation"), + "tracing": self._autorestapi.get_boolean_value("trace"), + "multiapi": self._autorestapi.get_boolean_value("multiapi", False), + "polymorphic-examples": self._autorestapi.get_value("polymorphic-examples"), + "models-mode": self._autorestapi.get_value("models-mode"), + "builders-visibility": self._autorestapi.get_value("builders-visibility"), + "show-send-request": self._autorestapi.get_boolean_value("show-send-request"), + "only-path-and-body-params-positional": self._autorestapi.get_boolean_value( + "only-path-and-body-params-positional" + ), + "combine-operation-files": self._autorestapi.get_boolean_value("combine-operation-files"), + "package-mode": self._autorestapi.get_value("package-mode"), + "package-pprint-name": self._autorestapi.get_value("package-pprint-name"), + "packaging-files-config": self._autorestapi.get_value("package-configuration"), + "default-optional-constants-to-none": self._autorestapi.get_boolean_value( + "default-optional-constants-to-none" + ), + "generate-sample": self._autorestapi.get_boolean_value("generate-sample"), + "generate-test": self._autorestapi.get_boolean_value("generate-test"), + "default-api-version": self._autorestapi.get_value("default-api-version"), + } + return {k: v for k, v in options.items() if v is not None} + + def get_yaml(self) -> Dict[str, Any]: + inputs = self._autorestapi.list_inputs() + _LOGGER.debug("Possible Inputs: %s", inputs) + if "code-model-v4-no-tags.yaml" not in inputs: + raise ValueError("code-model-v4-no-tags.yaml must be a possible input") + + if self._autorestapi.get_value("input-yaml"): + input_yaml = self._autorestapi.get_value("input-yaml") + file_content = self._autorestapi.read_file(input_yaml) + else: + inputs = self._autorestapi.list_inputs() + _LOGGER.debug("Possible Inputs: %s", inputs) + if "code-model-v4-no-tags.yaml" not in inputs: + raise ValueError("code-model-v4-no-tags.yaml must be a possible input") + + file_content = self._autorestapi.read_file("code-model-v4-no-tags.yaml") + + # Parse the received YAML + return yaml.safe_load(file_content) + + def get_serializer(self, code_model: CodeModel): + return JinjaSerializerAutorest( + self._autorestapi, + code_model, + output_folder=self.output_folder, + ) diff --git a/packages/autorest.python/autorest/m2r.py b/packages/autorest.python/autorest/m2r.py new file mode 100644 index 00000000000..418a9ca2899 --- /dev/null +++ b/packages/autorest.python/autorest/m2r.py @@ -0,0 +1,16 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""An autorest MD to RST plugin. +""" +from typing import Any, Dict + +from pygen.m2r import M2R +from . import YamlUpdatePluginAutorest + + +class M2RAutorest(YamlUpdatePluginAutorest, M2R): + def get_options(self) -> Dict[str, Any]: + return {} diff --git a/packages/autorest.python/autorest/m4reformatter/__init__.py b/packages/autorest.python/autorest/m4reformatter/__init__.py index 9ae4a5cd4d8..1d0423058ca 100644 --- a/packages/autorest.python/autorest/m4reformatter/__init__.py +++ b/packages/autorest.python/autorest/m4reformatter/__init__.py @@ -11,7 +11,7 @@ import logging from typing import Callable, Dict, Any, Iterable, List, Optional, Set -from .._utils import ( +from pygen.utils import ( to_snake_case, KNOWN_TYPES, get_body_type_for_description, diff --git a/packages/autorest.python/autorest/multiapi/__init__.py b/packages/autorest.python/autorest/multiapi/__init__.py index 4b321cc8d97..c1df35c21cf 100644 --- a/packages/autorest.python/autorest/multiapi/__init__.py +++ b/packages/autorest.python/autorest/multiapi/__init__.py @@ -10,12 +10,13 @@ from collections import defaultdict from pathlib import Path from typing import Dict, List, Optional, cast, Any +from pygen import Plugin, ReaderAndWriter from .serializers import MultiAPISerializer, MultiAPISerializerAutorest from .models import CodeModel from .utils import _get_default_api_version_from_list -from .. import Plugin, PluginAutorest, ReaderAndWriter, ReaderAndWriterAutorest +from .. import PluginAutorest, ReaderAndWriterAutorest _LOGGER = logging.getLogger(__name__) diff --git a/packages/autorest.python/autorest/multiapi/serializers/__init__.py b/packages/autorest.python/autorest/multiapi/serializers/__init__.py index 99c2be1547d..9f197a7e2e0 100644 --- a/packages/autorest.python/autorest/multiapi/serializers/__init__.py +++ b/packages/autorest.python/autorest/multiapi/serializers/__init__.py @@ -6,13 +6,15 @@ from pathlib import Path from typing import Any, Optional, Union, List from jinja2 import PackageLoader, Environment +from pygen import ReaderAndWriter +from pygen.utils import build_policies from .import_serializer import FileImportSerializer from ...jsonrpc import AutorestAPI from ..models import CodeModel, GlobalParameter -from ... import ReaderAndWriter, ReaderAndWriterAutorest -from ..._utils import build_policies +from ... import ReaderAndWriterAutorest + __all__ = [ "MultiAPISerializer", @@ -123,7 +125,7 @@ def serialize(self, code_model: CodeModel, no_async: Optional[bool]) -> None: if not code_model.client.client_side_validation: codegen_env = Environment( - loader=PackageLoader("autorest.codegen", "templates"), + loader=PackageLoader("pygen.codegen", "templates"), keep_trailing_newline=True, line_statement_prefix="##", line_comment_prefix="###", diff --git a/packages/autorest.python/autorest/multiclient/__init__.py b/packages/autorest.python/autorest/multiclient/__init__.py index 3d397f6b086..f0e0c32bba9 100644 --- a/packages/autorest.python/autorest/multiclient/__init__.py +++ b/packages/autorest.python/autorest/multiclient/__init__.py @@ -7,7 +7,8 @@ from typing import Any, Dict from pathlib import Path from jinja2 import Environment, PackageLoader -from .. import Plugin, PluginAutorest +from pygen import Plugin +from .. import PluginAutorest _LOGGER = logging.getLogger(__name__) diff --git a/packages/autorest.python/autorest/postprocess.py b/packages/autorest.python/autorest/postprocess.py new file mode 100644 index 00000000000..5aa9f951de3 --- /dev/null +++ b/packages/autorest.python/autorest/postprocess.py @@ -0,0 +1,14 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from typing import Any, Dict + +from pygen.postprocess import PostProcessPlugin +from . import PluginAutorest + + +class PostProcessPluginAutorest(PostProcessPlugin, PluginAutorest): + def get_options(self) -> Dict[str, Any]: + return {"outputFolderUri": self._autorestapi.get_value("outputFolderUri")} diff --git a/packages/autorest.python/autorest/preprocess.py b/packages/autorest.python/autorest/preprocess.py new file mode 100644 index 00000000000..ebe66c56e3e --- /dev/null +++ b/packages/autorest.python/autorest/preprocess.py @@ -0,0 +1,20 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +"""The preprocessing autorest plugin. +""" +from typing import Dict, Any +from pygen.preprocess import PreProcessPlugin +from . import YamlUpdatePluginAutorest + + +class PreProcessPluginAutorest(YamlUpdatePluginAutorest, PreProcessPlugin): + def get_options(self) -> Dict[str, Any]: + options = { + "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant"), + "azure-arm": self._autorestapi.get_boolean_value("azure-arm"), + "models-mode": self._autorestapi.get_value("models-mode"), + } + return {k: v for k, v in options.items() if v is not None} diff --git a/packages/autorest.python/dev_requirements.txt b/packages/autorest.python/dev_requirements.txt index d458756b7d8..0ac67bacc8a 100644 --- a/packages/autorest.python/dev_requirements.txt +++ b/packages/autorest.python/dev_requirements.txt @@ -1,4 +1,5 @@ -e . +-e ./generator -r ../../eng/requirements.txt -r ../../eng/dev_requirements.txt ptvsd==4.3.2 diff --git a/packages/autorest.python/mypy.ini b/packages/autorest.python/mypy.ini index 611cc27aa2a..7fbbd82b849 100644 --- a/packages/autorest.python/mypy.ini +++ b/packages/autorest.python/mypy.ini @@ -22,7 +22,7 @@ ignore_missing_imports = True [mypy-autorest.common.python_mappings.*] ignore_missing_imports = True -[mypy-autorest.codegen.models.*] +[mypy-pygen.codegen.models.*] ignore_missing_imports = True [mypy-setuptools] @@ -30,3 +30,6 @@ ignore_missing_imports = True [mypy-*._patch] ignore_missing_imports = True + +[mypy-pygen.*] +ignore_missing_imports = True diff --git a/packages/autorest.python/package.json b/packages/autorest.python/package.json index 055443acf8b..1ddb7fbe15f 100644 --- a/packages/autorest.python/package.json +++ b/packages/autorest.python/package.json @@ -3,10 +3,11 @@ "version": "6.14.3", "description": "The Python extension for generators in AutoRest.", "scripts": { - "prepare": "node run-python3.js prepare.py", - "start": "node run-python3.js start.py", - "install": "node run-python3.js install.py", - "debug": "node run-python3.js start.py --debug" + "start": "node ./scripts/run-python3.js ./scripts/start.py", + "prepare": "node ./scripts/run-python3.js ./scripts/prepare.py", + "build": "node ./scripts/copy-generator.js --force", + "install": "node ./scripts/copy-generator.js && node ./scripts/run-python3.js ./scripts/install.py", + "debug": "node ./scripts/run-python3.js ./scripts/start.py --debug" }, "main": "index.js", "repository": { @@ -25,22 +26,20 @@ }, "homepage": "https://github.com/Azure/autorest.python/blob/main/README.md", "dependencies": { - "@autorest/system-requirements": "~1.0.2" + "@autorest/system-requirements": "~1.0.2", + "fs-extra": "~11.2.0" }, "devDependencies": { "@microsoft.azure/autorest.testserver": "^3.3.46", - "typescript": "~5.1.3" + "typescript": "~5.1.3", + "@azure-tools/typespec-python": "workspace:^" }, "files": [ "autorest/**/*.py", "autorest/**/*.jinja2", + "scripts/", "setup.py", - "install.py", - "prepare.py", - "start.py", - "venvtools.py", - "run-python3.js", "requirements.txt", - "run_cadl.py" + "generator/" ] } diff --git a/packages/autorest.python/requirements.txt b/packages/autorest.python/requirements.txt index 17ae43bada8..3f977ca26b8 100644 --- a/packages/autorest.python/requirements.txt +++ b/packages/autorest.python/requirements.txt @@ -2,7 +2,6 @@ black==24.4.0 click==8.1.3 docutils==0.19 Jinja2==3.1.4 -json-rpc==1.14.0 m2r2==0.3.3 MarkupSafe==2.1.2 mistune==0.8.4 @@ -11,3 +10,4 @@ platformdirs==3.2.0 PyYAML==6.0.1 tomli==2.0.1 setuptools==69.2.0 +json-rpc==1.14.0 diff --git a/packages/autorest.python/run-python3.js b/packages/autorest.python/run-python3.js index 020cbce0a0f..70ec3bc1e6e 100644 --- a/packages/autorest.python/run-python3.js +++ b/packages/autorest.python/run-python3.js @@ -4,7 +4,7 @@ // path resolution algorithm as AutoRest so that the behavior // is fully consistent (and also supports AUTOREST_PYTHON_EXE). // -// Invoke it like so: "node run-python3.js script.py" +// Invoke it like so: "node run-python3.cjs script.py" const cp = require("child_process"); const extension = require("@autorest/system-requirements"); diff --git a/packages/autorest.python/scripts/copy-generator.js b/packages/autorest.python/scripts/copy-generator.js new file mode 100644 index 00000000000..c85af504e10 --- /dev/null +++ b/packages/autorest.python/scripts/copy-generator.js @@ -0,0 +1,19 @@ +const fs = require("fs-extra"); +const path = require("path"); + +const force = process.argv[2] === "--force" ? true : false; + +const typespecModulePath = path.join(__dirname, "..", "node_modules", "@azure-tools", "typespec-python"); + +// Define the source and destination directories +const sourceDir = path.join(typespecModulePath, "generator"); +const destDir = path.join(__dirname, "..", "generator"); + +// Delete the destination directory if it exists +if (fs.existsSync(destDir)) { + if (force) fs.removeSync(destDir); + else process.exit(0); +} + +// Copy the source directory to the destination directory +fs.copySync(sourceDir, destDir); diff --git a/packages/autorest.python/install.py b/packages/autorest.python/scripts/install.py similarity index 85% rename from packages/autorest.python/install.py rename to packages/autorest.python/scripts/install.py index 14ae38dd278..7b687f1171d 100644 --- a/packages/autorest.python/install.py +++ b/packages/autorest.python/scripts/install.py @@ -23,17 +23,15 @@ # Now we have pip and Py >= 3.8, go to work -import subprocess from pathlib import Path from venvtools import ExtendedEnvBuilder, python_run -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent def main(): venv_path = _ROOT_DIR / "venv" - if venv_path.exists(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) @@ -43,8 +41,8 @@ def main(): venv_context = env_builder.context python_run(venv_context, "pip", ["install", "-U", "pip"]) - python_run(venv_context, "pip", ["install", "-r", "requirements.txt"]) - python_run(venv_context, "pip", ["install", "-e", str(_ROOT_DIR)]) + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/requirements.txt"]) + python_run(venv_context, "pip", ["install", "-e", f"{_ROOT_DIR}/generator"]) if __name__ == "__main__": diff --git a/packages/autorest.python/prepare.py b/packages/autorest.python/scripts/prepare.py similarity index 77% rename from packages/autorest.python/prepare.py rename to packages/autorest.python/scripts/prepare.py index 84b592e30ee..eb89f7555b2 100644 --- a/packages/autorest.python/prepare.py +++ b/packages/autorest.python/scripts/prepare.py @@ -6,6 +6,8 @@ # license information. # -------------------------------------------------------------------------- import sys +import os +import argparse if not sys.version_info >= (3, 8, 0): raise Exception("Autorest for Python extension requires Python 3.8 at least") @@ -15,7 +17,7 @@ from venvtools import python_run -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent def main(): @@ -26,9 +28,10 @@ def main(): env_builder = venv.EnvBuilder(with_pip=True) venv_context = env_builder.ensure_directories(venv_path) - requirements_path = _ROOT_DIR / "dev_requirements.txt" - - python_run(venv_context, "pip", ["install", "-r", str(requirements_path)]) + try: + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/dev_requirements.txt"]) + except FileNotFoundError as e: + raise ValueError(e.filename) if __name__ == "__main__": diff --git a/packages/autorest.python/scripts/run-python3.js b/packages/autorest.python/scripts/run-python3.js new file mode 100644 index 00000000000..020cbce0a0f --- /dev/null +++ b/packages/autorest.python/scripts/run-python3.js @@ -0,0 +1,22 @@ +// This script wraps logic in @azure-tools/extension to resolve +// the path to Python 3 so that a Python script file can be run +// from an npm script in package.json. It uses the same Python 3 +// path resolution algorithm as AutoRest so that the behavior +// is fully consistent (and also supports AUTOREST_PYTHON_EXE). +// +// Invoke it like so: "node run-python3.js script.py" + +const cp = require("child_process"); +const extension = require("@autorest/system-requirements"); + +async function runPython3(scriptName, ...args) { + const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE" }); + cp.execSync(command.join(" "), { + stdio: [0, 1, 2] + }); +} + +runPython3(...process.argv.slice(2)).catch(err => { + console.error(err.toString()); + process.exit(1); +}); diff --git a/packages/autorest.python/start.py b/packages/autorest.python/scripts/start.py similarity index 96% rename from packages/autorest.python/start.py rename to packages/autorest.python/scripts/start.py index fdf57593653..714fc50155f 100644 --- a/packages/autorest.python/start.py +++ b/packages/autorest.python/scripts/start.py @@ -15,7 +15,7 @@ from venvtools import python_run -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent def main(): diff --git a/packages/autorest.python/venvtools.py b/packages/autorest.python/scripts/venvtools.py similarity index 98% rename from packages/autorest.python/venvtools.py rename to packages/autorest.python/scripts/venvtools.py index 01e1300b1c8..944ff96e36b 100644 --- a/packages/autorest.python/venvtools.py +++ b/packages/autorest.python/scripts/venvtools.py @@ -11,7 +11,7 @@ from pathlib import Path -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent class ExtendedEnvBuilder(venv.EnvBuilder): diff --git a/packages/autorest.python/test/unittests/requirements.txt b/packages/autorest.python/test/unittests/requirements.txt index e68ca624cb2..64ba07b3d92 100644 --- a/packages/autorest.python/test/unittests/requirements.txt +++ b/packages/autorest.python/test/unittests/requirements.txt @@ -4,3 +4,4 @@ pytest pytest-cov azure-core==1.30.0 -e ../../. +-e ../../node_modules/@azure-tools/typespec-python/generator diff --git a/packages/autorest.python/test/unittests/test_m2r.py b/packages/autorest.python/test/unittests/test_m2r.py index 8ac1287dda3..f61941a3b7e 100644 --- a/packages/autorest.python/test/unittests/test_m2r.py +++ b/packages/autorest.python/test/unittests/test_m2r.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from autorest.m2r import M2R +from pygen.m2r import M2R _MD_LINK = "[inline link](https://github.com/Azure/autorest.python)" diff --git a/packages/autorest.python/test/unittests/test_name_converter.py b/packages/autorest.python/test/unittests/test_name_converter.py index 1e650033e9b..930eb0fc60d 100644 --- a/packages/autorest.python/test/unittests/test_name_converter.py +++ b/packages/autorest.python/test/unittests/test_name_converter.py @@ -3,8 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from autorest.preprocess import PreProcessPlugin -from autorest.preprocess.python_mappings import PadType +from pygen.preprocess import PreProcessPlugin +from pygen.preprocess.python_mappings import PadType def pad_reserved_words(name: str, pad_type: PadType) -> str: diff --git a/packages/autorest.python/test/unittests/test_optional_return_type.py b/packages/autorest.python/test/unittests/test_optional_return_type.py index 1a4cfae46e5..26f413714cf 100644 --- a/packages/autorest.python/test/unittests/test_optional_return_type.py +++ b/packages/autorest.python/test/unittests/test_optional_return_type.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest -from autorest.codegen.models import ( +from pygen.codegen.models import ( Operation, LROOperation, PagingOperation, @@ -15,8 +15,8 @@ RequestBuilder, Client, ) -from autorest.codegen.models.parameter_list import RequestBuilderParameterList -from autorest.codegen.models.primitive_types import StringType +from pygen.codegen.models.parameter_list import RequestBuilderParameterList +from pygen.codegen.models.primitive_types import StringType @pytest.fixture diff --git a/packages/autorest.python/test/unittests/test_parameter_ordering.py b/packages/autorest.python/test/unittests/test_parameter_ordering.py index 8c68fd14863..4f64c991adf 100644 --- a/packages/autorest.python/test/unittests/test_parameter_ordering.py +++ b/packages/autorest.python/test/unittests/test_parameter_ordering.py @@ -3,8 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from autorest.codegen.models import Parameter, AnyType, CodeModel, StringType -from autorest.codegen.models.parameter_list import ParameterList +from pygen.codegen.models import Parameter, AnyType, CodeModel, StringType +from pygen.codegen.models.parameter_list import ParameterList def get_code_model(): diff --git a/packages/autorest.python/test/unittests/test_sort_schema.py b/packages/autorest.python/test/unittests/test_sort_schema.py index 156b162598b..253b83f100e 100644 --- a/packages/autorest.python/test/unittests/test_sort_schema.py +++ b/packages/autorest.python/test/unittests/test_sort_schema.py @@ -4,7 +4,7 @@ # license information. # -------------------------------------------------------------------------- -from autorest.codegen.models import CodeModel, DPGModelType +from pygen.codegen.models import CodeModel, DPGModelType def get_code_model(): diff --git a/packages/typespec-python/dev_requirements.txt b/packages/typespec-python/dev_requirements.txt index 3d4fe7e401b..6dc5274999a 100644 --- a/packages/typespec-python/dev_requirements.txt +++ b/packages/typespec-python/dev_requirements.txt @@ -1,3 +1,4 @@ -r ../../eng/requirements.txt -r ../../eng/dev_requirements.txt +-e generator diff --git a/packages/typespec-python/generator/LICENSE b/packages/typespec-python/generator/LICENSE new file mode 100644 index 00000000000..21071075c24 --- /dev/null +++ b/packages/typespec-python/generator/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/packages/typespec-python/generator/README.md b/packages/typespec-python/generator/README.md new file mode 100644 index 00000000000..3d8a65a8919 --- /dev/null +++ b/packages/typespec-python/generator/README.md @@ -0,0 +1 @@ +# Core Library for Python Generation diff --git a/packages/typespec-python/generator/dev_requirements.txt b/packages/typespec-python/generator/dev_requirements.txt new file mode 100644 index 00000000000..867580b72a4 --- /dev/null +++ b/packages/typespec-python/generator/dev_requirements.txt @@ -0,0 +1,5 @@ +-e ./generator +-r ../../../eng/requirements.txt +-r ../../../eng/dev_requirements.txt +ptvsd==4.3.2 +types-PyYAML==6.0.12.8 diff --git a/packages/typespec-python/generator/pygen/__init__.py b/packages/typespec-python/generator/pygen/__init__.py new file mode 100644 index 00000000000..6050cbec00f --- /dev/null +++ b/packages/typespec-python/generator/pygen/__init__.py @@ -0,0 +1,107 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import logging +from pathlib import Path +import json +from abc import ABC, abstractmethod +from typing import Any, Dict, Union, List + +import yaml + +from ._version import VERSION + + +__version__ = VERSION +_LOGGER = logging.getLogger(__name__) + + +class ReaderAndWriter: + def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None: + self.output_folder = Path(output_folder) + self._list_file: List[str] = [] + try: + with open( + Path(self.output_folder) / Path("..") / Path("python.json"), + "r", + encoding="utf-8-sig", + ) as fd: + python_json = json.load(fd) + except Exception: # pylint: disable=broad-except + python_json = {} + self.options = kwargs + if python_json: + _LOGGER.warning("Loading python.json file. This behavior will be depreacted") + self.options.update(python_json) + + def read_file(self, path: Union[str, Path]) -> str: + """Directly reading from disk""" + # make path relative to output folder + try: + with open(self.output_folder / Path(path), "r", encoding="utf-8-sig") as fd: + return fd.read() + except FileNotFoundError: + return "" + + def write_file(self, filename: Union[str, Path], file_content: str) -> None: + """Directly writing to disk""" + file_folder = Path(filename).parent + if not Path.is_dir(self.output_folder / file_folder): + Path.mkdir(self.output_folder / file_folder, parents=True) + with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd: + fd.write(file_content) + + def list_file(self) -> List[str]: + return [str(f) for f in self.output_folder.glob("**/*") if f.is_file()] + + +class Plugin(ReaderAndWriter, ABC): + """A base class for autorest plugin. + + :param autorestapi: An autorest API instance + """ + + @abstractmethod + def process(self) -> bool: + """The plugin process. + + :rtype: bool + :returns: True if everything's ok, False optherwise + :raises Exception: Could raise any exception, stacktrace will be sent to autorest API + """ + raise NotImplementedError() + + +class YamlUpdatePlugin(Plugin): + """A plugin that update the YAML as input.""" + + def get_yaml(self) -> Dict[str, Any]: + # cadl file doesn't have to be relative to output folder + with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd: + return yaml.safe_load(fd.read()) + + def write_yaml(self, yaml_string: str) -> None: + with open(self.options["cadl_file"], "w", encoding="utf-8-sig") as fd: + fd.write(yaml_string) + + def process(self) -> bool: + # List the input file, should be only one + yaml_data = self.get_yaml() + + self.update_yaml(yaml_data) + + yaml_string = yaml.safe_dump(yaml_data) + + self.write_yaml(yaml_string) + return True + + @abstractmethod + def update_yaml(self, yaml_data: Dict[str, Any]) -> None: + """The code-model-v4-no-tags yaml model tree. + + :rtype: updated yaml + :raises Exception: Could raise any exception, stacktrace will be sent to autorest API + """ + raise NotImplementedError() diff --git a/packages/typespec-python/generator/pygen/_version.py b/packages/typespec-python/generator/pygen/_version.py new file mode 100644 index 00000000000..aadc0f31ff8 --- /dev/null +++ b/packages/typespec-python/generator/pygen/_version.py @@ -0,0 +1,7 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +VERSION = "0.1.0" diff --git a/packages/autorest.python/autorest/black/__init__.py b/packages/typespec-python/generator/pygen/black.py similarity index 89% rename from packages/autorest.python/autorest/black/__init__.py rename to packages/typespec-python/generator/pygen/black.py index 8df1c79241c..b1d48aff821 100644 --- a/packages/autorest.python/autorest/black/__init__.py +++ b/packages/typespec-python/generator/pygen/black.py @@ -10,8 +10,8 @@ import black from black.report import NothingChanged -from .. import Plugin, PluginAutorest -from .._utils import parse_args +from . import Plugin +from .utils import parse_args _LOGGER = logging.getLogger("blib2to3") @@ -65,11 +65,6 @@ def format_file(self, file: Path) -> None: self.write_file(file, file_content) -class BlackScriptPluginAutorest(BlackScriptPlugin, PluginAutorest): - def get_options(self) -> Dict[str, Any]: - return {"output_folder": self._autorestapi.get_value("outputFolderUri")} - - if __name__ == "__main__": # CADL pipeline will call this args, unknown_args = parse_args(need_cadl_file=False) diff --git a/packages/autorest.python/autorest/codegen/__init__.py b/packages/typespec-python/generator/pygen/codegen/__init__.py similarity index 72% rename from packages/autorest.python/autorest/codegen/__init__.py rename to packages/typespec-python/generator/pygen/codegen/__init__.py index ecfb70f5752..ea4716f86e6 100644 --- a/packages/autorest.python/autorest/codegen/__init__.py +++ b/packages/typespec-python/generator/pygen/codegen/__init__.py @@ -9,10 +9,10 @@ import yaml -from .. import Plugin, PluginAutorest -from .._utils import parse_args +from .. import Plugin +from ..utils import parse_args from .models.code_model import CodeModel -from .serializers import JinjaSerializer, JinjaSerializerAutorest +from .serializers import JinjaSerializer from ._utils import DEFAULT_HEADER_TEXT, VALID_PACKAGE_MODE, TYPESPEC_PACKAGE_MODE @@ -324,86 +324,6 @@ def process(self) -> bool: return True -class CodeGeneratorAutorest(CodeGenerator, PluginAutorest): - def get_options(self) -> Dict[str, Any]: - if self._autorestapi.get_boolean_value("python3-only") is False: - _LOGGER.warning("You have passed in --python3-only=False. We have force overriden this to True.") - if self._autorestapi.get_boolean_value("add-python3-operation-files"): - _LOGGER.warning( - "You have passed in --add-python3-operation-files. " - "This flag no longer has an effect bc all SDKs are now Python3 only." - ) - if self._autorestapi.get_boolean_value("reformat-next-link"): - _LOGGER.warning( - "You have passed in --reformat-next-link. We have force overriden " - "this to False because we no longer reformat initial query parameters into next " - "calls unless explicitly defined in the service definition." - ) - options = { - "azure-arm": self._autorestapi.get_boolean_value("azure-arm"), - "header-text": self._autorestapi.get_value("header-text"), - "low-level-client": self._autorestapi.get_boolean_value("low-level-client", False), - "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant", True), - "show-operations": self._autorestapi.get_boolean_value("show-operations"), - "python3-only": self._autorestapi.get_boolean_value("python3-only"), - "head-as-boolean": self._autorestapi.get_boolean_value("head-as-boolean", False), - "keep-version-file": self._autorestapi.get_boolean_value("keep-version-file"), - "no-async": self._autorestapi.get_boolean_value("no-async"), - "no-namespace-folders": self._autorestapi.get_boolean_value("no-namespace-folders"), - "basic-setup-py": self._autorestapi.get_boolean_value("basic-setup-py"), - "package-name": self._autorestapi.get_value("package-name"), - "package-version": self._autorestapi.get_value("package-version"), - "client-side-validation": self._autorestapi.get_boolean_value("client-side-validation"), - "tracing": self._autorestapi.get_boolean_value("trace"), - "multiapi": self._autorestapi.get_boolean_value("multiapi", False), - "polymorphic-examples": self._autorestapi.get_value("polymorphic-examples"), - "models-mode": self._autorestapi.get_value("models-mode"), - "builders-visibility": self._autorestapi.get_value("builders-visibility"), - "show-send-request": self._autorestapi.get_boolean_value("show-send-request"), - "only-path-and-body-params-positional": self._autorestapi.get_boolean_value( - "only-path-and-body-params-positional" - ), - "combine-operation-files": self._autorestapi.get_boolean_value("combine-operation-files"), - "package-mode": self._autorestapi.get_value("package-mode"), - "package-pprint-name": self._autorestapi.get_value("package-pprint-name"), - "packaging-files-config": self._autorestapi.get_value("package-configuration"), - "default-optional-constants-to-none": self._autorestapi.get_boolean_value( - "default-optional-constants-to-none" - ), - "generate-sample": self._autorestapi.get_boolean_value("generate-sample"), - "generate-test": self._autorestapi.get_boolean_value("generate-test"), - "default-api-version": self._autorestapi.get_value("default-api-version"), - } - return {k: v for k, v in options.items() if v is not None} - - def get_yaml(self) -> Dict[str, Any]: - inputs = self._autorestapi.list_inputs() - _LOGGER.debug("Possible Inputs: %s", inputs) - if "code-model-v4-no-tags.yaml" not in inputs: - raise ValueError("code-model-v4-no-tags.yaml must be a possible input") - - if self._autorestapi.get_value("input-yaml"): - input_yaml = self._autorestapi.get_value("input-yaml") - file_content = self._autorestapi.read_file(input_yaml) - else: - inputs = self._autorestapi.list_inputs() - _LOGGER.debug("Possible Inputs: %s", inputs) - if "code-model-v4-no-tags.yaml" not in inputs: - raise ValueError("code-model-v4-no-tags.yaml must be a possible input") - - file_content = self._autorestapi.read_file("code-model-v4-no-tags.yaml") - - # Parse the received YAML - return yaml.safe_load(file_content) - - def get_serializer(self, code_model: CodeModel): - return JinjaSerializerAutorest( - self._autorestapi, - code_model, - output_folder=self.output_folder, - ) - - if __name__ == "__main__": # CADL pipeline will call this parsed_args, unknown_args = parse_args() diff --git a/packages/autorest.python/autorest/codegen/_utils.py b/packages/typespec-python/generator/pygen/codegen/_utils.py similarity index 100% rename from packages/autorest.python/autorest/codegen/_utils.py rename to packages/typespec-python/generator/pygen/codegen/_utils.py diff --git a/packages/autorest.python/autorest/codegen/models/__init__.py b/packages/typespec-python/generator/pygen/codegen/models/__init__.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/__init__.py rename to packages/typespec-python/generator/pygen/codegen/models/__init__.py diff --git a/packages/autorest.python/autorest/codegen/models/base.py b/packages/typespec-python/generator/pygen/codegen/models/base.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/base.py rename to packages/typespec-python/generator/pygen/codegen/models/base.py diff --git a/packages/autorest.python/autorest/codegen/models/base_builder.py b/packages/typespec-python/generator/pygen/codegen/models/base_builder.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/base_builder.py rename to packages/typespec-python/generator/pygen/codegen/models/base_builder.py diff --git a/packages/autorest.python/autorest/codegen/models/client.py b/packages/typespec-python/generator/pygen/codegen/models/client.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/client.py rename to packages/typespec-python/generator/pygen/codegen/models/client.py diff --git a/packages/autorest.python/autorest/codegen/models/code_model.py b/packages/typespec-python/generator/pygen/codegen/models/code_model.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/code_model.py rename to packages/typespec-python/generator/pygen/codegen/models/code_model.py diff --git a/packages/autorest.python/autorest/codegen/models/combined_type.py b/packages/typespec-python/generator/pygen/codegen/models/combined_type.py similarity index 98% rename from packages/autorest.python/autorest/codegen/models/combined_type.py rename to packages/typespec-python/generator/pygen/codegen/models/combined_type.py index ef714c949ad..5eeea95c625 100644 --- a/packages/autorest.python/autorest/codegen/models/combined_type.py +++ b/packages/typespec-python/generator/pygen/codegen/models/combined_type.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- from typing import Any, Dict, List, Optional, TYPE_CHECKING, Type, Tuple, Union import re -from autorest.codegen.models.imports import FileImport, ImportType, TypingSection +from .imports import FileImport, ImportType, TypingSection from .base import BaseType from .model_type import ModelType diff --git a/packages/autorest.python/autorest/codegen/models/constant_type.py b/packages/typespec-python/generator/pygen/codegen/models/constant_type.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/constant_type.py rename to packages/typespec-python/generator/pygen/codegen/models/constant_type.py diff --git a/packages/autorest.python/autorest/codegen/models/credential_types.py b/packages/typespec-python/generator/pygen/codegen/models/credential_types.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/credential_types.py rename to packages/typespec-python/generator/pygen/codegen/models/credential_types.py diff --git a/packages/autorest.python/autorest/codegen/models/dictionary_type.py b/packages/typespec-python/generator/pygen/codegen/models/dictionary_type.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/dictionary_type.py rename to packages/typespec-python/generator/pygen/codegen/models/dictionary_type.py diff --git a/packages/autorest.python/autorest/codegen/models/enum_type.py b/packages/typespec-python/generator/pygen/codegen/models/enum_type.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/enum_type.py rename to packages/typespec-python/generator/pygen/codegen/models/enum_type.py diff --git a/packages/autorest.python/autorest/codegen/models/imports.py b/packages/typespec-python/generator/pygen/codegen/models/imports.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/imports.py rename to packages/typespec-python/generator/pygen/codegen/models/imports.py diff --git a/packages/autorest.python/autorest/codegen/models/list_type.py b/packages/typespec-python/generator/pygen/codegen/models/list_type.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/list_type.py rename to packages/typespec-python/generator/pygen/codegen/models/list_type.py diff --git a/packages/autorest.python/autorest/codegen/models/lro_operation.py b/packages/typespec-python/generator/pygen/codegen/models/lro_operation.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/lro_operation.py rename to packages/typespec-python/generator/pygen/codegen/models/lro_operation.py diff --git a/packages/autorest.python/autorest/codegen/models/lro_paging_operation.py b/packages/typespec-python/generator/pygen/codegen/models/lro_paging_operation.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/lro_paging_operation.py rename to packages/typespec-python/generator/pygen/codegen/models/lro_paging_operation.py diff --git a/packages/autorest.python/autorest/codegen/models/model_type.py b/packages/typespec-python/generator/pygen/codegen/models/model_type.py similarity index 99% rename from packages/autorest.python/autorest/codegen/models/model_type.py rename to packages/typespec-python/generator/pygen/codegen/models/model_type.py index c46c1467d1e..f2a7963e074 100644 --- a/packages/autorest.python/autorest/codegen/models/model_type.py +++ b/packages/typespec-python/generator/pygen/codegen/models/model_type.py @@ -7,7 +7,7 @@ from collections import OrderedDict from typing import Any, Dict, List, Optional, TYPE_CHECKING, cast import sys -from autorest.codegen.models.utils import ( +from .utils import ( add_to_pylint_disable, NAME_LENGTH_LIMIT, ) diff --git a/packages/autorest.python/autorest/codegen/models/operation.py b/packages/typespec-python/generator/pygen/codegen/models/operation.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/operation.py rename to packages/typespec-python/generator/pygen/codegen/models/operation.py diff --git a/packages/autorest.python/autorest/codegen/models/operation_group.py b/packages/typespec-python/generator/pygen/codegen/models/operation_group.py similarity index 99% rename from packages/autorest.python/autorest/codegen/models/operation_group.py rename to packages/typespec-python/generator/pygen/codegen/models/operation_group.py index bf890b6565e..23b6f4daaba 100644 --- a/packages/autorest.python/autorest/codegen/models/operation_group.py +++ b/packages/typespec-python/generator/pygen/codegen/models/operation_group.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- from typing import Dict, List, Any, TYPE_CHECKING -from autorest.codegen.models.utils import OrderedSet +from .utils import OrderedSet from .base import BaseModel from .operation import get_operation diff --git a/packages/autorest.python/autorest/codegen/models/paging_operation.py b/packages/typespec-python/generator/pygen/codegen/models/paging_operation.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/paging_operation.py rename to packages/typespec-python/generator/pygen/codegen/models/paging_operation.py diff --git a/packages/autorest.python/autorest/codegen/models/parameter.py b/packages/typespec-python/generator/pygen/codegen/models/parameter.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/parameter.py rename to packages/typespec-python/generator/pygen/codegen/models/parameter.py diff --git a/packages/autorest.python/autorest/codegen/models/parameter_list.py b/packages/typespec-python/generator/pygen/codegen/models/parameter_list.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/parameter_list.py rename to packages/typespec-python/generator/pygen/codegen/models/parameter_list.py diff --git a/packages/autorest.python/autorest/codegen/models/primitive_types.py b/packages/typespec-python/generator/pygen/codegen/models/primitive_types.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/primitive_types.py rename to packages/typespec-python/generator/pygen/codegen/models/primitive_types.py diff --git a/packages/autorest.python/autorest/codegen/models/property.py b/packages/typespec-python/generator/pygen/codegen/models/property.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/property.py rename to packages/typespec-python/generator/pygen/codegen/models/property.py diff --git a/packages/autorest.python/autorest/codegen/models/request_builder.py b/packages/typespec-python/generator/pygen/codegen/models/request_builder.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/request_builder.py rename to packages/typespec-python/generator/pygen/codegen/models/request_builder.py diff --git a/packages/autorest.python/autorest/codegen/models/request_builder_parameter.py b/packages/typespec-python/generator/pygen/codegen/models/request_builder_parameter.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/request_builder_parameter.py rename to packages/typespec-python/generator/pygen/codegen/models/request_builder_parameter.py diff --git a/packages/autorest.python/autorest/codegen/models/response.py b/packages/typespec-python/generator/pygen/codegen/models/response.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/response.py rename to packages/typespec-python/generator/pygen/codegen/models/response.py diff --git a/packages/autorest.python/autorest/codegen/models/utils.py b/packages/typespec-python/generator/pygen/codegen/models/utils.py similarity index 100% rename from packages/autorest.python/autorest/codegen/models/utils.py rename to packages/typespec-python/generator/pygen/codegen/models/utils.py diff --git a/packages/autorest.python/autorest/codegen/serializers/__init__.py b/packages/typespec-python/generator/pygen/codegen/serializers/__init__.py similarity index 97% rename from packages/autorest.python/autorest/codegen/serializers/__init__.py rename to packages/typespec-python/generator/pygen/codegen/serializers/__init__.py index 377a6b20ad1..ee3d27ff8ad 100644 --- a/packages/autorest.python/autorest/codegen/serializers/__init__.py +++ b/packages/typespec-python/generator/pygen/codegen/serializers/__init__.py @@ -8,8 +8,7 @@ from pathlib import Path from jinja2 import PackageLoader, Environment, FileSystemLoader, StrictUndefined -from ... import ReaderAndWriter, ReaderAndWriterAutorest -from ...jsonrpc import AutorestAPI +from ... import ReaderAndWriter from ..models import ( OperationGroup, RequestBuilder, @@ -29,7 +28,7 @@ from .sample_serializer import SampleSerializer from .test_serializer import TestSerializer, TestGeneralSerializer from .types_serializer import TypesSerializer -from ..._utils import to_snake_case +from ...utils import to_snake_case from .._utils import VALID_PACKAGE_MODE from .utils import ( extract_sample_name, @@ -138,7 +137,7 @@ def _serialize_namespace_level(self, env: Environment, namespace_path: Path, cli def serialize(self) -> None: env = Environment( - loader=PackageLoader("autorest.codegen", "templates"), + loader=PackageLoader("pygen.codegen", "templates"), keep_trailing_newline=True, line_statement_prefix="##", line_comment_prefix="###", @@ -196,7 +195,7 @@ def _serialize_and_write_package_files(self, namespace_path: Path) -> None: root_of_sdk = self._package_root_folder(namespace_path) if self.code_model.options["package_mode"] in VALID_PACKAGE_MODE: env = Environment( - loader=PackageLoader("autorest.codegen", "templates/packaging_templates"), + loader=PackageLoader("pygen.codegen", "templates/packaging_templates"), undefined=StrictUndefined, ) @@ -569,20 +568,3 @@ def _serialize_and_write_test(self, env: Environment, namespace_path: Path): log_error = f"error happens in test generation for operation group {og.class_name}: {e}" _LOGGER.error(log_error) self.code_model.for_test = False - - -class JinjaSerializerAutorest(JinjaSerializer, ReaderAndWriterAutorest): - def __init__( - self, - autorestapi: AutorestAPI, - code_model: CodeModel, - *, - output_folder: Union[str, Path], - **kwargs: Any, - ) -> None: - super().__init__( - autorestapi=autorestapi, - code_model=code_model, - output_folder=output_folder, - **kwargs, - ) diff --git a/packages/autorest.python/autorest/codegen/serializers/base_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/base_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/base_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/base_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/builder_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/builder_serializer.py similarity index 99% rename from packages/autorest.python/autorest/codegen/serializers/builder_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/builder_serializer.py index 3d1196354fb..e65439ea156 100644 --- a/packages/autorest.python/autorest/codegen/serializers/builder_serializer.py +++ b/packages/typespec-python/generator/pygen/codegen/serializers/builder_serializer.py @@ -36,7 +36,7 @@ from .parameter_serializer import ParameterSerializer, PopKwargType from ..models.parameter_list import ParameterType from . import utils -from ..._utils import JSON_REGEXP +from ...utils import JSON_REGEXP T = TypeVar("T") OrderedSet = Dict[T, None] diff --git a/packages/autorest.python/autorest/codegen/serializers/client_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/client_serializer.py similarity index 99% rename from packages/autorest.python/autorest/codegen/serializers/client_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/client_serializer.py index ce8dfd87964..28041370efb 100644 --- a/packages/autorest.python/autorest/codegen/serializers/client_serializer.py +++ b/packages/typespec-python/generator/pygen/codegen/serializers/client_serializer.py @@ -8,7 +8,7 @@ from . import utils from ..models import Client, ParameterMethodLocation from .parameter_serializer import ParameterSerializer, PopKwargType -from ..._utils import build_policies +from ...utils import build_policies class ClientSerializer: diff --git a/packages/autorest.python/autorest/codegen/serializers/enum_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/enum_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/enum_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/enum_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/general_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/general_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/general_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/general_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/import_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/import_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/import_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/import_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/metadata_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/metadata_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/metadata_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/metadata_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/model_init_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/model_init_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/model_init_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/model_init_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/model_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/model_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/model_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/model_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/operation_groups_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/operation_groups_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/operation_groups_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/operation_groups_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/operations_init_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/operations_init_serializer.py similarity index 95% rename from packages/autorest.python/autorest/codegen/serializers/operations_init_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/operations_init_serializer.py index 83966ffa899..02232c527c5 100644 --- a/packages/autorest.python/autorest/codegen/serializers/operations_init_serializer.py +++ b/packages/typespec-python/generator/pygen/codegen/serializers/operations_init_serializer.py @@ -6,7 +6,7 @@ from typing import List from jinja2 import Environment -from autorest.codegen.models.operation_group import OperationGroup +from ..models.operation_group import OperationGroup from ..models import CodeModel, Client diff --git a/packages/autorest.python/autorest/codegen/serializers/parameter_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/parameter_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/parameter_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/parameter_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/patch_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/patch_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/patch_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/patch_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/request_builders_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/request_builders_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/request_builders_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/request_builders_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/sample_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/sample_serializer.py similarity index 98% rename from packages/autorest.python/autorest/codegen/serializers/sample_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/sample_serializer.py index ccf1e9dfe7a..5694804c687 100644 --- a/packages/autorest.python/autorest/codegen/serializers/sample_serializer.py +++ b/packages/typespec-python/generator/pygen/codegen/serializers/sample_serializer.py @@ -8,7 +8,7 @@ from typing import Dict, Any, Union, Tuple from jinja2 import Environment -from autorest.codegen.models.operation import OperationBase +from ..models.operation import OperationBase from .import_serializer import FileImportSerializer from .base_serializer import BaseSerializer from ..models import ( @@ -22,7 +22,7 @@ FileImport, ) from .utils import get_namespace_config, get_namespace_from_package_name -from ..._utils import to_snake_case +from ...utils import to_snake_case _LOGGER = logging.getLogger(__name__) diff --git a/packages/autorest.python/autorest/codegen/serializers/test_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/test_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/test_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/test_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/types_serializer.py b/packages/typespec-python/generator/pygen/codegen/serializers/types_serializer.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/types_serializer.py rename to packages/typespec-python/generator/pygen/codegen/serializers/types_serializer.py diff --git a/packages/autorest.python/autorest/codegen/serializers/utils.py b/packages/typespec-python/generator/pygen/codegen/serializers/utils.py similarity index 100% rename from packages/autorest.python/autorest/codegen/serializers/utils.py rename to packages/typespec-python/generator/pygen/codegen/serializers/utils.py diff --git a/packages/autorest.python/autorest/codegen/templates/client.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/client.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/client.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/client.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/client_container.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/client_container.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/client_container.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/client_container.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/config.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/config.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/config.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/config.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/config_container.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/config_container.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/config_container.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/config_container.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/conftest.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/conftest.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/conftest.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/conftest.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/enum.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/enum.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/enum.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/enum.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/enum_container.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/enum_container.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/enum_container.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/enum_container.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/init.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/init.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/init.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/init.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/keywords.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/keywords.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/keywords.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/keywords.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/lro_operation.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/lro_operation.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/lro_operation.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/lro_operation.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/lro_paging_operation.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/lro_paging_operation.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/lro_paging_operation.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/macros.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/macros.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/macros.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/macros.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/metadata.json.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/metadata.json.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/metadata.json.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/metadata.json.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/model_base.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/model_base.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/model_base.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/model_base.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/model_container.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/model_container.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/model_container.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/model_container.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/model_dpg.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/model_dpg.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/model_dpg.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/model_dpg.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/model_init.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/model_init.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/model_init.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/model_init.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/model_msrest.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/model_msrest.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/model_msrest.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/model_msrest.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/operation.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/operation.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/operation.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/operation.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/operation_group.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/operation_group.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/operation_group.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/operation_group.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/operation_groups_container.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/operation_groups_container.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/operation_groups_container.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/operation_tools.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/operation_tools.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/operation_tools.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/operation_tools.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/operations_folder_init.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/operations_folder_init.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/operations_folder_init.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/CHANGELOG.md.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/LICENSE.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/LICENSE.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/LICENSE.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/MANIFEST.in.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/MANIFEST.in.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/MANIFEST.in.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/README.md.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/README.md.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/README.md.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/dev_requirements.txt.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/packaging_templates/setup.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/packaging_templates/setup.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/packaging_templates/setup.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/paging_operation.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/paging_operation.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/paging_operation.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/paging_operation.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/patch.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/patch.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/patch.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/patch.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/pkgutil_init.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/pkgutil_init.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/pkgutil_init.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/request_builder.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/request_builder.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/request_builder.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/request_builder.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/request_builders.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/request_builders.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/request_builders.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/request_builders.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/rest_init.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/rest_init.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/rest_init.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/rest_init.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/sample.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/sample.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/sample.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/sample.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/serialization.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/serialization.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/serialization.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/serialization.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/test.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/test.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/test.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/test.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/testpreparer.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/testpreparer.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/testpreparer.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/testpreparer.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/types.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/types.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/types.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/types.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/validation.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/validation.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/validation.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/validation.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/vendor.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/vendor.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/vendor.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/vendor.py.jinja2 diff --git a/packages/autorest.python/autorest/codegen/templates/version.py.jinja2 b/packages/typespec-python/generator/pygen/codegen/templates/version.py.jinja2 similarity index 100% rename from packages/autorest.python/autorest/codegen/templates/version.py.jinja2 rename to packages/typespec-python/generator/pygen/codegen/templates/version.py.jinja2 diff --git a/packages/autorest.python/autorest/m2r/__init__.py b/packages/typespec-python/generator/pygen/m2r.py similarity index 85% rename from packages/autorest.python/autorest/m2r/__init__.py rename to packages/typespec-python/generator/pygen/m2r.py index 88ea00c7f16..d709863e7c4 100644 --- a/packages/autorest.python/autorest/m2r/__init__.py +++ b/packages/typespec-python/generator/pygen/m2r.py @@ -3,21 +3,21 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -"""An autorest MD to RST plugin. +"""An MD to RST plugin. """ import logging from typing import Any, Dict, Set, Union import m2r2 -from .. import YamlUpdatePluginAutorest, YamlUpdatePlugin -from .._utils import parse_args +from . import YamlUpdatePlugin +from .utils import parse_args _LOGGER = logging.getLogger(__name__) -class AutorestRender(m2r2.RestRenderer): +class GeneratorRenderer(m2r2.RestRenderer): """Redefine the concept of inline HTML in the renderer, we don't want to define a new format in the description/summary. """ @@ -54,16 +54,11 @@ def _convert_docstring_no_cycles(self, yaml_data: Union[Dict[str, Any], str], no def convert_to_rst(string_to_convert: str) -> str: """Convert that string from MD to RST.""" try: - return m2r2.convert(string_to_convert, renderer=AutorestRender()).strip() + return m2r2.convert(string_to_convert, renderer=GeneratorRenderer()).strip() except Exception: # pylint: disable=broad-except return string_to_convert -class M2RAutorest(YamlUpdatePluginAutorest, M2R): - def get_options(self) -> Dict[str, Any]: - return {} - - if __name__ == "__main__": # CADL pipeline will call this args, unknown_args = parse_args() diff --git a/packages/autorest.python/autorest/postprocess/__init__.py b/packages/typespec-python/generator/pygen/postprocess/__init__.py similarity index 96% rename from packages/autorest.python/autorest/postprocess/__init__.py rename to packages/typespec-python/generator/pygen/postprocess/__init__.py index a0f16378cee..771177261cb 100644 --- a/packages/autorest.python/autorest/postprocess/__init__.py +++ b/packages/typespec-python/generator/pygen/postprocess/__init__.py @@ -12,7 +12,7 @@ from black.report import NothingChanged from .venvtools import ExtendedEnvBuilder, python_run -from .. import Plugin, PluginAutorest +from .. import Plugin _BLACK_MODE = black.Mode() # pyright: ignore [reportPrivateImportUsage] _BLACK_MODE.line_length = 120 @@ -181,8 +181,3 @@ def fix_imports_in_init(self, generated_file_name: str, folder_path: Path, names file_content = file_content.replace("__all__ = [", f"__all__ = [\n{added_objs_all}", 1) formatted_file = format_file(file, file_content) self.write_file(file, formatted_file) - - -class PostProcessPluginAutorest(PostProcessPlugin, PluginAutorest): - def get_options(self) -> Dict[str, Any]: - return {"outputFolderUri": self._autorestapi.get_value("outputFolderUri")} diff --git a/packages/autorest.python/autorest/postprocess/get_all.py b/packages/typespec-python/generator/pygen/postprocess/get_all.py similarity index 100% rename from packages/autorest.python/autorest/postprocess/get_all.py rename to packages/typespec-python/generator/pygen/postprocess/get_all.py diff --git a/packages/autorest.python/autorest/postprocess/venvtools.py b/packages/typespec-python/generator/pygen/postprocess/venvtools.py similarity index 100% rename from packages/autorest.python/autorest/postprocess/venvtools.py rename to packages/typespec-python/generator/pygen/postprocess/venvtools.py diff --git a/packages/autorest.python/autorest/preprocess/__init__.py b/packages/typespec-python/generator/pygen/preprocess/__init__.py similarity index 97% rename from packages/autorest.python/autorest/preprocess/__init__.py rename to packages/typespec-python/generator/pygen/preprocess/__init__.py index a38719ac4d7..7c6002f1b43 100644 --- a/packages/autorest.python/autorest/preprocess/__init__.py +++ b/packages/typespec-python/generator/pygen/preprocess/__init__.py @@ -8,7 +8,7 @@ import copy from typing import Callable, Dict, Any, List, Optional -from .._utils import to_snake_case +from ..utils import to_snake_case from .helpers import ( add_redefined_builtin_info, pad_builtin_namespaces, @@ -16,8 +16,8 @@ ) from .python_mappings import CADL_RESERVED_WORDS, RESERVED_WORDS, PadType -from .. import YamlUpdatePlugin, YamlUpdatePluginAutorest -from .._utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES, update_enum_value +from .. import YamlUpdatePlugin +from ..utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES, update_enum_value def update_overload_section( @@ -497,16 +497,6 @@ def update_yaml(self, yaml_data: Dict[str, Any]) -> None: yaml_data["namespace"] = pad_builtin_namespaces(yaml_data["namespace"]) -class PreProcessPluginAutorest(YamlUpdatePluginAutorest, PreProcessPlugin): - def get_options(self) -> Dict[str, Any]: - options = { - "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant"), - "azure-arm": self._autorestapi.get_boolean_value("azure-arm"), - "models-mode": self._autorestapi.get_value("models-mode"), - } - return {k: v for k, v in options.items() if v is not None} - - if __name__ == "__main__": # CADL pipeline will call this args, unknown_args = parse_args() diff --git a/packages/autorest.python/autorest/preprocess/helpers.py b/packages/typespec-python/generator/pygen/preprocess/helpers.py similarity index 100% rename from packages/autorest.python/autorest/preprocess/helpers.py rename to packages/typespec-python/generator/pygen/preprocess/helpers.py diff --git a/packages/autorest.python/autorest/preprocess/python_mappings.py b/packages/typespec-python/generator/pygen/preprocess/python_mappings.py similarity index 98% rename from packages/autorest.python/autorest/preprocess/python_mappings.py rename to packages/typespec-python/generator/pygen/preprocess/python_mappings.py index d83df4793be..4edebd7120b 100644 --- a/packages/autorest.python/autorest/preprocess/python_mappings.py +++ b/packages/typespec-python/generator/pygen/preprocess/python_mappings.py @@ -112,7 +112,7 @@ class PadType(str, Enum): PadType.METHOD: [*_always_reserved], PadType.PARAMETER: [ "self", - # these are kwargs we've reserved for our autorest generated operations + # these are kwargs we've reserved for our generated operations "content_type", "accept", "cls", diff --git a/packages/autorest.python/autorest/_utils.py b/packages/typespec-python/generator/pygen/utils.py similarity index 100% rename from packages/autorest.python/autorest/_utils.py rename to packages/typespec-python/generator/pygen/utils.py diff --git a/packages/typespec-python/generator/requirements.txt b/packages/typespec-python/generator/requirements.txt new file mode 100644 index 00000000000..bebbbb5a645 --- /dev/null +++ b/packages/typespec-python/generator/requirements.txt @@ -0,0 +1,12 @@ +black==24.4.0 +click==8.1.3 +docutils==0.19 +Jinja2==3.1.4 +m2r2==0.3.3 +MarkupSafe==2.1.2 +mistune==0.8.4 +pathspec==0.11.1 +platformdirs==3.2.0 +PyYAML==6.0.1 +tomli==2.0.1 +setuptools==69.2.0 diff --git a/packages/typespec-python/generator/setup.py b/packages/typespec-python/generator/setup.py new file mode 100644 index 00000000000..b19d4f2692e --- /dev/null +++ b/packages/typespec-python/generator/setup.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + + +import os +import re + +from setuptools import setup, find_packages + + +# Version extraction inspired from 'requests' +with open(os.path.join("pygen", "_version.py"), "r") as fd: + version = re.search(r'^VERSION\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) + +if not version: + raise RuntimeError("Cannot find version information") + +setup( + name="pygen", + version=version, + description="Core Library for Python Generation", + long_description=open("README.md", "r").read(), + long_description_content_type="text/markdown", + license="MIT License", + author="Microsoft Corporation", + author_email="azpysdkhelp@microsoft.com", + url="https://github.com/Azure/autorest.python/packages/core", + classifiers=[ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "License :: OSI Approved :: MIT License", + ], + packages=find_packages( + exclude=[ + "test", + ] + ), + install_requires=[ + "Jinja2 >= 2.11", # I need "include" and auto-context + blank line are not indented by default + "pyyaml", + "m2r2", + "black", + ], +) diff --git a/packages/typespec-python/package.json b/packages/typespec-python/package.json index 4ba8106e223..86750dde6de 100644 --- a/packages/typespec-python/package.json +++ b/packages/typespec-python/package.json @@ -33,13 +33,15 @@ "test": "mocha", "test-official": "c8 mocha --forbid-only", "lint": "eslint . --ext .ts --max-warnings=0", - "lint:fix": "eslint . --fix --ext .ts" + "lint:fix": "eslint . --fix --ext .ts", + "install": "node ./scripts/run-python3.cjs ./scripts/install.py", + "prepare": "node ./scripts/run-python3.cjs ./scripts/prepare.py" }, "files": [ - "lib/*.cadl", "dist/**", "!dist/test/**", - "get-autorest-python-path.cjs" + "generator/**", + "scripts/**" ], "peerDependencies": { "@azure-tools/typespec-azure-core": ">=0.43.0 <1.0.0", @@ -58,9 +60,11 @@ } }, "dependencies": { - "@autorest/python": "workspace:^", "js-yaml": "~4.1.0", - "@typespec/openapi3": "~0.57.0" + "@typespec/openapi3": "~0.57.0", + "@autorest/system-requirements": "~1.0.2", + "fs-extra": "~11.2.0", + "semver": "~7.6.2" }, "devDependencies": { "@azure-tools/typespec-azure-resource-manager": "~0.43.0", diff --git a/packages/typespec-python/scripts/install.py b/packages/typespec-python/scripts/install.py new file mode 100644 index 00000000000..415ac9455d6 --- /dev/null +++ b/packages/typespec-python/scripts/install.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys + +if not sys.version_info >= (3, 8, 0): + raise Exception("Autorest for Python extension requires Python 3.8 at least") + +try: + import pip +except ImportError: + raise Exception("Your Python installation doesn't have pip available") + +try: + import venv +except ImportError: + raise Exception("Your Python installation doesn't have venv available") + + +# Now we have pip and Py >= 3.8, go to work + +from pathlib import Path + +from venvtools import ExtendedEnvBuilder, python_run + +_ROOT_DIR = Path(__file__).parent.parent + + +def main(): + venv_path = _ROOT_DIR / "venv" + if venv_path.exists(): + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + else: + env_builder = ExtendedEnvBuilder(with_pip=True, upgrade_deps=True) + env_builder.create(venv_path) + venv_context = env_builder.context + + python_run(venv_context, "pip", ["install", "-U", "pip"]) + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/generator/requirements.txt"]) + python_run(venv_context, "pip", ["install", "-e", f"{_ROOT_DIR}/generator"]) + + +if __name__ == "__main__": + main() diff --git a/packages/typespec-python/scripts/prepare.py b/packages/typespec-python/scripts/prepare.py new file mode 100644 index 00000000000..f01a5256676 --- /dev/null +++ b/packages/typespec-python/scripts/prepare.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +import sys +import os +import argparse + +if not sys.version_info >= (3, 8, 0): + raise Exception("Autorest for Python extension requires Python 3.8 at least") + +from pathlib import Path +import venv + +from venvtools import python_run + +_ROOT_DIR = Path(__file__).parent.parent + + +def main(): + venv_path = _ROOT_DIR / "venv" + venv_prexists = venv_path.exists() + + assert venv_prexists # Otherwise install was not done + + env_builder = venv.EnvBuilder(with_pip=True) + venv_context = env_builder.ensure_directories(venv_path) + try: + python_run(venv_context, "pip", ["install", "-r", f"{_ROOT_DIR}/generator/dev_requirements.txt"]) + except FileNotFoundError as e: + raise ValueError(e.filename) + + +if __name__ == "__main__": + main() diff --git a/packages/typespec-python/scripts/run-python3.cjs b/packages/typespec-python/scripts/run-python3.cjs new file mode 100644 index 00000000000..6059b4cce09 --- /dev/null +++ b/packages/typespec-python/scripts/run-python3.cjs @@ -0,0 +1,22 @@ +// This script wraps logic in @azure-tools/extension to resolve +// the path to Python 3 so that a Python script file can be run +// from an npm script in package.json. It uses the same Python 3 +// path resolution algorithm as AutoRest so that the behavior +// is fully consistent (and also supports AUTOREST_PYTHON_EXE). +// +// Invoke it like so: "node run-python3.cjs script.py" + +const cp = require("child_process"); +const extension = require("./system-requirements.cjs"); + +async function runPython3(scriptName, ...args) { + const command = await extension.patchPythonPath(["python", scriptName, ...args], { version: ">=3.8", environmentVariable: "AUTOREST_PYTHON_EXE" }); + cp.execSync(command.join(" "), { + stdio: [0, 1, 2] + }); + } + + runPython3(...process.argv.slice(2)).catch(err => { + console.error(err.toString()); + process.exit(1); + }); diff --git a/packages/autorest.python/run_cadl.py b/packages/typespec-python/scripts/run_tsp.py similarity index 77% rename from packages/autorest.python/run_cadl.py rename to packages/typespec-python/scripts/run_tsp.py index acc4404a90d..0cb34ee9277 100644 --- a/packages/autorest.python/run_cadl.py +++ b/packages/typespec-python/scripts/run_tsp.py @@ -9,7 +9,7 @@ from pathlib import Path from venvtools import python_run -_ROOT_DIR = Path(__file__).parent +_ROOT_DIR = Path(__file__).parent.parent _LOGGER = logging.getLogger(__name__) @@ -34,7 +34,7 @@ breakpoint() # pylint: disable=undefined-variable # run m2r - python_run(venv_context, "autorest.m2r.__init__", command=sys.argv[1:]) - python_run(venv_context, "autorest.preprocess.__init__", command=sys.argv[1:]) - python_run(venv_context, "autorest.codegen.__init__", command=sys.argv[1:]) - python_run(venv_context, "autorest.black.__init__", command=sys.argv[1:]) + python_run(venv_context, "generator.pygen.m2r", command=sys.argv[1:]) + python_run(venv_context, "generator.pygen.preprocess.__init__", command=sys.argv[1:]) + python_run(venv_context, "generator.pygen.codegen.__init__", command=sys.argv[1:]) + python_run(venv_context, "generator.pygen.black", command=sys.argv[1:]) diff --git a/packages/typespec-python/scripts/system-requirements.cjs b/packages/typespec-python/scripts/system-requirements.cjs new file mode 100644 index 00000000000..2b7ced2c400 --- /dev/null +++ b/packages/typespec-python/scripts/system-requirements.cjs @@ -0,0 +1,180 @@ +const semver = require("semver"); +const child_process = require("child_process"); + +/* + * Copied from @autorest/system-requirements + */ + +const PythonRequirement = "python"; +const PRINT_PYTHON_VERSION_SCRIPT = "import sys; print('.'.join(map(str, sys.version_info[:3])))"; + + +function execute(command, cmdlineargs, options = {}) { + return new Promise((resolve, reject) => { + const cp = child_process.spawn(command, cmdlineargs, { ...options, stdio: "pipe", shell: true }); + if (options.onCreate) { + options.onCreate(cp); + } + + options.onStdOutData ? cp.stdout.on("data", options.onStdOutData) : cp; + options.onStdErrData ? cp.stderr.on("data", options.onStdErrData) : cp; + + let err = ""; + let out = ""; + let all = ""; + cp.stderr.on("data", (chunk) => { + err += chunk; + all += chunk; + }); + cp.stdout.on("data", (chunk) => { + out += chunk; + all += chunk; + }); + + cp.on("error", (err) => { + reject(err); + }); + cp.on("close", (code, signal) => + resolve({ + stdout: out, + stderr: err, + log: all, + error: code ? new Error("Process Failed.") : null, + code, + }), + ); + }); +}; + +function versionIsSatisfied(version, requirement) { + const cleanedVersion = semver.coerce(version); + if (!cleanedVersion) { + throw new Error(`Invalid version ${version}.`); + } + return semver.satisfies(cleanedVersion, requirement, true); +}; + +/** + * Validate the provided system requirement resolution is satisfying the version requirement if applicable. + * @param resolution Command resolution. + * @param actualVersion Version for that resolution. + * @param requirement Requirement. + * @returns the resolution if it is valid or an @see SystemRequirementError if not. + */ +function validateVersionRequirement(resolution, actualVersion, requirement) { + if (!requirement.version) { + return resolution; // No version requirement. + } + + try { + if (versionIsSatisfied(actualVersion, requirement.version)) { + return resolution; + } + return { + ...resolution, + error: true, + message: `'${resolution.command}' version is '${actualVersion}' but doesn't satisfy requirement '${requirement.version}'. Please update.`, + actualVersion: actualVersion, + neededVersion: requirement.version, + }; + } catch { + return { + ...resolution, + error: true, + message: `Couldn't parse the version ${actualVersion}. This is not a valid semver version.`, + actualVersion: actualVersion, + neededVersion: requirement.version, + }; + } +}; + +async function tryPython(requirement, command, additionalArgs = []) { + const resolution = { + name: PythonRequirement, + command, + additionalArgs: additionalArgs.length > 0 ? additionalArgs : undefined, + }; + + try { + const result = await execute(command, [...additionalArgs, "-c", `"${PRINT_PYTHON_VERSION_SCRIPT}"`]); + return validateVersionRequirement(resolution, result.stdout.trim(), requirement); + } catch (e) { + return { + error: true, + ...resolution, + message: `'${command}' command line is not found in the path. Make sure to have it installed.`, + }; + } +}; + +/** + * Returns the path to the executable as asked in the requirement. + * @param requirement System requirement definition. + * @returns If the requirement provide an environment variable for the path returns the value of that environment variable. undefined otherwise. + */ +function getExecutablePath(requirement) { + return requirement.environmentVariable && process.env[requirement.environmentVariable]; +} + +function createPythonErrorMessage(requirement, errors) { + const versionReq = requirement.version ?? "*"; + const lines = [ + `Couldn't find a valid python interpreter satisfying the requirement (version: ${versionReq}). Tried:`, + ...errors.map((x) => ` - ${x.command} (${x.message})`), + ]; + + return { + error: true, + name: "python", + command: "python", + message: lines.join("\n"), + }; +}; + +async function resolvePythonRequirement(requirement) { + const path = getExecutablePath(requirement) ?? process.env["AUTOREST_PYTHON_EXE"]; + if (path) { + return await tryPython(requirement, path); + } + + const errors = []; + // On windows try `py` executable with `-3` flag. + if (process.platform === "win32") { + const pyResult = await tryPython(requirement, "py", ["-3"]); + if ("error" in pyResult) { + errors.push(pyResult); + } else { + return pyResult; + } + } + + const python3Result = await tryPython(requirement, "python3"); + if ("error" in python3Result) { + errors.push(python3Result); + } else { + return python3Result; + } + + const pythonResult = await tryPython(requirement, "python"); + if ("error" in pythonResult) { + errors.push(pythonResult); + } else { + return pythonResult; + } + + return createPythonErrorMessage(requirement, errors); +}; + +/** + * @param command list of the command and arguments. First item in array must be a python exe @see KnownPythonExe. (e.g. ["python", "mypythonfile.py"] + * @param requirement + */ +async function patchPythonPath(command, requirement) { + const [_, ...args] = command; + const resolution = await resolvePythonRequirement(requirement); + if ("error" in resolution) { + throw new Error(`Failed to find compatible python version. ${resolution.message}`); + } + return [resolution.command, ...(resolution.additionalArgs ?? []), ...args]; +}; +module.exports.patchPythonPath = patchPythonPath; diff --git a/packages/typespec-python/scripts/venvtools.py b/packages/typespec-python/scripts/venvtools.py new file mode 100644 index 00000000000..944ff96e36b --- /dev/null +++ b/packages/typespec-python/scripts/venvtools.py @@ -0,0 +1,81 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from contextlib import contextmanager +import tempfile +import subprocess +import venv +import sys +from pathlib import Path + + +_ROOT_DIR = Path(__file__).parent.parent + + +class ExtendedEnvBuilder(venv.EnvBuilder): + """An extended env builder which saves the context, to have access + easily to bin path and such. + """ + + def __init__(self, *args, **kwargs): + self.context = None + if sys.version_info < (3, 9, 0): + # Not supported on Python 3.8, and we don't need it + kwargs.pop("upgrade_deps", None) + super().__init__(*args, **kwargs) + + def ensure_directories(self, env_dir): + self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir) + return self.context + + +def create( + env_dir, system_site_packages=False, clear=False, symlinks=False, with_pip=False, prompt=None, upgrade_deps=False +): + """Create a virtual environment in a directory.""" + builder = ExtendedEnvBuilder( + system_site_packages=system_site_packages, + clear=clear, + symlinks=symlinks, + with_pip=with_pip, + prompt=prompt, + upgrade_deps=upgrade_deps, + ) + builder.create(env_dir) + return builder.context + + +@contextmanager +def create_venv_with_package(packages): + """Create a venv with these packages in a temp dir and yielf the env. + + packages should be an iterable of pip version instructio (e.g. package~=1.2.3) + """ + with tempfile.TemporaryDirectory() as tempdir: + myenv = create(tempdir, with_pip=True, upgrade_deps=True) + pip_call = [ + myenv.env_exe, + "-m", + "pip", + "install", + ] + subprocess.check_call(pip_call + ["-U", "pip"]) + if packages: + subprocess.check_call(pip_call + packages) + yield myenv + + +def python_run(venv_context, module, command=None, *, additional_dir="."): + try: + cmd_line = [venv_context.env_exe, "-m", module] + (command if command else []) + print("Executing: {}".format(" ".join(cmd_line))) + subprocess.run( + cmd_line, + cwd=_ROOT_DIR / additional_dir, + check=True, + ) + except subprocess.CalledProcessError as err: + print(err) + sys.exit(1) diff --git a/packages/typespec-python/src/emitter.ts b/packages/typespec-python/src/emitter.ts index cc376200ab9..bbe4b8d2b77 100644 --- a/packages/typespec-python/src/emitter.ts +++ b/packages/typespec-python/src/emitter.ts @@ -5,13 +5,14 @@ import { SdkHttpOperation, SdkServiceOperation, } from "@azure-tools/typespec-client-generator-core"; -import { resolveModuleRoot, saveCodeModelAsYaml } from "./external-process.js"; +import { saveCodeModelAsYaml } from "./external-process.js"; import { dirname } from "path"; import { fileURLToPath } from "url"; import { execFileSync } from "child_process"; import { PythonEmitterOptions, PythonSdkContext } from "./lib.js"; import { emitCodeModel } from "./code-model.js"; import { removeUnderscoresFromNamespace } from "./utils.js"; +import path from "path"; export function getModelsMode(context: SdkContext): "dpg" | "none" { const specifiedModelsMode = context.emitContext.options["models-mode"]; @@ -66,14 +67,14 @@ function createPythonSdkContext( export async function $onEmit(context: EmitContext) { const program = context.program; const sdkContext = createPythonSdkContext(context); - const root = await resolveModuleRoot(program, "@autorest/python", dirname(fileURLToPath(import.meta.url))); + const root = path.join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const outputDir = context.emitterOutputDir; const yamlMap = emitCodeModel(sdkContext); addDefaultOptions(sdkContext); const yamlPath = await saveCodeModelAsYaml("typespec-python-yaml-map", yamlMap); const commandArgs = [ - `${root}/run-python3.js`, - `${root}/run_cadl.py`, + `${root}/scripts/run-python3.cjs`, + `${root}/scripts/run_tsp.py`, `--output-folder=${outputDir}`, `--cadl-file=${yamlPath}`, ]; diff --git a/packages/typespec-python/src/external-process.ts b/packages/typespec-python/src/external-process.ts index 74eb017c1cf..bb2719aed0f 100644 --- a/packages/typespec-python/src/external-process.ts +++ b/packages/typespec-python/src/external-process.ts @@ -1,4 +1,4 @@ -import { CompilerHost, joinPaths, Program, resolveModule, ResolveModuleHost } from "@typespec/compiler"; +import { joinPaths } from "@typespec/compiler"; import { ChildProcess, spawn, SpawnOptions } from "child_process"; import { randomUUID } from "crypto"; import { mkdir, writeFile } from "fs/promises"; @@ -50,31 +50,3 @@ export async function execAsync( }); }); } - -/** - * Resolve root of module. - * @param program Cadl Program - * @param name Name of the module (e.g. "@autorest/python") - * @param baseDir Base directory to start looking from. Use `dirname(fileURLToPath(import.meta.url))` for current dir. - * @returns Path to the module root if found. - */ -export async function resolveModuleRoot(program: Program, name: string, baseDir: string): Promise { - const moduleHost = getResolveModuleHost(program.host); - - const resolved = await resolveModule(moduleHost, name, { - baseDir, - }); - - return resolved.path; -} - -function getResolveModuleHost(host: CompilerHost): ResolveModuleHost { - return { - realpath: host.realpath, - stat: host.stat, - readFile: async (path) => { - const file = await host.readFile(path); - return file.text; - }, - }; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a3f172a8b6f..b57afdba416 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,9 +1,5 @@ lockfileVersion: '6.0' -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - importers: .: @@ -50,7 +46,13 @@ importers: '@autorest/system-requirements': specifier: ~1.0.2 version: 1.0.2 + fs-extra: + specifier: ~11.2.0 + version: 11.2.0 devDependencies: + '@azure-tools/typespec-python': + specifier: workspace:^ + version: link:../typespec-python '@microsoft.azure/autorest.testserver': specifier: ^3.3.46 version: 3.3.46 @@ -60,15 +62,21 @@ importers: packages/typespec-python: dependencies: - '@autorest/python': - specifier: workspace:^ - version: link:../autorest.python + '@autorest/system-requirements': + specifier: ~1.0.2 + version: 1.0.2 '@typespec/openapi3': specifier: ~0.57.0 version: 0.57.0(@typespec/compiler@0.57.0)(@typespec/http@0.57.0)(@typespec/openapi@0.57.0)(@typespec/versioning@0.57.0) + fs-extra: + specifier: ~11.2.0 + version: 11.2.0 js-yaml: specifier: ~4.1.0 version: 4.1.0 + semver: + specifier: ~7.6.2 + version: 7.6.2 devDependencies: '@azure-tools/cadl-ranch-expect': specifier: ~0.14.0 @@ -163,7 +171,7 @@ packages: dependencies: '@azure/logger': 1.0.4 command-exists: 1.2.9 - semver: 7.5.4 + semver: 7.6.2 dev: false /@azure-tools/cadl-ranch-api@0.4.3: @@ -584,6 +592,7 @@ packages: engines: {node: '>=14.0.0'} dependencies: tslib: 2.6.2 + dev: false /@azure/logger@1.1.1: resolution: {integrity: sha512-/+4TtokaGgC+MnThdf6HyIH9Wrjp+CnCn3Nx3ggevN7FFjjNyjqg0yLlc2i9S+Z2uAzI8GYOo35Nzb1MhQ89MA==} @@ -622,7 +631,7 @@ packages: '@azure/core-lro': 2.5.4 '@azure/core-paging': 1.5.0 '@azure/core-tracing': 1.0.0-preview.13 - '@azure/logger': 1.0.4 + '@azure/logger': 1.1.1 events: 3.3.0 tslib: 2.6.2 transitivePeerDependencies: @@ -3472,6 +3481,15 @@ packages: universalify: 2.0.1 dev: true + /fs-extra@11.2.0: + resolution: {integrity: sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==} + engines: {node: '>=14.14'} + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.1 + dev: false + /fs-minipass@2.1.0: resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} engines: {node: '>= 8'} @@ -3662,7 +3680,6 @@ packages: /graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} - dev: true /graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} @@ -4219,7 +4236,6 @@ packages: universalify: 2.0.1 optionalDependencies: graceful-fs: 4.2.11 - dev: true /jsonparse@1.3.1: resolution: {integrity: sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==} @@ -4386,6 +4402,7 @@ packages: engines: {node: '>=10'} dependencies: yallist: 4.0.0 + dev: true /make-dir@4.0.0: resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} @@ -5547,14 +5564,6 @@ packages: lru-cache: 6.0.0 dev: true - /semver@7.5.4: - resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} - engines: {node: '>=10'} - hasBin: true - dependencies: - lru-cache: 6.0.0 - dev: false - /semver@7.6.0: resolution: {integrity: sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==} engines: {node: '>=10'} @@ -6249,7 +6258,6 @@ packages: /universalify@2.0.1: resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} engines: {node: '>= 10.0.0'} - dev: true /unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} @@ -6514,6 +6522,7 @@ packages: /yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: true /yaml@2.4.5: resolution: {integrity: sha512-aBx2bnqDzVOyNKfsysjA2ms5ZlnjSAW2eG3/L5G/CSujfjLJTJsEw1bGw8kCf04KodQWk1pxlGnZ56CRxiawmg==} @@ -6585,3 +6594,7 @@ packages: /zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} dev: true + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false