autorest generation working from scripts in pygen file

packages/autorest.python/start.py

@@ -13,13 +13,13 @@
 from pathlib import Path
 import venv
 
-from pygen.utils.venvtools import python_run
+from venvtools import python_run
 
 _ROOT_DIR = Path(__file__).parent
 
 
 def main():
-    venv_path = _ROOT_DIR / "venv"
+    venv_path = _ROOT_DIR / "node_modules/@azure-tools/python-client-generator-core/venv"
     venv_prexists = venv_path.exists()
 
     assert venv_prexists  # Otherwise install was not done

packages/pygen/install.py

@@ -26,7 +26,7 @@
 import subprocess
 from pathlib import Path
 
-from pygen.utils.venvtools import ExtendedEnvBuilder, python_run
+from venvtools import ExtendedEnvBuilder, python_run
 
 _ROOT_DIR = Path(__file__).parent
 

packages/pygen/prepare.py

@@ -13,7 +13,7 @@
 from pathlib import Path
 import venv
 
-from pygen.utils.venvtools import python_run
+from venvtools import python_run
 
 _ROOT_DIR = Path(__file__).parent
 
@@ -27,8 +27,10 @@ def main():
     env_builder = venv.EnvBuilder(with_pip=True)
     venv_context = env_builder.ensure_directories(venv_path)
     requirements_path = _ROOT_DIR / "dev_requirements.txt"
-
-    python_run(venv_context, "pip", ["install", "-r", str(requirements_path)])
+    try:
+        python_run(venv_context, "pip", ["install", "-r", str(requirements_path)])
+    except FileNotFoundError as e:
+        raise ValueError(e.filename)
 
 
 if __name__ == "__main__":

packages/pygen/pygen/black/__init__.py

@@ -11,7 +11,7 @@
 from black.report import NothingChanged
 
 from .. import Plugin
-from ..utils import parse_args
+from .._utils import parse_args
 
 _LOGGER = logging.getLogger("blib2to3")
 

packages/pygen/pygen/codegen/__init__.py

@@ -10,7 +10,7 @@
 
 
 from .. import Plugin
-from ..utils import parse_args
+from .._utils import parse_args
 from .models.code_model import CodeModel
 from .serializers import JinjaSerializer
 from ._utils import DEFAULT_HEADER_TEXT, VALID_PACKAGE_MODE, TYPESPEC_PACKAGE_MODE

packages/pygen/pygen/codegen/serializers/__init__.py

@@ -28,7 +28,7 @@
 from .sample_serializer import SampleSerializer
 from .test_serializer import TestSerializer, TestGeneralSerializer
 from .types_serializer import TypesSerializer
-from ...utils import to_snake_case
+from ..._utils import to_snake_case
 from .._utils import VALID_PACKAGE_MODE
 from .utils import (
     extract_sample_name,

packages/pygen/pygen/codegen/serializers/builder_serializer.py

@@ -36,7 +36,7 @@
 from .parameter_serializer import ParameterSerializer, PopKwargType
 from ..models.parameter_list import ParameterType
 from . import utils
-from ...utils import JSON_REGEXP
+from ..._utils import JSON_REGEXP
 
 T = TypeVar("T")
 OrderedSet = Dict[T, None]

packages/pygen/pygen/codegen/serializers/client_serializer.py

@@ -8,7 +8,7 @@
 from . import utils
 from ..models import Client, ParameterMethodLocation
 from .parameter_serializer import ParameterSerializer, PopKwargType
-from ...utils import build_policies
+from ..._utils import build_policies
 
 
 class ClientSerializer:

packages/pygen/pygen/codegen/serializers/sample_serializer.py

@@ -22,7 +22,7 @@
     FileImport,
 )
 from .utils import get_namespace_config, get_namespace_from_package_name
-from ...utils import to_snake_case
+from ..._utils import to_snake_case
 
 _LOGGER = logging.getLogger(__name__)
 

packages/pygen/pygen/m2r/__init__.py

@@ -11,7 +11,7 @@
 import m2r2
 
 from .. import YamlUpdatePlugin
-from ..utils import parse_args
+from .._utils import parse_args
 
 
 _LOGGER = logging.getLogger(__name__)

packages/pygen/pygen/preprocess/__init__.py

@@ -8,7 +8,7 @@
 import copy
 from typing import Callable, Dict, Any, List, Optional
 
-from ..utils import to_snake_case
+from .._utils import to_snake_case
 from .helpers import (
     add_redefined_builtin_info,
     pad_builtin_namespaces,
@@ -17,7 +17,7 @@
 from .python_mappings import CADL_RESERVED_WORDS, RESERVED_WORDS, PadType
 
 from .. import YamlUpdatePlugin
-from ..utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES
+from .._utils import parse_args, get_body_type_for_description, JSON_REGEXP, KNOWN_TYPES
 
 
 def update_overload_section(

packages/pygen/run_tsp.py

@@ -7,7 +7,7 @@
 import venv
 import logging
 from pathlib import Path
-from pygen.utils.venvtools import python_run
+from venvtools import python_run
 
 _ROOT_DIR = Path(__file__).parent
 

packages/pygen/venvtools.py

@@ -0,0 +1,81 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from contextlib import contextmanager
+import tempfile
+import subprocess
+import venv
+import sys
+from pathlib import Path
+
+
+_ROOT_DIR = Path(__file__).parent
+
+
+class ExtendedEnvBuilder(venv.EnvBuilder):
+    """An extended env builder which saves the context, to have access
+    easily to bin path and such.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.context = None
+        if sys.version_info < (3, 9, 0):
+            # Not supported on Python 3.8, and we don't need it
+            kwargs.pop("upgrade_deps", None)
+        super().__init__(*args, **kwargs)
+
+    def ensure_directories(self, env_dir):
+        self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir)
+        return self.context
+
+
+def create(
+    env_dir, system_site_packages=False, clear=False, symlinks=False, with_pip=False, prompt=None, upgrade_deps=False
+):
+    """Create a virtual environment in a directory."""
+    builder = ExtendedEnvBuilder(
+        system_site_packages=system_site_packages,
+        clear=clear,
+        symlinks=symlinks,
+        with_pip=with_pip,
+        prompt=prompt,
+        upgrade_deps=upgrade_deps,
+    )
+    builder.create(env_dir)
+    return builder.context
+
+
+@contextmanager
+def create_venv_with_package(packages):
+    """Create a venv with these packages in a temp dir and yielf the env.
+
+    packages should be an iterable of pip version instructio (e.g. package~=1.2.3)
+    """
+    with tempfile.TemporaryDirectory() as tempdir:
+        myenv = create(tempdir, with_pip=True, upgrade_deps=True)
+        pip_call = [
+            myenv.env_exe,
+            "-m",
+            "pip",
+            "install",
+        ]
+        subprocess.check_call(pip_call + ["-U", "pip"])
+        if packages:
+            subprocess.check_call(pip_call + packages)
+        yield myenv
+
+
+def python_run(venv_context, module, command=None, *, additional_dir="."):
+    try:
+        cmd_line = [venv_context.env_exe, "-m", module] + (command if command else [])
+        print("Executing: {}".format(" ".join(cmd_line)))
+        subprocess.run(
+            cmd_line,
+            cwd=_ROOT_DIR / additional_dir,
+            check=True,
+        )
+    except subprocess.CalledProcessError as err:
+        print(err)
+        sys.exit(1)