Since ensurepip wheels are in unpredictable locations, use a long-liv…

…ed bootstrap environment instead (closes #930)

server/pypi/.gitignore

@@ -1,2 +1,2 @@
-
+/build
 /packages/*/build

server/pypi/build-wheel.py

@@ -9,7 +9,7 @@
 import jsonschema
 import multiprocessing
 import os
-from os.path import abspath, basename, dirname, exists, isdir, join
+from os.path import abspath, basename, dirname, exists, isdir, join, splitext
 import pkg_resources
 import re
 import shlex
@@ -23,7 +23,7 @@
 import yaml
 
 
-PROGRAM_NAME = basename(__file__)
+PROGRAM_NAME = splitext(basename(__file__))[0]
 PYPI_DIR = abspath(dirname(__file__))
 RECIPES_DIR = f"{PYPI_DIR}/packages"
 
@@ -117,24 +117,20 @@ def unpack_and_build(self):
         self.build_dir = f"{self.version_dir}/{self.compat_tag}"
         self.src_dir = f"{self.build_dir}/src"
 
-        build_env_dir = f"{self.build_dir}/env"
-        self.pip = f"{build_env_dir}/bin/pip --disable-pip-version-check"
+        build_env = f"{self.build_dir}/env"
+        os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
+        self.pip = f"{build_env}/bin/pip"
 
         if self.no_unpack:
             log("Skipping download and unpack due to --no-unpack")
             assert_isdir(self.build_dir)
         else:
             ensure_empty(self.build_dir)
             if self.python:
-                run(f"python{self.python} {PYPI_DIR}/create-build-env.py {build_env_dir}")
+                self.create_build_env(build_env)
             self.unpack_source()
             self.apply_patches()
 
-        build_reqs = self.get_requirements("build")
-        if build_reqs:
-            run(f"{self.pip} install{' -v' if self.verbose else ''} " +
-                " ".join(f"{name}=={version}" for name, version in build_reqs))
-
         self.reqs_dir = f"{self.build_dir}/requirements"
         if self.no_reqs:
             log("Skipping requirements extraction due to --no-reqs")
@@ -206,6 +202,46 @@ def find_target(self):
             raise CommandError(f"Found {len(zips)} {self.abi} ZIPs in {target_version_dir}")
         self.target_zip = zips[0]
 
+    def create_build_env(self, build_env):
+        # Installing Python's bundled pip and setuptools into the environment is
+        # pointless since we'd immediately have to replace them anyway. Instead, we
+        # create one bootstrap environment per Python version, shared between all
+        # packages, and use that to install the build environments. This saves about 3.5
+        # seconds per build on Python 3.8, and 6 seconds on Python 3.11.
+        bootstrap_env = self.get_bootstrap_env()
+        assert not exists(build_env)
+        run(f"python{self.python} -m venv --without-pip {build_env}")
+
+        build_reqs = {"pip": "19.3", "setuptools": "67.0.0", "wheel": "0.33.6"}
+        build_reqs.update(self.get_requirements("build"))
+        run(f"{bootstrap_env}/bin/pip --python {build_env}/bin/python install " +
+            ("-v " if self.verbose else "") +
+            (" ".join(f"{name}=={version}" for name, version in build_reqs.items())))
+
+    def get_bootstrap_env(self):
+        bootstrap_env = f"{PYPI_DIR}/build/_bootstrap/{self.python}"
+        pip_version = "23.2.1"
+
+        def check_bootstrap_env():
+            if not run(
+                f"{bootstrap_env}/bin/pip --version", capture_output=True
+            ).stdout.startswith(f"pip {pip_version} "):
+                raise CommandError("pip version mismatch")
+
+        if exists(bootstrap_env):
+            try:
+                check_bootstrap_env()
+                return bootstrap_env
+            except CommandError as e:
+                log(e)
+                log("Invalid bootstrap environment: recreating it")
+                ensure_empty(bootstrap_env)
+
+        run(f"python{self.python} -m venv {bootstrap_env}")
+        run(f"{bootstrap_env}/bin/pip install pip=={pip_version}")
+        check_bootstrap_env()
+        return bootstrap_env
+
     def unpack_source(self):
         source = self.meta["source"]
         if not source:
@@ -758,13 +794,14 @@ def run(command, **kwargs):
     log(command)
     kwargs.setdefault("check", True)
     kwargs.setdefault("shell", False)
+    kwargs.setdefault("text", True)
 
     if isinstance(command, str) and not kwargs["shell"]:
         command = shlex.split(command)
     try:
         return subprocess.run(command, **kwargs)
-    except subprocess.CalledProcessError as e:
-        raise CommandError(f"Command returned exit status {e.returncode}")
+    except (OSError, subprocess.CalledProcessError) as e:
+        raise CommandError(e)
 
 
 def ensure_empty(dir_name):

server/pypi/requirements.txt

@@ -1,6 +1,6 @@
 # These are the requirements of build-wheel itself.
 #
-# For requirements of the build environment, see requirements-build-env.txt.
+# Requirements of the build environment are within the script at `create_build_env`.
 
 # Direct requirements
 Jinja2==2.11.3