From ace6611de602906414d612b5649c13b9d8115a1e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 20 Oct 2022 07:33:12 -0700 Subject: [PATCH] gh-98360: multiprocessing now spawns children on Windows with correct argv[0] in virtual environments (GH-98462) (cherry picked from commit e48f9b2b7e73f4a89a9b9c287f3b93dc13a60460) Co-authored-by: Steve Dower --- Lib/multiprocessing/popen_spawn_win32.py | 5 ++- Lib/test/_test_venv_multiprocessing.py | 40 +++++++++++++++++++ Lib/test/test_venv.py | 16 +++++++- ...2-10-19-20-00-28.gh-issue-98360.O2m6YG.rst | 4 ++ 4 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 Lib/test/_test_venv_multiprocessing.py create mode 100644 Misc/NEWS.d/next/Windows/2022-10-19-20-00-28.gh-issue-98360.O2m6YG.rst diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 9c4098d0fa4f1e..4d60ffc030bea6 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -54,19 +54,20 @@ def __init__(self, process_obj): wfd = msvcrt.open_osfhandle(whandle, 0) cmd = spawn.get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle) - cmd = ' '.join('"%s"' % x for x in cmd) python_exe = spawn.get_executable() # bpo-35797: When running in a venv, we bypass the redirect # executor and launch our base Python. if WINENV and _path_eq(python_exe, sys.executable): - python_exe = sys._base_executable + cmd[0] = python_exe = sys._base_executable env = os.environ.copy() env["__PYVENV_LAUNCHER__"] = sys.executable else: env = None + cmd = ' '.join('"%s"' % x for x in cmd) + with open(wfd, 'wb', closefd=True) as to_child: # start process try: diff --git a/Lib/test/_test_venv_multiprocessing.py b/Lib/test/_test_venv_multiprocessing.py new file mode 100644 index 00000000000000..af72e915ba52bb --- /dev/null +++ b/Lib/test/_test_venv_multiprocessing.py @@ -0,0 +1,40 @@ +import multiprocessing +import random +import sys +import time + +def fill_queue(queue, code): + queue.put(code) + + +def drain_queue(queue, code): + if code != queue.get(): + sys.exit(1) + + +def test_func(): + code = random.randrange(0, 1000) + queue = multiprocessing.Queue() + fill_pool = multiprocessing.Process( + target=fill_queue, + args=(queue, code) + ) + drain_pool = multiprocessing.Process( + target=drain_queue, + args=(queue, code) + ) + drain_pool.start() + fill_pool.start() + fill_pool.join() + drain_pool.join() + + +def main(): + test_pool = multiprocessing.Process(target=test_func) + test_pool.start() + test_pool.join() + sys.exit(test_pool.exitcode) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index b0a86ee4abc541..c88df1795fda2e 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -20,7 +20,7 @@ from test.support import (captured_stdout, captured_stderr, requires_zlib, skip_if_broken_multiprocessing_synchronize, verbose, requires_subprocess, is_emscripten, is_wasi, - requires_venv_with_pip) + requires_venv_with_pip, TEST_HOME_DIR) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest import venv @@ -482,6 +482,20 @@ def test_multiprocessing(self): 'pool.terminate()']) self.assertEqual(out.strip(), "python".encode()) + @requireVenvCreate + def test_multiprocessing_recursion(self): + """ + Test that the multiprocessing is able to spawn itself + """ + skip_if_broken_multiprocessing_synchronize() + + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + envpy = os.path.join(os.path.realpath(self.env_dir), + self.bindir, self.exe) + script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py') + subprocess.check_call([envpy, script]) + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') def test_deactivate_with_strict_bash_opts(self): bash = shutil.which("bash") diff --git a/Misc/NEWS.d/next/Windows/2022-10-19-20-00-28.gh-issue-98360.O2m6YG.rst b/Misc/NEWS.d/next/Windows/2022-10-19-20-00-28.gh-issue-98360.O2m6YG.rst new file mode 100644 index 00000000000000..61c1e5e837fe3e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2022-10-19-20-00-28.gh-issue-98360.O2m6YG.rst @@ -0,0 +1,4 @@ +Fixes :mod:`multiprocessing` spawning child processes on Windows from a +virtual environment to ensure that child processes that also use +:mod:`multiprocessing` to spawn more children will recognize that they are +in a virtual environment.