From 4113bc31a8e6282b1fc8497a397ca6e45bc8b19c Mon Sep 17 00:00:00 2001 From: Long Nguyen Date: Thu, 4 Feb 2021 18:59:30 +0700 Subject: [PATCH 01/13] Add clang mingw support --- distutils/cygwinccompiler.py | 93 ++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 41 deletions(-) diff --git a/distutils/cygwinccompiler.py b/distutils/cygwinccompiler.py index 66c12dd3583..2f76fb2604e 100644 --- a/distutils/cygwinccompiler.py +++ b/distutils/cygwinccompiler.py @@ -44,6 +44,8 @@ # (ld supports -shared) # * mingw gcc 3.2/ld 2.13 works # (ld supports -shared) +# * llvm-mingw with Clang 11 works +# (lld supports -shared) import os import sys @@ -109,41 +111,46 @@ def __init__(self, verbose=0, dry_run=0, force=0): "Compiling may fail because of undefined preprocessor macros." % details) - self.gcc_version, self.ld_version, self.dllwrap_version = \ - get_versions() - self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % - (self.gcc_version, - self.ld_version, - self.dllwrap_version) ) - - # ld_version >= "2.10.90" and < "2.13" should also be able to use - # gcc -mdll instead of dllwrap - # Older dllwraps had own version numbers, newer ones use the - # same as the rest of binutils ( also ld ) - # dllwrap 2.10.90 is buggy - if self.ld_version >= "2.10.90": - self.linker_dll = "gcc" - else: - self.linker_dll = "dllwrap" + self.cc = os.environ.get('CC', 'gcc') + self.cxx = os.environ.get('CXX', 'g++') + + if ('gcc' in self.cc): # Start gcc workaround + self.gcc_version, self.ld_version, self.dllwrap_version = \ + get_gcc_versions() + self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % + (self.gcc_version, + self.ld_version, + self.dllwrap_version) ) + + # ld_version >= "2.10.90" and < "2.13" should also be able to use + # gcc -mdll instead of dllwrap + # Older dllwraps had own version numbers, newer ones use the + # same as the rest of binutils ( also ld ) + # dllwrap 2.10.90 is buggy + if self.ld_version >= "2.10.90": + self.linker_dll = self.cc + else: + self.linker_dll = "dllwrap" - # ld_version >= "2.13" support -shared so use it instead of - # -mdll -static - if self.ld_version >= "2.13": + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + else: # Assume linker is up to date + self.linker_dll = self.cc shared_option = "-shared" - else: - shared_option = "-mdll -static" - # Hard-code GCC because that's what this is all about. - # XXX optimization, warnings etc. should be customizable. - self.set_executables(compiler='gcc -mcygwin -O -Wall', - compiler_so='gcc -mcygwin -mdll -O -Wall', - compiler_cxx='g++ -mcygwin -O -Wall', - linker_exe='gcc -mcygwin', + self.set_executables(compiler='%s -mcygwin -O -Wall' % self.cc, + compiler_so='%s -mcygwin -mdll -O -Wall' % self.cc, + compiler_cxx='%s -mcygwin -O -Wall' % self.cxx, + linker_exe='%s -mcygwin' % self.cc, linker_so=('%s -mcygwin %s' % (self.linker_dll, shared_option))) # cygwin and mingw32 need different sets of libraries - if self.gcc_version == "2.91.57": + if ('gcc' in self.cc and self.gcc_version == "2.91.57"): # cygwin shouldn't need msvcrt, but without the dlls will crash # (gcc version 2.91.57) -- perhaps something about initialization self.dll_libraries=["msvcrt"] @@ -281,26 +288,26 @@ def __init__(self, verbose=0, dry_run=0, force=0): # ld_version >= "2.13" support -shared so use it instead of # -mdll -static - if self.ld_version >= "2.13": - shared_option = "-shared" - else: + if ('gcc' in self.cc and self.ld_version < "2.13"): shared_option = "-mdll -static" + else: + shared_option = "-shared" # A real mingw32 doesn't need to specify a different entry point, # but cygwin 2.91.57 in no-cygwin-mode needs it. - if self.gcc_version <= "2.91.57": + if ('gcc' in self.cc and self.gcc_version <= "2.91.57"): entry_point = '--entry _DllMain@12' else: entry_point = '' - if is_cygwingcc(): + if is_cygwincc(self.cc): raise CCompilerError( 'Cygwin gcc cannot be used with --compiler=mingw32') - self.set_executables(compiler='gcc -O -Wall', - compiler_so='gcc -mdll -O -Wall', - compiler_cxx='g++ -O -Wall', - linker_exe='gcc', + self.set_executables(compiler='%s -O -Wall' % self.cc, + compiler_so='%s -mdll -O -Wall' % self.cc, + compiler_cxx='%s -O -Wall' % self.cxx, + linker_exe='%s' % self.cc, linker_so='%s %s %s' % (self.linker_dll, shared_option, entry_point)) @@ -351,6 +358,10 @@ def check_config_h(): if "GCC" in sys.version: return CONFIG_H_OK, "sys.version mentions 'GCC'" + # Clang would also work + if "Clang" in sys.version: + return CONFIG_H_OK, "sys.version mentions 'Clang'" + # let's see if __GNUC__ is mentioned in python.h fn = sysconfig.get_config_h_filename() try: @@ -389,7 +400,7 @@ def _find_exe_version(cmd): # so we need to decode our bytes return LooseVersion(result.group(1).decode()) -def get_versions(): +def get_gcc_versions(): """ Try to find out the versions of gcc, ld and dllwrap. If not possible it returns None for it. @@ -397,7 +408,7 @@ def get_versions(): commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] return tuple([_find_exe_version(cmd) for cmd in commands]) -def is_cygwingcc(): - '''Try to determine if the gcc that would be used is from cygwin.''' - out_string = check_output(['gcc', '-dumpmachine']) +def is_cygwincc(cc): + '''Try to determine if the compiler that would be used is from cygwin.''' + out_string = check_output([cc, '-dumpmachine']) return out_string.strip().endswith(b'cygwin') From 0d00e6ee898abe2f3954ce08b510175ec5feacf6 Mon Sep 17 00:00:00 2001 From: da-woods Date: Sun, 18 Apr 2021 07:53:22 +0100 Subject: [PATCH 02/13] Fixed get_export_symbols for unicode filenames The implementation of `get_export_symbols` from distutils doesn't quite conform to PEP-489 (because `"_".encode('punycode') != "".encode('punycode')`). This affects the linking of unicode modules on Windows --- distutils/command/build_ext.py | 8 +++++--- distutils/tests/test_build_ext.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/distutils/command/build_ext.py b/distutils/command/build_ext.py index bbb348331b2..f7ab32cfe1d 100644 --- a/distutils/command/build_ext.py +++ b/distutils/command/build_ext.py @@ -690,13 +690,15 @@ def get_export_symbols(self, ext): provided, "PyInit_" + module_name. Only relevant on Windows, where the .pyd file (DLL) must export the module "PyInit_" function. """ - suffix = '_' + ext.name.split('.')[-1] + name = ext.name.split('.')[-1] try: # Unicode module name support as defined in PEP-489 # https://www.python.org/dev/peps/pep-0489/#export-hook-name - suffix.encode('ascii') + name.encode('ascii') except UnicodeEncodeError: - suffix = 'U' + suffix.encode('punycode').replace(b'-', b'_').decode('ascii') + suffix = 'U_' + name.encode('punycode').replace(b'-', b'_').decode('ascii') + else: + suffix = "_" + name initfunc_name = "PyInit" + suffix if initfunc_name not in ext.export_symbols: diff --git a/distutils/tests/test_build_ext.py b/distutils/tests/test_build_ext.py index 5a72458c169..85ecf4b720e 100644 --- a/distutils/tests/test_build_ext.py +++ b/distutils/tests/test_build_ext.py @@ -316,7 +316,7 @@ def test_unicode_module_names(self): self.assertRegex(cmd.get_ext_filename(modules[0].name), r'foo(_d)?\..*') self.assertRegex(cmd.get_ext_filename(modules[1].name), r'föö(_d)?\..*') self.assertEqual(cmd.get_export_symbols(modules[0]), ['PyInit_foo']) - self.assertEqual(cmd.get_export_symbols(modules[1]), ['PyInitU_f_gkaa']) + self.assertEqual(cmd.get_export_symbols(modules[1]), ['PyInitU_f_1gaa']) def test_compiler_option(self): # cmd.compiler is an option and From 044adaee8092b632cf61640e30f126d427aebf5c Mon Sep 17 00:00:00 2001 From: Long Nguyen Date: Sun, 25 Apr 2021 08:08:32 +0700 Subject: [PATCH 03/13] Change get_gcc_versions back to get_versions --- distutils/cygwinccompiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distutils/cygwinccompiler.py b/distutils/cygwinccompiler.py index 2f76fb2604e..f1c38e390cf 100644 --- a/distutils/cygwinccompiler.py +++ b/distutils/cygwinccompiler.py @@ -116,7 +116,7 @@ def __init__(self, verbose=0, dry_run=0, force=0): if ('gcc' in self.cc): # Start gcc workaround self.gcc_version, self.ld_version, self.dllwrap_version = \ - get_gcc_versions() + get_versions() self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % (self.gcc_version, self.ld_version, @@ -400,7 +400,7 @@ def _find_exe_version(cmd): # so we need to decode our bytes return LooseVersion(result.group(1).decode()) -def get_gcc_versions(): +def get_versions(): """ Try to find out the versions of gcc, ld and dllwrap. If not possible it returns None for it. From 221e8715518c23a45b5b81d884d05035e9c896dd Mon Sep 17 00:00:00 2001 From: Alexei Colin Date: Mon, 18 Mar 2019 13:35:20 -0400 Subject: [PATCH 04/13] distutils: pass -rpath to linker on macOS >=10.5 Fix -R option of build_ext for macOS (darwin) Resolves this old bug against distutils that expired due to PIP 632: https://bugs.python.org/issue36353 Applies patch originally submitted to CPython: https://github.com/python/cpython/pull/12418 Contributor: Toon Verstraelen Signed-off-by: Alexei Colin --- distutils/spawn.py | 31 ++-------- distutils/tests/test_unixccompiler.py | 81 ++++++++++++++++++++++++++- distutils/unixccompiler.py | 8 ++- distutils/util.py | 54 ++++++++++++++++++ 4 files changed, 142 insertions(+), 32 deletions(-) diff --git a/distutils/spawn.py b/distutils/spawn.py index b012d00d4ec..6e1c89f1f23 100644 --- a/distutils/spawn.py +++ b/distutils/spawn.py @@ -15,11 +15,6 @@ from distutils import log -if sys.platform == 'darwin': - _cfg_target = None - _cfg_target_split = None - - def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None): """Run another program, specified as a command list 'cmd', in a new process. @@ -52,28 +47,10 @@ def spawn(cmd, search_path=1, verbose=0, dry_run=0, env=None): env = env if env is not None else dict(os.environ) if sys.platform == 'darwin': - global _cfg_target, _cfg_target_split - if _cfg_target is None: - from distutils import sysconfig - _cfg_target = sysconfig.get_config_var( - 'MACOSX_DEPLOYMENT_TARGET') or '' - if _cfg_target: - _cfg_target_split = [int(x) for x in _cfg_target.split('.')] - if _cfg_target: - # Ensure that the deployment target of the build process is not - # less than 10.3 if the interpreter was built for 10.3 or later. - # This ensures extension modules are built with correct - # compatibility values, specifically LDSHARED which can use - # '-undefined dynamic_lookup' which only works on >= 10.3. - cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) - cur_target_split = [int(x) for x in cur_target.split('.')] - if _cfg_target_split[:2] >= [10, 3] and cur_target_split[:2] < [10, 3]: - my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' - 'now "%s" but "%s" during configure;' - 'must use 10.3 or later' - % (cur_target, _cfg_target)) - raise DistutilsPlatformError(my_msg) - env.update(MACOSX_DEPLOYMENT_TARGET=cur_target) + from distutils.util import MACOSX_VERSION_VAR, get_macosx_target_ver + macosx_target_ver = get_macosx_target_ver() + if macosx_target_ver: + env[MACOSX_VERSION_VAR] = macosx_target_ver try: proc = subprocess.Popen(cmd, env=env) diff --git a/distutils/tests/test_unixccompiler.py b/distutils/tests/test_unixccompiler.py index 1828ba1ae0e..ebd7c161e03 100644 --- a/distutils/tests/test_unixccompiler.py +++ b/distutils/tests/test_unixccompiler.py @@ -1,4 +1,5 @@ """Tests for distutils.unixccompiler.""" +import os import sys import unittest from test.support import run_unittest @@ -6,7 +7,9 @@ from .py38compat import EnvironmentVarGuard from distutils import sysconfig +from distutils.errors import DistutilsPlatformError from distutils.unixccompiler import UnixCCompiler +from distutils.util import _clear_cached_macosx_ver class UnixCCompilerTestCase(unittest.TestCase): @@ -26,18 +29,90 @@ def tearDown(self): @unittest.skipIf(sys.platform == 'win32', "can't test on Windows") def test_runtime_libdir_option(self): - # Issue#5900 + # Issue #5900; GitHub Issue #37 # # Ensure RUNPATH is added to extension modules with RPATH if # GNU ld is used # darwin sys.platform = 'darwin' - self.assertEqual(self.cc.rpath_foo(), '-L/foo') + darwin_ver_var = 'MACOSX_DEPLOYMENT_TARGET' + darwin_rpath_flag = '-Wl,-rpath,/foo' + darwin_lib_flag = '-L/foo' + + # (macOS version from syscfg, macOS version from env var) -> flag + # Version value of None generates two tests: as None and as empty string + # Expected flag value of None means an mismatch exception is expected + darwin_test_cases = [ + ((None , None ), darwin_lib_flag), + ((None , '11' ), darwin_rpath_flag), + (('10' , None ), darwin_lib_flag), + (('10.3' , None ), darwin_lib_flag), + (('10.3.1', None ), darwin_lib_flag), + (('10.5' , None ), darwin_rpath_flag), + (('10.5.1', None ), darwin_rpath_flag), + (('10.3' , '10.3' ), darwin_lib_flag), + (('10.3' , '10.5' ), darwin_rpath_flag), + (('10.5' , '10.3' ), darwin_lib_flag), + (('10.5' , '11' ), darwin_rpath_flag), + (('10.4' , '10' ), None), + ] + + def make_darwin_gcv(syscfg_macosx_ver): + def gcv(var): + if var == darwin_ver_var: + return syscfg_macosx_ver + return "xxx" + return gcv + + def do_darwin_test(syscfg_macosx_ver, env_macosx_ver, expected_flag): + env = os.environ + msg = "macOS version = (sysconfig=%r, env=%r)" % \ + (syscfg_macosx_ver, env_macosx_ver) + + # Save + old_gcv = sysconfig.get_config_var + old_env_macosx_ver = env.get(darwin_ver_var) + + # Setup environment + _clear_cached_macosx_ver() + sysconfig.get_config_var = make_darwin_gcv(syscfg_macosx_ver) + if env_macosx_ver is not None: + env[darwin_ver_var] = env_macosx_ver + elif darwin_ver_var in env: + env.pop(darwin_ver_var) + + # Run the test + if expected_flag is not None: + self.assertEqual(self.cc.rpath_foo(), expected_flag, msg=msg) + else: + with self.assertRaisesRegex(DistutilsPlatformError, + darwin_ver_var + r' mismatch', msg=msg): + self.cc.rpath_foo() + + # Restore + if old_env_macosx_ver is not None: + env[darwin_ver_var] = old_env_macosx_ver + elif darwin_ver_var in env: + env.pop(darwin_ver_var) + sysconfig.get_config_var = old_gcv + _clear_cached_macosx_ver() + + for macosx_vers, expected_flag in darwin_test_cases: + syscfg_macosx_ver, env_macosx_ver = macosx_vers + do_darwin_test(syscfg_macosx_ver, env_macosx_ver, expected_flag) + # Bonus test cases with None interpreted as empty string + if syscfg_macosx_ver is None: + do_darwin_test("", env_macosx_ver, expected_flag) + if env_macosx_ver is None: + do_darwin_test(syscfg_macosx_ver, "", expected_flag) + if syscfg_macosx_ver is None and env_macosx_ver is None: + do_darwin_test("", "", expected_flag) + + old_gcv = sysconfig.get_config_var # hp-ux sys.platform = 'hp-ux' - old_gcv = sysconfig.get_config_var def gcv(v): return 'xxx' sysconfig.get_config_var = gcv diff --git a/distutils/unixccompiler.py b/distutils/unixccompiler.py index 4d7a6de740a..f51977a5ae4 100644 --- a/distutils/unixccompiler.py +++ b/distutils/unixccompiler.py @@ -233,8 +233,12 @@ def runtime_library_dir_option(self, dir): # we use this hack. compiler = os.path.basename(sysconfig.get_config_var("CC")) if sys.platform[:6] == "darwin": - # MacOSX's linker doesn't understand the -R flag at all - return "-L" + dir + from distutils.util import get_macosx_target_ver, split_version + macosx_target_ver = get_macosx_target_ver() + if macosx_target_ver and split_version(macosx_target_ver) >= [10, 5]: + return "-Wl,-rpath," + dir + else: # no support for -rpath on earlier macOS versions + return "-L" + dir elif sys.platform[:7] == "freebsd": return "-Wl,-rpath=" + dir elif sys.platform[:5] == "hp-ux": diff --git a/distutils/util.py b/distutils/util.py index f5aca79421b..76657d2edd3 100644 --- a/distutils/util.py +++ b/distutils/util.py @@ -108,6 +108,60 @@ def get_platform(): else: return get_host_platform() + +if sys.platform == 'darwin': + _syscfg_macosx_ver = None # cache the version pulled from sysconfig +MACOSX_VERSION_VAR = 'MACOSX_DEPLOYMENT_TARGET' + +def _clear_cached_macosx_ver(): + """For testing only. Do not call.""" + global _syscfg_macosx_ver + _syscfg_macosx_ver = None + +def get_macosx_target_ver_from_syscfg(): + """Get the version of macOS latched in the Python interpreter configuration. + Returns the version as a string or None if can't obtain one. Cached.""" + global _syscfg_macosx_ver + if _syscfg_macosx_ver is None: + from distutils import sysconfig + ver = sysconfig.get_config_var(MACOSX_VERSION_VAR) or '' + if ver: + _syscfg_macosx_ver = ver + return _syscfg_macosx_ver + +def get_macosx_target_ver(): + """Return the version of macOS for which we are building. + + The target version defaults to the version in sysconfig latched at time + the Python interpreter was built, unless overriden by an environment + variable. If neither source has a value, then None is returned""" + + syscfg_ver = get_macosx_target_ver_from_syscfg() + env_ver = os.environ.get(MACOSX_VERSION_VAR) + + if env_ver: + # Validate overriden version against sysconfig version, if have both. + # Ensure that the deployment target of the build process is not less + # than 10.3 if the interpreter was built for 10.3 or later. This + # ensures extension modules are built with correct compatibility + # values, specifically LDSHARED which can use + # '-undefined dynamic_lookup' which only works on >= 10.3. + if syscfg_ver and split_version(syscfg_ver) >= [10, 3] and \ + split_version(env_ver) < [10, 3]: + my_msg = ('$' + MACOSX_VERSION_VAR + ' mismatch: ' + 'now "%s" but "%s" during configure; ' + 'must use 10.3 or later' + % (env_ver, syscfg_ver)) + raise DistutilsPlatformError(my_msg) + return env_ver + return syscfg_ver + + +def split_version(s): + """Convert a dot-separated string into a list of numbers for comparisons""" + return [int(n) for n in s.split('.')] + + def convert_path (pathname): """Return 'pathname' as a name that will work on the native filesystem, i.e. split it on '/' and put it back together again using the current From a9b8f6c205a6072b46acfd1a7d210083f3497cb8 Mon Sep 17 00:00:00 2001 From: messense Date: Sun, 20 Jun 2021 18:39:57 +0800 Subject: [PATCH 05/13] Prefer using `Distribution.has_ext_modules` method --- distutils/command/build.py | 2 +- distutils/command/install.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distutils/command/build.py b/distutils/command/build.py index a86df0bc7f9..4355a63235c 100644 --- a/distutils/command/build.py +++ b/distutils/command/build.py @@ -102,7 +102,7 @@ def finalize_options(self): # particular module distribution -- if user didn't supply it, pick # one of 'build_purelib' or 'build_platlib'. if self.build_lib is None: - if self.distribution.ext_modules: + if self.distribution.has_ext_modules(): self.build_lib = self.build_platlib else: self.build_lib = self.build_purelib diff --git a/distutils/command/install.py b/distutils/command/install.py index 13feeb890ff..400fb45dd08 100644 --- a/distutils/command/install.py +++ b/distutils/command/install.py @@ -348,7 +348,7 @@ def finalize_options(self): # module distribution is pure or not. Of course, if the user # already specified install_lib, use their selection. if self.install_lib is None: - if self.distribution.ext_modules: # has extensions: non-pure + if self.distribution.has_ext_modules(): # has extensions: non-pure self.install_lib = self.install_platlib else: self.install_lib = self.install_purelib From 2f406eccbe356c79ab1473961e01a80ed09e12dc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 4 Jul 2021 10:07:10 -0400 Subject: [PATCH 06/13] =?UTF-8?q?=F0=9F=91=B9=20Feed=20the=20hobgoblins=20?= =?UTF-8?q?(delint).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- distutils/filelist.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/distutils/filelist.py b/distutils/filelist.py index c92d5fdba39..1d5e4c87443 100644 --- a/distutils/filelist.py +++ b/distutils/filelist.py @@ -4,13 +4,16 @@ and building lists of files. """ -import os, re +import os +import re import fnmatch import functools + from distutils.util import convert_path from distutils.errors import DistutilsTemplateError, DistutilsInternalError from distutils import log + class FileList: """A list of files built by on exploring the filesystem and filtered by applying various patterns to what we find there. @@ -46,7 +49,7 @@ def debug_print(self, msg): if DEBUG: print(msg) - # -- List-like methods --------------------------------------------- + # Collection methods def append(self, item): self.files.append(item) @@ -61,8 +64,7 @@ def sort(self): for sort_tuple in sortable_files: self.files.append(os.path.join(*sort_tuple)) - - # -- Other miscellaneous utility methods --------------------------- + # Other miscellaneous utility methods def remove_duplicates(self): # Assumes list has been sorted! @@ -70,8 +72,7 @@ def remove_duplicates(self): if self.files[i] == self.files[i - 1]: del self.files[i] - - # -- "File template" methods --------------------------------------- + # "File template" methods def _parse_template_line(self, line): words = line.split() @@ -146,9 +147,11 @@ def process_template_line(self, line): (dir, ' '.join(patterns))) for pattern in patterns: if not self.include_pattern(pattern, prefix=dir): - log.warn(("warning: no files found matching '%s' " - "under directory '%s'"), - pattern, dir) + msg = ( + "warning: no files found matching '%s' " + "under directory '%s'" + ) + log.warn(msg, pattern, dir) elif action == 'recursive-exclude': self.debug_print("recursive-exclude %s %s" % @@ -174,8 +177,7 @@ def process_template_line(self, line): raise DistutilsInternalError( "this cannot happen: invalid action '%s'" % action) - - # -- Filtering/selection methods ----------------------------------- + # Filtering/selection methods def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0): """Select strings (presumably filenames) from 'self.files' that @@ -219,9 +221,8 @@ def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0): files_found = True return files_found - - def exclude_pattern (self, pattern, - anchor=1, prefix=None, is_regex=0): + def exclude_pattern( + self, pattern, anchor=1, prefix=None, is_regex=0): """Remove strings (presumably filenames) from 'files' that match 'pattern'. Other parameters are the same as for 'include_pattern()', above. @@ -240,7 +241,6 @@ def exclude_pattern (self, pattern, return files_found -# ---------------------------------------------------------------------- # Utility functions def _find_all_simple(path): @@ -319,7 +319,8 @@ def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0): if os.sep == '\\': sep = r'\\' pattern_re = pattern_re[len(start): len(pattern_re) - len(end)] - pattern_re = r'%s\A%s%s.*%s%s' % (start, prefix_re, sep, pattern_re, end) + pattern_re = r'%s\A%s%s.*%s%s' % ( + start, prefix_re, sep, pattern_re, end) else: # no prefix -- respect anchor flag if anchor: pattern_re = r'%s\A%s' % (start, pattern_re[len(start):]) From a0580a3460030c219c2a017ee3f3804e78e5daae Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 4 Jul 2021 10:07:52 -0400 Subject: [PATCH 07/13] Remove automerge. --- .github/workflows/automerge.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .github/workflows/automerge.yml diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml deleted file mode 100644 index 4f70acfbcbb..00000000000 --- a/.github/workflows/automerge.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: automerge -on: - pull_request: - types: - - labeled - - unlabeled - - synchronize - - opened - - edited - - ready_for_review - - reopened - - unlocked - pull_request_review: - types: - - submitted - check_suite: - types: - - completed - status: {} -jobs: - automerge: - runs-on: ubuntu-latest - steps: - - name: automerge - uses: "pascalgn/automerge-action@v0.12.0" - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From 88f5c729bdb83a8e392959aaef4c781a371f97e3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 4 Jul 2021 10:40:53 -0400 Subject: [PATCH 08/13] Add test capturing failure. Ref pypa/distutils#44. --- distutils/tests/test_filelist.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/distutils/tests/test_filelist.py b/distutils/tests/test_filelist.py index d8e4b39fd23..b56f82e47bc 100644 --- a/distutils/tests/test_filelist.py +++ b/distutils/tests/test_filelist.py @@ -331,6 +331,13 @@ def test_non_local_discovery(self): expected = [file1] self.assertEqual(filelist.findall(temp_dir), expected) + @os_helper.skip_unless_symlink + def test_symlink_loop(self): + with os_helper.temp_dir() as temp_dir: + link = os.path.join(temp_dir, 'link-to-parent') + os.symlink('.', link) + filelist.findall(temp_dir) + def test_suite(): return unittest.TestSuite([ From 43fa2144bfcfb47c15d1d323d7c855c883bc510e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 4 Jul 2021 11:35:08 -0400 Subject: [PATCH 09/13] Ensure that the result contains only the one file, not all the different symlink variants to the same file. --- distutils/tests/test_filelist.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/distutils/tests/test_filelist.py b/distutils/tests/test_filelist.py index b56f82e47bc..9ec507b5d0b 100644 --- a/distutils/tests/test_filelist.py +++ b/distutils/tests/test_filelist.py @@ -335,8 +335,11 @@ def test_non_local_discovery(self): def test_symlink_loop(self): with os_helper.temp_dir() as temp_dir: link = os.path.join(temp_dir, 'link-to-parent') + content = os.path.join(temp_dir, 'somefile') + os_helper.create_empty_file(content) os.symlink('.', link) - filelist.findall(temp_dir) + files = filelist.findall(temp_dir) + assert len(files) == 1 def test_suite(): From 150fed249278621e8d32c3a64b585e81cc9ac257 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 4 Jul 2021 10:38:22 -0400 Subject: [PATCH 10/13] Wrap walk result to prevent infinite recursion. Fixes bpo-44497. --- distutils/filelist.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/distutils/filelist.py b/distutils/filelist.py index 1d5e4c87443..7d7598596f7 100644 --- a/distutils/filelist.py +++ b/distutils/filelist.py @@ -249,12 +249,28 @@ def _find_all_simple(path): """ results = ( os.path.join(base, file) - for base, dirs, files in os.walk(path, followlinks=True) + for base, dirs, files in unique_dirs(os.walk(path, followlinks=True)) for file in files ) return filter(os.path.isfile, results) +def unique_dirs(items): + """ + Given a walk result, remove any previously-seen dirs, + avoiding infinite recursion. + Ref https://bugs.python.org/issue44497. + """ + seen = set() + for base, dirs, files in items: + real = os.path.realpath(base) + if real in seen: + del dirs[:] + continue + seen.add(real) + yield base, dirs, files + + def findall(dir=os.curdir): """ Find all files under 'dir' and return the list of full filenames. From 68f88ce30cfbe3ceb211652a6584bb12907d10a1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 4 Jul 2021 12:12:34 -0400 Subject: [PATCH 11/13] Extract UniqueDirs for checking uniqueness. --- distutils/filelist.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/distutils/filelist.py b/distutils/filelist.py index 7d7598596f7..d7c95deb1fe 100644 --- a/distutils/filelist.py +++ b/distutils/filelist.py @@ -255,19 +255,25 @@ def _find_all_simple(path): return filter(os.path.isfile, results) +class UniqueDirs(set): + def __call__(self, item): + candidate = os.path.realpath(item) + result = candidate in self + self.add(candidate) + return not result + + def unique_dirs(items): """ Given a walk result, remove any previously-seen dirs, avoiding infinite recursion. Ref https://bugs.python.org/issue44497. """ - seen = set() + seen = UniqueDirs() for base, dirs, files in items: - real = os.path.realpath(base) - if real in seen: + if seen(base): del dirs[:] continue - seen.add(real) yield base, dirs, files From f1f635dc80b5ae20b29b2c04e08c50e6dcdcdd3e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 4 Jul 2021 12:18:39 -0400 Subject: [PATCH 12/13] Rely on stat (inode and device) to deduplicate. --- distutils/filelist.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/distutils/filelist.py b/distutils/filelist.py index d7c95deb1fe..10b24bc5758 100644 --- a/distutils/filelist.py +++ b/distutils/filelist.py @@ -249,32 +249,36 @@ def _find_all_simple(path): """ results = ( os.path.join(base, file) - for base, dirs, files in unique_dirs(os.walk(path, followlinks=True)) + for base, dirs, files in _unique_dirs(os.walk(path, followlinks=True)) for file in files ) return filter(os.path.isfile, results) -class UniqueDirs(set): - def __call__(self, item): - candidate = os.path.realpath(item) - result = candidate in self +class _UniqueDirs(set): + def __call__(self, walk_item): + """ + Given an item from an os.walk result, determine + if the item represents a unique dir for this instance + and if not, prevent further traversal. + """ + base, dirs, files = walk_item + stat = os.stat(base) + candidate = stat.st_dev, stat.st_ino + found = candidate in self + if found: + del dirs[:] self.add(candidate) - return not result + return not found -def unique_dirs(items): +def _unique_dirs(items): """ Given a walk result, remove any previously-seen dirs, avoiding infinite recursion. Ref https://bugs.python.org/issue44497. """ - seen = UniqueDirs() - for base, dirs, files in items: - if seen(base): - del dirs[:] - continue - yield base, dirs, files + return filter(_UniqueDirs(), items) def findall(dir=os.curdir): From d616ed7d78de61805ed0bcd57ca20fda313fa3bb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 4 Jul 2021 12:27:40 -0400 Subject: [PATCH 13/13] Move _unique_dirs into classmethod on _UniqueDirs. --- distutils/filelist.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/distutils/filelist.py b/distutils/filelist.py index 10b24bc5758..82a77384dcb 100644 --- a/distutils/filelist.py +++ b/distutils/filelist.py @@ -247,15 +247,21 @@ def _find_all_simple(path): """ Find all files under 'path' """ + all_unique = _UniqueDirs.filter(os.walk(path, followlinks=True)) results = ( os.path.join(base, file) - for base, dirs, files in _unique_dirs(os.walk(path, followlinks=True)) + for base, dirs, files in all_unique for file in files ) return filter(os.path.isfile, results) class _UniqueDirs(set): + """ + Exclude previously-seen dirs from walk results, + avoiding infinite recursion. + Ref https://bugs.python.org/issue44497. + """ def __call__(self, walk_item): """ Given an item from an os.walk result, determine @@ -271,14 +277,9 @@ def __call__(self, walk_item): self.add(candidate) return not found - -def _unique_dirs(items): - """ - Given a walk result, remove any previously-seen dirs, - avoiding infinite recursion. - Ref https://bugs.python.org/issue44497. - """ - return filter(_UniqueDirs(), items) + @classmethod + def filter(cls, items): + return filter(cls(), items) def findall(dir=os.curdir):