From 7362387473003c0c98bf26b87b877fbe168a6fd2 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 11 Feb 2023 20:54:28 -0800 Subject: [PATCH] gh-89792: Prevent test_tools from copying 1000M of "source" in freeze test (GH-101837) Prevent test_tools from copying 1000M of "source" It doesn't need a git repo, just the checkout. We skip .git metadata, Doc/build, Doc/venv, and `__pycache__` subdirs, that developers often have in their clients to reduce the size of the source tree copy ten-fold. This should significantly reduce IO and presumably time on buildbots during this long test. (cherry picked from commit 1d194235e4d5981b5fea25c75318d61189103a58) Co-authored-by: Gregory P. Smith --- .../2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst | 3 +++ Tools/freeze/test/freeze.py | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst new file mode 100644 index 00000000000000..a3a3070d7f3790 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst @@ -0,0 +1,3 @@ +``test_tools`` now copies up to 10x less source data to a temporary +directory during the ``freeze`` test by ignoring git metadata and other +artifacts. diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index ddbfd7fc9c2f41..0ae983b15c98f0 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -80,7 +80,19 @@ def copy_source_tree(newroot, oldroot): if newroot == SRCDIR: raise Exception('this probably isn\'t what you wanted') shutil.rmtree(newroot) - shutil.copytree(oldroot, newroot) + + def ignore_non_src(src, names): + """Turns what could be a 1000M copy into a 100M copy.""" + # Don't copy the ~600M+ of needless git repo metadata. + # source only, ignore cached .pyc files. + subdirs_to_skip = {'.git', '__pycache__'} + if os.path.basename(src) == 'Doc': + # Another potential ~250M+ of non test related data. + subdirs_to_skip.add('build') + subdirs_to_skip.add('venv') + return subdirs_to_skip + + shutil.copytree(oldroot, newroot, ignore=ignore_non_src) if os.path.exists(os.path.join(newroot, 'Makefile')): _run_quiet([MAKE, 'clean'], newroot)