From 6b485629d2e3e232460db7da3f8b18b67d4f4da8 Mon Sep 17 00:00:00 2001 From: JasonYZ Date: Sat, 8 Oct 2022 05:46:23 +0100 Subject: [PATCH 01/28] gh-97822: Fix http.server documentation reference to test() function (#98027) Co-authored-by: Jelle Zijlstra --- Doc/library/http.server.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 48f952daae12f5a..81b6bf5373b495c 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -392,8 +392,8 @@ provides three different variants: contents of the file are output. If the file's MIME type starts with ``text/`` the file is opened in text mode; otherwise binary mode is used. - For example usage, see the implementation of the :func:`test` function - invocation in the :mod:`http.server` module. + For example usage, see the implementation of the ``test`` function + in :source:`Lib/http/server.py`. .. versionchanged:: 3.7 Support of the ``'If-Modified-Since'`` header. From 840fd19590935b91f9125bbf1cb93282d6073359 Mon Sep 17 00:00:00 2001 From: partev Date: Sat, 8 Oct 2022 01:55:35 -0400 Subject: [PATCH 02/28] [doc] Fix broken links to C extensions accelerating stdlib modules (#96914) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa Co-authored-by: C.A.M. Gerlach --- Doc/license.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/license.rst b/Doc/license.rst index 00691b30ba6d3ed..54643744dcf9c1b 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -302,7 +302,8 @@ for third-party software incorporated in the Python distribution. Mersenne Twister ---------------- -The :mod:`_random` module includes code based on a download from +The :mod:`!_random` C extension underlying the :mod:`random` module +includes code based on a download from http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html. The following are the verbatim comments from the original code:: @@ -819,7 +820,8 @@ sources unless the build is configured ``--with-system-expat``:: libffi ------ -The :mod:`_ctypes` extension is built using an included copy of the libffi +The :mod:`!_ctypes` C extension underlying the :mod:`ctypes` module +is built using an included copy of the libffi sources unless the build is configured ``--with-system-libffi``:: Copyright (c) 1996-2008 Red Hat, Inc and others. @@ -920,7 +922,8 @@ on the cfuhash project:: libmpdec -------- -The :mod:`_decimal` module is built using an included copy of the libmpdec +The :mod:`!_decimal` C extension underlying the :mod:`decimal` module +is built using an included copy of the libmpdec library unless the build is configured ``--with-system-libmpdec``:: Copyright (c) 2008-2020 Stefan Krah. All rights reserved. From 296313002fde56f52d6c81f17d7ba5c2eb57d098 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Fri, 7 Oct 2022 23:54:16 -0700 Subject: [PATCH 03/28] gh-97913 Docs: Add walrus operator to the index (#97921) * Add walrus operator to the index * Add named expression to the index Co-authored-by: Mariatta Wijaya * Fix indentation and add missing newline Co-authored-by: Ezio Melotti Co-authored-by: Mariatta Wijaya Co-authored-by: Ezio Melotti --- Doc/reference/expressions.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 11f49a8c33dc880..28c17566009fbdc 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1741,6 +1741,12 @@ returns a boolean value regardless of the type of its argument (for example, ``not 'foo'`` produces ``False`` rather than ``''``.) +.. index:: + single: := (colon equals) + single: assignment expression + single: walrus operator + single: named expression + Assignment expressions ====================== From 531ffaa7cdc58c5df2abe505803394dbd5293602 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 8 Oct 2022 09:11:38 +0200 Subject: [PATCH 04/28] Add `@ezio-melotti` as codeowner for `.github/`. (#98079) --- .github/CODEOWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2568560c074f64b..585589d6ce3bf7a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,6 +4,9 @@ # It uses the same pattern rule for gitignore file # https://git-scm.com/docs/gitignore#_pattern_format +# GitHub +.github/** @ezio-melotti + # asyncio **/*asyncio* @1st1 @asvetlov @gvanrossum From c66dbddfbaa374a6954897809574ee9fb463e393 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 8 Oct 2022 11:13:34 +0300 Subject: [PATCH 05/28] GitHub Workflows security hardening (#96492) * Update project-updater.yml Signed-off-by: sashashura <93376818+sashashura@users.noreply.github.com> * Update project-updater.yml repository-projects: write is not needed because a separate secrets.ADD_TO_PROJECT_PAT is used Signed-off-by: sashashura <93376818+sashashura@users.noreply.github.com> --- .github/workflows/project-updater.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml index 77e55ed019b25a3..99c7a05ae8cab07 100644 --- a/.github/workflows/project-updater.yml +++ b/.github/workflows/project-updater.yml @@ -6,6 +6,9 @@ on: - opened - labeled +permissions: + contents: read + jobs: add-to-project: name: Add issues to projects From 83eb827247dd28b13fd816936c74c162e9f52a2d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 8 Oct 2022 07:57:09 -0700 Subject: [PATCH 06/28] gh-97922: Run the GC only on eval breaker (#97920) --- Doc/whatsnew/3.12.rst | 7 +++++ Include/internal/pycore_gc.h | 2 ++ Include/internal/pycore_interp.h | 2 ++ Lib/test/test_frame.py | 2 +- ...2-10-05-11-37-15.gh-issue-97922.Zu9Bge.rst | 5 ++++ Modules/gcmodule.c | 27 +++++++++++++++-- Modules/signalmodule.c | 13 ++++++++ Python/ceval_gil.c | 30 ++++++++++++------- 8 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-37-15.gh-issue-97922.Zu9Bge.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index f873974b3e78fed..341e85103a3cf7b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -93,6 +93,13 @@ Other Language Changes when parsing source code containing null bytes. (Contributed by Pablo Galindo in :gh:`96670`.) +* The Garbage Collector now runs only on the eval breaker mechanism of the + Python bytecode evaluation loop instead on object allocations. The GC can + also run when :c:func:`PyErr_CheckSignals` is called so C extensions that + need to run for a long time without executing any Python code also have a + chance to execute the GC periodically. (Contributed by Pablo Galindo in + :gh:`97922`.) + New Modules =========== diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index bfab0adfffc9ff6..b3abe2030a03dac 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -202,6 +202,8 @@ extern void _PyList_ClearFreeList(PyInterpreterState *interp); extern void _PyDict_ClearFreeList(PyInterpreterState *interp); extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp); extern void _PyContext_ClearFreeList(PyInterpreterState *interp); +extern void _Py_ScheduleGC(PyInterpreterState *interp); +extern void _Py_RunGC(PyThreadState *tstate); #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 8cecd5ab3e541e2..c11e897305d42bb 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -49,6 +49,8 @@ struct _ceval_state { _Py_atomic_int eval_breaker; /* Request for dropping the GIL */ _Py_atomic_int gil_drop_request; + /* The GC is ready to be executed */ + _Py_atomic_int gc_scheduled; struct _pending_calls pending; }; diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 4b86a60d2f4c362..4b5bb7f94ac4692 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -277,7 +277,7 @@ def callback(phase, info): frame! """ nonlocal sneaky_frame_object - sneaky_frame_object = sys._getframe().f_back + sneaky_frame_object = sys._getframe().f_back.f_back # We're done here: gc.callbacks.remove(callback) diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-37-15.gh-issue-97922.Zu9Bge.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-37-15.gh-issue-97922.Zu9Bge.rst new file mode 100644 index 000000000000000..bf78709362f4642 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-37-15.gh-issue-97922.Zu9Bge.rst @@ -0,0 +1,5 @@ +The Garbage Collector now runs only on the eval breaker mechanism of the +Python bytecode evaluation loop instead on object allocations. The GC can +also run when :c:func:`PyErr_CheckSignals` is called so C extensions that +need to run for a long time without executing any Python code also have a +chance to execute the GC periodically. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 97cb6e6e1efb1f1..75832e9dd3da635 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2252,6 +2252,20 @@ PyObject_IS_GC(PyObject *obj) return _PyObject_IS_GC(obj); } +void +_Py_ScheduleGC(PyInterpreterState *interp) +{ + GCState *gcstate = &interp->gc; + if (gcstate->collecting == 1) { + return; + } + struct _ceval_state *ceval = &interp->ceval; + if (!_Py_atomic_load_relaxed(&ceval->gc_scheduled)) { + _Py_atomic_store_relaxed(&ceval->gc_scheduled, 1); + _Py_atomic_store_relaxed(&ceval->eval_breaker, 1); + } +} + void _PyObject_GC_Link(PyObject *op) { @@ -2269,12 +2283,19 @@ _PyObject_GC_Link(PyObject *op) !gcstate->collecting && !_PyErr_Occurred(tstate)) { - gcstate->collecting = 1; - gc_collect_generations(tstate); - gcstate->collecting = 0; + _Py_ScheduleGC(tstate->interp); } } +void +_Py_RunGC(PyThreadState *tstate) +{ + GCState *gcstate = &tstate->interp->gc; + gcstate->collecting = 1; + gc_collect_generations(tstate); + gcstate->collecting = 0; +} + static PyObject * gc_alloc(size_t basicsize, size_t presize) { diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 0f30b4da036313e..b85d6d19e8cd057 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1798,6 +1798,19 @@ int PyErr_CheckSignals(void) { PyThreadState *tstate = _PyThreadState_GET(); + + /* Opportunistically check if the GC is scheduled to run and run it + if we have a request. This is done here because native code needs + to call this API if is going to run for some time without executing + Python code to ensure signals are handled. Checking for the GC here + allows long running native code to clean cycles created using the C-API + even if it doesn't run the evaluation loop */ + struct _ceval_state *interp_ceval_state = &tstate->interp->ceval; + if (_Py_atomic_load_relaxed(&interp_ceval_state->gc_scheduled)) { + _Py_atomic_store_relaxed(&interp_ceval_state->gc_scheduled, 0); + _Py_RunGC(tstate); + } + if (!_Py_ThreadCanHandleSignals(tstate->interp)) { return 0; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index fd737b5738e8896..9b9d7dc1d1af1e7 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -5,6 +5,7 @@ #include "pycore_pyerrors.h" // _PyErr_Fetch() #include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_interp.h" // _Py_RunGC() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() /* @@ -69,7 +70,8 @@ COMPUTE_EVAL_BREAKER(PyInterpreterState *interp, && _Py_ThreadCanHandleSignals(interp)) | (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do) && _Py_ThreadCanHandlePendingCalls()) - | ceval2->pending.async_exc); + | ceval2->pending.async_exc + | _Py_atomic_load_relaxed_int32(&ceval2->gc_scheduled)); } @@ -938,6 +940,7 @@ _Py_HandlePending(PyThreadState *tstate) { _PyRuntimeState * const runtime = &_PyRuntime; struct _ceval_runtime_state *ceval = &runtime->ceval; + struct _ceval_state *interp_ceval_state = &tstate->interp->ceval; /* Pending signals */ if (_Py_atomic_load_relaxed_int32(&ceval->signals_pending)) { @@ -947,20 +950,26 @@ _Py_HandlePending(PyThreadState *tstate) } /* Pending calls */ - struct _ceval_state *ceval2 = &tstate->interp->ceval; - if (_Py_atomic_load_relaxed_int32(&ceval2->pending.calls_to_do)) { + if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->pending.calls_to_do)) { if (make_pending_calls(tstate->interp) != 0) { return -1; } } + /* GC scheduled to run */ + if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gc_scheduled)) { + _Py_atomic_store_relaxed(&interp_ceval_state->gc_scheduled, 0); + COMPUTE_EVAL_BREAKER(tstate->interp, ceval, interp_ceval_state); + _Py_RunGC(tstate); + } + /* GIL drop request */ - if (_Py_atomic_load_relaxed_int32(&ceval2->gil_drop_request)) { + if (_Py_atomic_load_relaxed_int32(&interp_ceval_state->gil_drop_request)) { /* Give another thread a chance */ if (_PyThreadState_Swap(&runtime->gilstate, NULL) != tstate) { Py_FatalError("tstate mix-up"); } - drop_gil(ceval, ceval2, tstate); + drop_gil(ceval, interp_ceval_state, tstate); /* Other threads may run now */ @@ -981,16 +990,17 @@ _Py_HandlePending(PyThreadState *tstate) return -1; } -#ifdef MS_WINDOWS - // bpo-42296: On Windows, _PyEval_SignalReceived() can be called in a - // different thread than the Python thread, in which case + + // It is possible that some of the conditions that trigger the eval breaker + // are called in a different thread than the Python thread. An example of + // this is bpo-42296: On Windows, _PyEval_SignalReceived() can be called in + // a different thread than the Python thread, in which case // _Py_ThreadCanHandleSignals() is wrong. Recompute eval_breaker in the // current Python thread with the correct _Py_ThreadCanHandleSignals() // value. It prevents to interrupt the eval loop at every instruction if // the current Python thread cannot handle signals (if // _Py_ThreadCanHandleSignals() is false). - COMPUTE_EVAL_BREAKER(tstate->interp, ceval, ceval2); -#endif + COMPUTE_EVAL_BREAKER(tstate->interp, ceval, interp_ceval_state); return 0; } From 4ed00be98f5b7dbac3ab71159dda907c931de486 Mon Sep 17 00:00:00 2001 From: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com> Date: Sat, 8 Oct 2022 07:57:47 -0700 Subject: [PATCH 07/28] gh-68686: Retire eptag ptag scripts (#98064) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Retire eptag ptag scripts * 📜🤖 Added by blurb_it. * fix news entry error Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- ...2-10-07-22-06-11.gh-issue-68686.6KNIQ4.rst | 1 + Tools/scripts/README | 1 - Tools/scripts/eptags.py | 57 ------------------- Tools/scripts/ptags.py | 54 ------------------ 4 files changed, 1 insertion(+), 112 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2022-10-07-22-06-11.gh-issue-68686.6KNIQ4.rst delete mode 100755 Tools/scripts/eptags.py delete mode 100755 Tools/scripts/ptags.py diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-10-07-22-06-11.gh-issue-68686.6KNIQ4.rst b/Misc/NEWS.d/next/Tools-Demos/2022-10-07-22-06-11.gh-issue-68686.6KNIQ4.rst new file mode 100644 index 000000000000000..a4289d675703b3f --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2022-10-07-22-06-11.gh-issue-68686.6KNIQ4.rst @@ -0,0 +1 @@ +Remove ptags and eptags scripts. diff --git a/Tools/scripts/README b/Tools/scripts/README index b53b0f21d7c293f..70ea5f4cd0feaef 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -5,7 +5,6 @@ useful while building, extending or managing Python. abitype.py Converts a C file to use the PEP 384 type definition API combinerefs.py A helper for analyzing PYTHONDUMPREFS output diff.py Print file diffs in context, unified, or ndiff formats -eptags.py Create Emacs TAGS file for Python modules gprof2html.py Transform gprof(1) output into useful HTML idle3 Main program to start IDLE md5sum.py Print MD5 checksums of argument files diff --git a/Tools/scripts/eptags.py b/Tools/scripts/eptags.py deleted file mode 100755 index 7f8059ba71adf33..000000000000000 --- a/Tools/scripts/eptags.py +++ /dev/null @@ -1,57 +0,0 @@ -#! /usr/bin/env python3 -"""Create a TAGS file for Python programs, usable with GNU Emacs. - -usage: eptags pyfiles... - -The output TAGS file is usable with Emacs version 18, 19, 20. -Tagged are: - - functions (even inside other defs or classes) - - classes - -eptags warns about files it cannot open. -eptags will not give warnings about duplicate tags. - -BUGS: - Because of tag duplication (methods with the same name in different - classes), TAGS files are not very useful for most object-oriented - python projects. -""" -import sys,re - -expr = r'^[ \t]*(def|class)[ \t]+([a-zA-Z_][a-zA-Z0-9_]*)[ \t]*[:\(]' -matcher = re.compile(expr) - -def treat_file(filename, outfp): - """Append tags found in file named 'filename' to the open file 'outfp'""" - try: - fp = open(filename, 'r') - except OSError: - sys.stderr.write('Cannot open %s\n'%filename) - return - with fp: - charno = 0 - lineno = 0 - tags = [] - size = 0 - while 1: - line = fp.readline() - if not line: - break - lineno = lineno + 1 - m = matcher.search(line) - if m: - tag = m.group(0) + '\177%d,%d\n' % (lineno, charno) - tags.append(tag) - size = size + len(tag) - charno = charno + len(line) - outfp.write('\f\n%s,%d\n' % (filename,size)) - for tag in tags: - outfp.write(tag) - -def main(): - with open('TAGS', 'w') as outfp: - for filename in sys.argv[1:]: - treat_file(filename, outfp) - -if __name__=="__main__": - main() diff --git a/Tools/scripts/ptags.py b/Tools/scripts/ptags.py deleted file mode 100755 index eedd411702c1991..000000000000000 --- a/Tools/scripts/ptags.py +++ /dev/null @@ -1,54 +0,0 @@ -#! /usr/bin/env python3 - -# ptags -# -# Create a tags file for Python programs, usable with vi. -# Tagged are: -# - functions (even inside other defs or classes) -# - classes -# - filenames -# Warns about files it cannot open. -# No warnings about duplicate tags. - -import sys, re, os - -tags = [] # Modified global variable! - -def main(): - args = sys.argv[1:] - for filename in args: - treat_file(filename) - if tags: - with open('tags', 'w') as fp: - tags.sort() - for s in tags: fp.write(s) - - -expr = r'^[ \t]*(def|class)[ \t]+([a-zA-Z0-9_]+)[ \t]*[:\(]' -matcher = re.compile(expr) - -def treat_file(filename): - try: - fp = open(filename, 'r') - except: - sys.stderr.write('Cannot open %s\n' % filename) - return - with fp: - base = os.path.basename(filename) - if base[-3:] == '.py': - base = base[:-3] - s = base + '\t' + filename + '\t' + '1\n' - tags.append(s) - while 1: - line = fp.readline() - if not line: - break - m = matcher.match(line) - if m: - content = m.group(0) - name = m.group(2) - s = name + '\t' + filename + '\t/^' + content + '/\n' - tags.append(s) - -if __name__ == '__main__': - main() From 5405537813750dbfb133c1af26360297954765f8 Mon Sep 17 00:00:00 2001 From: Noam Cohen Date: Sat, 8 Oct 2022 21:31:57 +0300 Subject: [PATCH 08/28] gh-95011: Migrate syslog module to Argument Clinic (GH-95012) --- Include/internal/pycore_global_strings.h | 3 + .../internal/pycore_runtime_init_generated.h | 21 ++ Modules/clinic/syslogmodule.c.h | 257 ++++++++++++++++++ Modules/syslogmodule.c | 168 +++++++----- 4 files changed, 377 insertions(+), 72 deletions(-) create mode 100644 Modules/clinic/syslogmodule.c.h diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 2966b60e0cd830b..f646979910c887f 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -350,6 +350,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(exception) STRUCT_FOR_ID(exp) STRUCT_FOR_ID(extend) + STRUCT_FOR_ID(facility) STRUCT_FOR_ID(factory) STRUCT_FOR_ID(family) STRUCT_FOR_ID(fanout) @@ -392,6 +393,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(id) + STRUCT_FOR_ID(ident) STRUCT_FOR_ID(ignore) STRUCT_FOR_ID(imag) STRUCT_FOR_ID(importlib) @@ -447,6 +449,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(lo) STRUCT_FOR_ID(locale) STRUCT_FOR_ID(locals) + STRUCT_FOR_ID(logoption) STRUCT_FOR_ID(loop) STRUCT_FOR_ID(mapping) STRUCT_FOR_ID(match) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 617582f96e33f59..bd1fedebd65cf56 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -859,6 +859,7 @@ extern "C" { INIT_ID(exception), \ INIT_ID(exp), \ INIT_ID(extend), \ + INIT_ID(facility), \ INIT_ID(factory), \ INIT_ID(family), \ INIT_ID(fanout), \ @@ -901,6 +902,7 @@ extern "C" { INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(id), \ + INIT_ID(ident), \ INIT_ID(ignore), \ INIT_ID(imag), \ INIT_ID(importlib), \ @@ -956,6 +958,7 @@ extern "C" { INIT_ID(lo), \ INIT_ID(locale), \ INIT_ID(locals), \ + INIT_ID(logoption), \ INIT_ID(loop), \ INIT_ID(mapping), \ INIT_ID(match), \ @@ -2026,6 +2029,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(extend); PyUnicode_InternInPlace(&string); + string = &_Py_ID(facility); + PyUnicode_InternInPlace(&string); string = &_Py_ID(factory); PyUnicode_InternInPlace(&string); string = &_Py_ID(family); @@ -2110,6 +2115,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(id); PyUnicode_InternInPlace(&string); + string = &_Py_ID(ident); + PyUnicode_InternInPlace(&string); string = &_Py_ID(ignore); PyUnicode_InternInPlace(&string); string = &_Py_ID(imag); @@ -2220,6 +2227,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(locals); PyUnicode_InternInPlace(&string); + string = &_Py_ID(logoption); + PyUnicode_InternInPlace(&string); string = &_Py_ID(loop); PyUnicode_InternInPlace(&string); string = &_Py_ID(mapping); @@ -5981,6 +5990,10 @@ _PyStaticObjects_CheckRefcnt(void) { _PyObject_Dump((PyObject *)&_Py_ID(extend)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); }; + if (Py_REFCNT((PyObject *)&_Py_ID(facility)) < _PyObject_IMMORTAL_REFCNT) { + _PyObject_Dump((PyObject *)&_Py_ID(facility)); + Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); + }; if (Py_REFCNT((PyObject *)&_Py_ID(factory)) < _PyObject_IMMORTAL_REFCNT) { _PyObject_Dump((PyObject *)&_Py_ID(factory)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); @@ -6149,6 +6162,10 @@ _PyStaticObjects_CheckRefcnt(void) { _PyObject_Dump((PyObject *)&_Py_ID(id)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); }; + if (Py_REFCNT((PyObject *)&_Py_ID(ident)) < _PyObject_IMMORTAL_REFCNT) { + _PyObject_Dump((PyObject *)&_Py_ID(ident)); + Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); + }; if (Py_REFCNT((PyObject *)&_Py_ID(ignore)) < _PyObject_IMMORTAL_REFCNT) { _PyObject_Dump((PyObject *)&_Py_ID(ignore)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); @@ -6369,6 +6386,10 @@ _PyStaticObjects_CheckRefcnt(void) { _PyObject_Dump((PyObject *)&_Py_ID(locals)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); }; + if (Py_REFCNT((PyObject *)&_Py_ID(logoption)) < _PyObject_IMMORTAL_REFCNT) { + _PyObject_Dump((PyObject *)&_Py_ID(logoption)); + Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); + }; if (Py_REFCNT((PyObject *)&_Py_ID(loop)) < _PyObject_IMMORTAL_REFCNT) { _PyObject_Dump((PyObject *)&_Py_ID(loop)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); diff --git a/Modules/clinic/syslogmodule.c.h b/Modules/clinic/syslogmodule.c.h new file mode 100644 index 000000000000000..0ce66ad4e1a4906 --- /dev/null +++ b/Modules/clinic/syslogmodule.c.h @@ -0,0 +1,257 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif + + +PyDoc_STRVAR(syslog_openlog__doc__, +"openlog($module, /, ident=, logoption=0,\n" +" facility=LOG_USER)\n" +"--\n" +"\n" +"Set logging options of subsequent syslog() calls."); + +#define SYSLOG_OPENLOG_METHODDEF \ + {"openlog", _PyCFunction_CAST(syslog_openlog), METH_FASTCALL|METH_KEYWORDS, syslog_openlog__doc__}, + +static PyObject * +syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, + long facility); + +static PyObject * +syslog_openlog(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(ident), &_Py_ID(logoption), &_Py_ID(facility), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"ident", "logoption", "facility", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "openlog", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *ident = NULL; + long logopt = 0; + long facility = LOG_USER; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 3, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("openlog", "argument 'ident'", "str", args[0]); + goto exit; + } + if (PyUnicode_READY(args[0]) == -1) { + goto exit; + } + ident = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } + if (args[1]) { + logopt = PyLong_AsLong(args[1]); + if (logopt == -1 && PyErr_Occurred()) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_pos; + } + } + facility = PyLong_AsLong(args[2]); + if (facility == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = syslog_openlog_impl(module, ident, logopt, facility); + +exit: + return return_value; +} + +PyDoc_STRVAR(syslog_syslog__doc__, +"syslog([priority=LOG_INFO,] message)\n" +"Send the string message to the system logger."); + +#define SYSLOG_SYSLOG_METHODDEF \ + {"syslog", (PyCFunction)syslog_syslog, METH_VARARGS, syslog_syslog__doc__}, + +static PyObject * +syslog_syslog_impl(PyObject *module, int group_left_1, int priority, + const char *message); + +static PyObject * +syslog_syslog(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int group_left_1 = 0; + int priority = LOG_INFO; + const char *message; + + switch (PyTuple_GET_SIZE(args)) { + case 1: + if (!PyArg_ParseTuple(args, "s:syslog", &message)) { + goto exit; + } + break; + case 2: + if (!PyArg_ParseTuple(args, "is:syslog", &priority, &message)) { + goto exit; + } + group_left_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "syslog.syslog requires 1 to 2 arguments"); + goto exit; + } + return_value = syslog_syslog_impl(module, group_left_1, priority, message); + +exit: + return return_value; +} + +PyDoc_STRVAR(syslog_closelog__doc__, +"closelog($module, /)\n" +"--\n" +"\n" +"Reset the syslog module values and call the system library closelog()."); + +#define SYSLOG_CLOSELOG_METHODDEF \ + {"closelog", (PyCFunction)syslog_closelog, METH_NOARGS, syslog_closelog__doc__}, + +static PyObject * +syslog_closelog_impl(PyObject *module); + +static PyObject * +syslog_closelog(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return syslog_closelog_impl(module); +} + +PyDoc_STRVAR(syslog_setlogmask__doc__, +"setlogmask($module, maskpri, /)\n" +"--\n" +"\n" +"Set the priority mask to maskpri and return the previous mask value."); + +#define SYSLOG_SETLOGMASK_METHODDEF \ + {"setlogmask", (PyCFunction)syslog_setlogmask, METH_O, syslog_setlogmask__doc__}, + +static long +syslog_setlogmask_impl(PyObject *module, long maskpri); + +static PyObject * +syslog_setlogmask(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + long maskpri; + long _return_value; + + maskpri = PyLong_AsLong(arg); + if (maskpri == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = syslog_setlogmask_impl(module, maskpri); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(syslog_LOG_MASK__doc__, +"LOG_MASK($module, pri, /)\n" +"--\n" +"\n" +"Calculates the mask for the individual priority pri."); + +#define SYSLOG_LOG_MASK_METHODDEF \ + {"LOG_MASK", (PyCFunction)syslog_LOG_MASK, METH_O, syslog_LOG_MASK__doc__}, + +static long +syslog_LOG_MASK_impl(PyObject *module, long pri); + +static PyObject * +syslog_LOG_MASK(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + long pri; + long _return_value; + + pri = PyLong_AsLong(arg); + if (pri == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = syslog_LOG_MASK_impl(module, pri); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(syslog_LOG_UPTO__doc__, +"LOG_UPTO($module, pri, /)\n" +"--\n" +"\n" +"Calculates the mask for all priorities up to and including pri."); + +#define SYSLOG_LOG_UPTO_METHODDEF \ + {"LOG_UPTO", (PyCFunction)syslog_LOG_UPTO, METH_O, syslog_LOG_UPTO__doc__}, + +static long +syslog_LOG_UPTO_impl(PyObject *module, long pri); + +static PyObject * +syslog_LOG_UPTO(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + long pri; + long _return_value; + + pri = PyLong_AsLong(arg); + if (pri == -1 && PyErr_Occurred()) { + goto exit; + } + _return_value = syslog_LOG_UPTO_impl(module, pri); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong(_return_value); + +exit: + return return_value; +} +/*[clinic end generated code: output=3b1bdb16565b8fda input=a9049054013a1b77]*/ diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 1593eea94a62bf4..b6296ed0a69aa55 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -54,6 +54,13 @@ Revision history: #include +/*[clinic input] +module syslog +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=478f4ac94a1d4cae]*/ + +#include "clinic/syslogmodule.c.h" + /* only one instance, only one syslog, so globals should be ok */ static PyObject *S_ident_o = NULL; /* identifier, held by openlog() */ static char S_log_open = 0; @@ -113,19 +120,23 @@ syslog_get_argv(void) } +/*[clinic input] +syslog.openlog + + ident: unicode = NULL + logoption as logopt: long = 0 + facility: long(c_default="LOG_USER") = LOG_USER + +Set logging options of subsequent syslog() calls. +[clinic start generated code]*/ + static PyObject * -syslog_openlog(PyObject * self, PyObject * args, PyObject *kwds) +syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, + long facility) +/*[clinic end generated code: output=5476c12829b6eb75 input=8a987a96a586eee7]*/ { - long logopt = 0; - long facility = LOG_USER; - PyObject *ident = NULL; - static char *keywords[] = {"ident", "logoption", "facility", 0}; const char *ident_str = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|Ull:openlog", keywords, &ident, &logopt, &facility)) - return NULL; - if (ident) { Py_INCREF(ident); } @@ -158,48 +169,37 @@ syslog_openlog(PyObject * self, PyObject * args, PyObject *kwds) } -static PyObject * -syslog_syslog(PyObject * self, PyObject * args) -{ - PyObject *message_object; - const char *message; - int priority = LOG_INFO; - if (!PyArg_ParseTuple(args, "iU;[priority,] message string", - &priority, &message_object)) { - PyErr_Clear(); - if (!PyArg_ParseTuple(args, "U;[priority,] message string", - &message_object)) - return NULL; - } +/*[clinic input] +syslog.syslog - message = PyUnicode_AsUTF8(message_object); - if (message == NULL) - return NULL; + [ + priority: int(c_default="LOG_INFO") = LOG_INFO + ] + + message: str + + / +Send the string message to the system logger. +[clinic start generated code]*/ + +static PyObject * +syslog_syslog_impl(PyObject *module, int group_left_1, int priority, + const char *message) +/*[clinic end generated code: output=c3dbc73445a0e078 input=ac83d92b12ea3d4e]*/ +{ if (PySys_Audit("syslog.syslog", "is", priority, message) < 0) { return NULL; } /* if log is not opened, open it now */ if (!S_log_open) { - PyObject *openargs; - - /* Continue even if PyTuple_New fails, because openlog(3) is optional. - * So, we can still do logging in the unlikely event things are so hosed - * that we can't do this tuple. - */ - if ((openargs = PyTuple_New(0))) { - PyObject *openlog_ret = syslog_openlog(self, openargs, NULL); - Py_DECREF(openargs); - if (openlog_ret == NULL) { - return NULL; - } - Py_DECREF(openlog_ret); - } - else { + PyObject *openlog_ret = syslog_openlog_impl(module, NULL, 0, LOG_USER); + if (openlog_ret == NULL) { return NULL; } + Py_DECREF(openlog_ret); } /* Incref ident, because it can be decrefed if syslog.openlog() is @@ -214,8 +214,16 @@ syslog_syslog(PyObject * self, PyObject * args) Py_RETURN_NONE; } + +/*[clinic input] +syslog.closelog + +Reset the syslog module values and call the system library closelog(). +[clinic start generated code]*/ + static PyObject * -syslog_closelog(PyObject *self, PyObject *unused) +syslog_closelog_impl(PyObject *module) +/*[clinic end generated code: output=97890a80a24b1b84 input=fb77a54d447acf07]*/ { if (PySys_Audit("syslog.closelog", NULL) < 0) { return NULL; @@ -228,51 +236,67 @@ syslog_closelog(PyObject *self, PyObject *unused) Py_RETURN_NONE; } -static PyObject * -syslog_setlogmask(PyObject *self, PyObject *args) -{ - long maskpri, omaskpri; +/*[clinic input] +syslog.setlogmask -> long - if (!PyArg_ParseTuple(args, "l;mask for priority", &maskpri)) - return NULL; + maskpri: long + / + +Set the priority mask to maskpri and return the previous mask value. +[clinic start generated code]*/ + +static long +syslog_setlogmask_impl(PyObject *module, long maskpri) +/*[clinic end generated code: output=d6ed163917b434bf input=adff2c2b76c7629c]*/ +{ if (PySys_Audit("syslog.setlogmask", "l", maskpri) < 0) { - return NULL; + return -1; } - omaskpri = setlogmask(maskpri); - return PyLong_FromLong(omaskpri); + + return setlogmask(maskpri); } -static PyObject * -syslog_log_mask(PyObject *self, PyObject *args) +/*[clinic input] +syslog.LOG_MASK -> long + + pri: long + / + +Calculates the mask for the individual priority pri. +[clinic start generated code]*/ + +static long +syslog_LOG_MASK_impl(PyObject *module, long pri) +/*[clinic end generated code: output=c4a5bbfcc74c7c94 input=534829cb7fb5f7d2]*/ { - long mask; - long pri; - if (!PyArg_ParseTuple(args, "l:LOG_MASK", &pri)) - return NULL; - mask = LOG_MASK(pri); - return PyLong_FromLong(mask); + return LOG_MASK(pri); } -static PyObject * -syslog_log_upto(PyObject *self, PyObject *args) +/*[clinic input] +syslog.LOG_UPTO -> long + + pri: long + / + +Calculates the mask for all priorities up to and including pri. +[clinic start generated code]*/ + +static long +syslog_LOG_UPTO_impl(PyObject *module, long pri) +/*[clinic end generated code: output=9eab083c90601d7e input=5e906d6c406b7458]*/ { - long mask; - long pri; - if (!PyArg_ParseTuple(args, "l:LOG_UPTO", &pri)) - return NULL; - mask = LOG_UPTO(pri); - return PyLong_FromLong(mask); + return LOG_UPTO(pri); } /* List of functions defined in the module */ static PyMethodDef syslog_methods[] = { - {"openlog", _PyCFunction_CAST(syslog_openlog), METH_VARARGS | METH_KEYWORDS}, - {"closelog", syslog_closelog, METH_NOARGS}, - {"syslog", syslog_syslog, METH_VARARGS}, - {"setlogmask", syslog_setlogmask, METH_VARARGS}, - {"LOG_MASK", syslog_log_mask, METH_VARARGS}, - {"LOG_UPTO", syslog_log_upto, METH_VARARGS}, + SYSLOG_OPENLOG_METHODDEF + SYSLOG_CLOSELOG_METHODDEF + SYSLOG_SYSLOG_METHODDEF + SYSLOG_SETLOGMASK_METHODDEF + SYSLOG_LOG_MASK_METHODDEF + SYSLOG_LOG_UPTO_METHODDEF {NULL, NULL, 0} }; From 75751f4aa5d70f65856645a9128fd42d92d6692c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sat, 8 Oct 2022 21:21:38 +0200 Subject: [PATCH 09/28] Auto-cancel old builds when new commit pushed to branch (#98009) * Auto-cancel old builds when new commit pushed to branch * Add a fallback Co-authored-by: Ezio Melotti * Use the same group for all workflows. Co-authored-by: Ezio Melotti --- .github/workflows/build.yml | 4 ++++ .github/workflows/build_msi.yml | 4 ++++ .github/workflows/doc.yml | 4 ++++ .github/workflows/verify-ensurepip-wheels.yml | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc91c0dbcb027bc..8f5676eec08e77e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,10 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: check_source: name: 'Check for source changes' diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index 528679c0ac6b374..5f1dcae190efbc2 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -18,6 +18,10 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: build: name: Windows Installer diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index d95d089ed66755e..10e4cf074a590a6 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -28,6 +28,10 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: build_doc: name: 'Docs' diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 61e3d1adf534d54..9f4754f912b09f6 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -16,6 +16,10 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: verify: runs-on: ubuntu-latest From d8765284f33ec221b5fe07b439ca2dc2d648251d Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sun, 9 Oct 2022 02:22:19 +0530 Subject: [PATCH 10/28] GH-94597: deprecate `SafeChildWatcher`, `FastChildWatcher` and `MultiLoopChildWatcher` child watchers (#98089) --- Doc/whatsnew/3.12.rst | 18 +++++++ Lib/asyncio/unix_events.py | 15 ++++++ Lib/test/test_asyncio/test_events.py | 10 ++-- Lib/test/test_asyncio/test_streams.py | 6 ++- Lib/test/test_asyncio/test_subprocess.py | 48 ++++++++----------- Lib/test/test_asyncio/test_unix_events.py | 21 ++++++-- ...2-10-08-06-59-46.gh-issue-94597.TsS0oT.rst | 1 + 7 files changed, 81 insertions(+), 38 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 341e85103a3cf7b..903533261006656 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -109,6 +109,24 @@ New Modules Improved Modules ================ +asyncio +------- + +* On Linux, :mod:`asyncio` uses :class:`~asyncio.PidfdChildWatcher` by default + if :func:`os.pidfd_open` is available and functional instead of + :class:`~asyncio.ThreadedChildWatcher`. + (Contributed by Kumar Aditya in :gh:`98024`.) + +* The child watcher classes :class:`~asyncio.MultiLoopChildWatcher`, + :class:`~asyncio.FastChildWatcher` and + :class:`~asyncio.SafeChildWatcher` are deprecated and + will be removed in Python 3.14. It is recommended to not manually + configure a child watcher as the event loop now uses the best available + child watcher for each platform (:class:`~asyncio.PidfdChildWatcher` + if supported and :class:`~asyncio.ThreadedChildWatcher` otherwise). + (Contributed by Kumar Aditya in :gh:`94597`.) + + pathlib ------- diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 7fc75cd17ef7415..bdffc032e318fdf 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -1022,6 +1022,13 @@ class SafeChildWatcher(BaseChildWatcher): big number of children (O(n) each time SIGCHLD is raised) """ + def __init__(self): + super().__init__() + warnings._deprecated("SafeChildWatcher", + "{name!r} is deprecated as of Python 3.12 and will be " + "removed in Python {remove}.", + remove=(3, 14)) + def close(self): self._callbacks.clear() super().close() @@ -1100,6 +1107,10 @@ def __init__(self): self._lock = threading.Lock() self._zombies = {} self._forks = 0 + warnings._deprecated("FastChildWatcher", + "{name!r} is deprecated as of Python 3.12 and will be " + "removed in Python {remove}.", + remove=(3, 14)) def close(self): self._callbacks.clear() @@ -1212,6 +1223,10 @@ class MultiLoopChildWatcher(AbstractChildWatcher): def __init__(self): self._callbacks = {} self._saved_sighandler = None + warnings._deprecated("MultiLoopChildWatcher", + "{name!r} is deprecated as of Python 3.12 and will be " + "removed in Python {remove}.", + remove=(3, 14)) def is_active(self): return self._saved_sighandler is not None diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 80d7152128c4693..98b55dec37f4784 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -22,7 +22,7 @@ import unittest from unittest import mock import weakref - +import warnings if sys.platform not in ('win32', 'vxworks'): import tty @@ -2055,7 +2055,9 @@ def test_remove_fds_after_closing(self): class UnixEventLoopTestsMixin(EventLoopTestsMixin): def setUp(self): super().setUp() - watcher = asyncio.SafeChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) asyncio.set_child_watcher(watcher) @@ -2652,7 +2654,9 @@ def setUp(self): asyncio.set_event_loop(self.loop) if sys.platform != 'win32': - watcher = asyncio.SafeChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) asyncio.set_child_watcher(watcher) diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 0c49099bc499a58..8fb9313e09dd0ea 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -9,6 +9,7 @@ import threading import unittest from unittest import mock +import warnings from test.support import socket_helper try: import ssl @@ -791,8 +792,9 @@ def test_read_all_from_pipe_reader(self): protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop) transport, _ = self.loop.run_until_complete( self.loop.connect_read_pipe(lambda: protocol, pipe)) - - watcher = asyncio.SafeChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + watcher = asyncio.SafeChildWatcher() watcher.attach_loop(self.loop) try: asyncio.set_child_watcher(watcher) diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 6ba889407b802e2..915ad5587f0a489 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -4,7 +4,6 @@ import sys import unittest import warnings -import functools from unittest import mock import asyncio @@ -31,19 +30,6 @@ 'sys.stdout.buffer.write(data)'))] -@functools.cache -def _has_pidfd_support(): - if not hasattr(os, 'pidfd_open'): - return False - - try: - os.close(os.pidfd_open(os.getpid())) - except OSError: - return False - - return True - - def tearDownModule(): asyncio.set_event_loop_policy(None) @@ -688,7 +674,7 @@ def setUp(self): self.loop = policy.new_event_loop() self.set_event_loop(self.loop) - watcher = self.Watcher() + watcher = self._get_watcher() watcher.attach_loop(self.loop) policy.set_child_watcher(watcher) @@ -703,32 +689,38 @@ def tearDown(self): class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - Watcher = unix_events.ThreadedChildWatcher - - @unittest.skip("bpo-38323: MultiLoopChildWatcher has a race condition \ - and these tests can hang the test suite") - class SubprocessMultiLoopWatcherTests(SubprocessWatcherMixin, - test_utils.TestCase): - - Watcher = unix_events.MultiLoopChildWatcher + def _get_watcher(self): + return unix_events.ThreadedChildWatcher() class SubprocessSafeWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - Watcher = unix_events.SafeChildWatcher + def _get_watcher(self): + with self.assertWarns(DeprecationWarning): + return unix_events.SafeChildWatcher() + + class MultiLoopChildWatcherTests(test_utils.TestCase): + + def test_warns(self): + with self.assertWarns(DeprecationWarning): + unix_events.MultiLoopChildWatcher() class SubprocessFastWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - Watcher = unix_events.FastChildWatcher + def _get_watcher(self): + with self.assertWarns(DeprecationWarning): + return unix_events.FastChildWatcher() @unittest.skipUnless( - _has_pidfd_support(), + unix_events.can_use_pidfd(), "operating system does not support pidfds", ) class SubprocessPidfdWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): - Watcher = unix_events.PidfdChildWatcher + + def _get_watcher(self): + return unix_events.PidfdChildWatcher() class GenericWatcherTests(test_utils.TestCase): @@ -758,7 +750,7 @@ async def execute(): @unittest.skipUnless( - _has_pidfd_support(), + unix_events.can_use_pidfd(), "operating system does not support pidfds", ) def test_create_subprocess_with_pidfd(self): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 03fb5e649d8e974..025da0f20ed47e3 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -12,6 +12,7 @@ import threading import unittest from unittest import mock +import warnings from test.support import os_helper from test.support import socket_helper @@ -1686,12 +1687,16 @@ def test_close(self, m_waitpid): class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): def create_watcher(self): - return asyncio.SafeChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return asyncio.SafeChildWatcher() class FastChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): def create_watcher(self): - return asyncio.FastChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return asyncio.FastChildWatcher() class PolicyTests(unittest.TestCase): @@ -1724,7 +1729,9 @@ def test_get_default_child_watcher(self): def test_get_child_watcher_after_set(self): policy = self.create_policy() - watcher = asyncio.FastChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + watcher = asyncio.FastChildWatcher() policy.set_child_watcher(watcher) self.assertIs(policy._watcher, watcher) @@ -1745,7 +1752,9 @@ def f(): policy.get_event_loop().close() policy = self.create_policy() - policy.set_child_watcher(asyncio.SafeChildWatcher()) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + policy.set_child_watcher(asyncio.SafeChildWatcher()) th = threading.Thread(target=f) th.start() @@ -1757,7 +1766,9 @@ def test_child_watcher_replace_mainloop_existing(self): # Explicitly setup SafeChildWatcher, # default ThreadedChildWatcher has no _loop property - watcher = asyncio.SafeChildWatcher() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + watcher = asyncio.SafeChildWatcher() policy.set_child_watcher(watcher) watcher.attach_loop(loop) diff --git a/Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst b/Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst new file mode 100644 index 000000000000000..f504ccf39ec9488 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-08-06-59-46.gh-issue-94597.TsS0oT.rst @@ -0,0 +1 @@ +The child watcher classes :class:`~asyncio.MultiLoopChildWatcher`, :class:`~asyncio.FastChildWatcher` and :class:`~asyncio.SafeChildWatcher` are deprecated and will be removed in Python 3.14. Patch by Kumar Aditya. From 3378ebb933b00e1b95f1112511aa2cfb597a4ebf Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Sat, 8 Oct 2022 18:16:52 -0400 Subject: [PATCH 11/28] Fix link to Lifecycle of a Pull Request in CONTRIBUTING (#98102) * Fix link to Lifecycle of a Pull Request in CONTRIBUTING * Remove trailing backslash. Co-authored-by: Ezio Melotti --- .github/CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 627f57070d200bb..f4affee76e1d445 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -38,7 +38,7 @@ also suggestions on how you can most effectively help the project. Please be aware that our workflow does deviate slightly from the typical GitHub project. Details on how to properly submit a pull request are covered in -`Lifecycle of a Pull Request `_. +`Lifecycle of a Pull Request `_. We utilize various bots and status checks to help with this, so do follow the comments they leave and their "Details" links, respectively. The key points of our workflow that are not covered by a bot or status check are: From 2d2e01aa4cb72db8dabcd04e87f1e60b3597267e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sun, 9 Oct 2022 03:54:21 +0200 Subject: [PATCH 12/28] Minor edits to the Descriptor HowTo Guide (GH-24901) Co-authored-by: Raymond Hettinger --- Doc/howto/descriptor.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 91a6c31c33b8bde..74710d9b3fc2edc 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -847,7 +847,7 @@ afterwards, :meth:`__set_name__` will need to be called manually. ORM example ----------- -The following code is simplified skeleton showing how data descriptors could +The following code is a simplified skeleton showing how data descriptors could be used to implement an `object relational mapping `_. @@ -1535,6 +1535,8 @@ by member descriptors: def __get__(self, obj, objtype=None): 'Emulate member_get() in Objects/descrobject.c' # Also see PyMember_GetOne() in Python/structmember.c + if obj is None: + return self value = obj._slotvalues[self.offset] if value is null: raise AttributeError(self.name) @@ -1563,13 +1565,13 @@ variables: class Type(type): 'Simulate how the type metaclass adds member objects for slots' - def __new__(mcls, clsname, bases, mapping): + def __new__(mcls, clsname, bases, mapping, **kwargs): 'Emulate type_new() in Objects/typeobject.c' # type_new() calls PyTypeReady() which calls add_methods() slot_names = mapping.get('slot_names', []) for offset, name in enumerate(slot_names): mapping[name] = Member(name, clsname, offset) - return type.__new__(mcls, clsname, bases, mapping) + return type.__new__(mcls, clsname, bases, mapping, **kwargs) The :meth:`object.__new__` method takes care of creating instances that have slots instead of an instance dictionary. Here is a rough simulation in pure @@ -1580,7 +1582,7 @@ Python: class Object: 'Simulate how object.__new__() allocates memory for __slots__' - def __new__(cls, *args): + def __new__(cls, *args, **kwargs): 'Emulate object_new() in Objects/typeobject.c' inst = super().__new__(cls) if hasattr(cls, 'slot_names'): @@ -1593,7 +1595,7 @@ Python: cls = type(self) if hasattr(cls, 'slot_names') and name not in cls.slot_names: raise AttributeError( - f'{type(self).__name__!r} object has no attribute {name!r}' + f'{cls.__name__!r} object has no attribute {name!r}' ) super().__setattr__(name, value) @@ -1602,7 +1604,7 @@ Python: cls = type(self) if hasattr(cls, 'slot_names') and name not in cls.slot_names: raise AttributeError( - f'{type(self).__name__!r} object has no attribute {name!r}' + f'{cls.__name__!r} object has no attribute {name!r}' ) super().__delattr__(name) From a04656ec32bd5f3684ea18b195f76000df4962eb Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 9 Oct 2022 20:16:33 +0900 Subject: [PATCH 13/28] gh-97841: Add methoddef for _filters_mutated (gh-98115) --- Python/_warnings.c | 11 ++++++++--- Python/clinic/_warnings.c.h | 19 ++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index 0d4c50f769b03c6..b46fbdca9db38c6 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1086,8 +1086,14 @@ warnings_warn_explicit_impl(PyObject *module, PyObject *message, return returned; } +/*[clinic input] +_filters_mutated as warnings_filters_mutated + +[clinic start generated code]*/ + static PyObject * -warnings_filters_mutated(PyObject *self, PyObject *Py_UNUSED(args)) +warnings_filters_mutated_impl(PyObject *module) +/*[clinic end generated code: output=8ce517abd12b88f4 input=35ecbf08ee2491b2]*/ { PyInterpreterState *interp = get_current_interp(); if (interp == NULL) { @@ -1344,8 +1350,7 @@ _PyErr_WarnUnawaitedCoroutine(PyObject *coro) static PyMethodDef warnings_functions[] = { WARNINGS_WARN_METHODDEF WARNINGS_WARN_EXPLICIT_METHODDEF - {"_filters_mutated", _PyCFunction_CAST(warnings_filters_mutated), METH_NOARGS, - NULL}, + WARNINGS_FILTERS_MUTATED_METHODDEF /* XXX(brett.cannon): add showwarning? */ /* XXX(brett.cannon): Reasonable to add formatwarning? */ {NULL, NULL} /* sentinel */ diff --git a/Python/clinic/_warnings.c.h b/Python/clinic/_warnings.c.h index 13ebbf45b8e1683..8838a42afc1c2ab 100644 --- a/Python/clinic/_warnings.c.h +++ b/Python/clinic/_warnings.c.h @@ -199,4 +199,21 @@ warnings_warn_explicit(PyObject *module, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=2eac4fabc87a4d56 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(warnings_filters_mutated__doc__, +"_filters_mutated($module, /)\n" +"--\n" +"\n"); + +#define WARNINGS_FILTERS_MUTATED_METHODDEF \ + {"_filters_mutated", (PyCFunction)warnings_filters_mutated, METH_NOARGS, warnings_filters_mutated__doc__}, + +static PyObject * +warnings_filters_mutated_impl(PyObject *module); + +static PyObject * +warnings_filters_mutated(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return warnings_filters_mutated_impl(module); +} +/*[clinic end generated code: output=0d264d1ddfc37100 input=a9049054013a1b77]*/ From f1879690aa0498ffc4427c95aed94e1ca629e072 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Mon, 10 Oct 2022 00:29:25 +0200 Subject: [PATCH 14/28] Update whatsnew instructions for GitHub (#98124) --- Doc/whatsnew/3.12.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 903533261006656..f8122ed1dc44f0b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -37,11 +37,11 @@ * Credit the author of a patch or bugfix. Just the name is sufficient; the e-mail address isn't necessary. - * It's helpful to add the bug/patch number as a comment: + * It's helpful to add the issue number as a comment: XXX Describe the transmogrify() function added to the socket module. - (Contributed by P.Y. Developer in :issue:`12345`.) + (Contributed by P.Y. Developer in :gh:`12345`.) This saves the maintainer the effort of going through the VCS log when researching a change. From 281a3f18cc2afac0fa92c75e807775971e531711 Mon Sep 17 00:00:00 2001 From: Stanley <46876382+slateny@users.noreply.github.com> Date: Sun, 9 Oct 2022 17:51:02 -0700 Subject: [PATCH 15/28] gh-56133: copyreg docs: Clarify function/constructor parameter (#95497) --- Doc/library/copyreg.rst | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index dc35965be3e40db..866b180f4bc3b82 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -25,20 +25,17 @@ Such constructors may be factory functions or class instances. hence not valid as a constructor), raises :exc:`TypeError`. -.. function:: pickle(type, function, constructor=None) +.. function:: pickle(type, function, constructor_ob=None) Declares that *function* should be used as a "reduction" function for objects of type *type*. *function* should return either a string or a tuple - containing two or three elements. + containing two or three elements. See the :attr:`~pickle.Pickler.dispatch_table` + for more details on the interface of *function*. - The optional *constructor* parameter, if provided, is a callable object which - can be used to reconstruct the object when called with the tuple of arguments - returned by *function* at pickling time. A :exc:`TypeError` is raised if the - *constructor* is not callable. + The *constructor_ob* parameter is a legacy feature and is now ignored, but if + passed it must be a callable. - See the :mod:`pickle` module for more details on the interface - expected of *function* and *constructor*. Note that the - :attr:`~pickle.Pickler.dispatch_table` attribute of a pickler + Note that the :attr:`~pickle.Pickler.dispatch_table` attribute of a pickler object or subclass of :class:`pickle.Pickler` can also be used for declaring reduction functions. From c459fedf7cfd5dadf72e088d789c7375b3a6e093 Mon Sep 17 00:00:00 2001 From: da-woods Date: Mon, 10 Oct 2022 01:55:53 +0100 Subject: [PATCH 16/28] Fix types in buffer/memoryview docs (#98118) The definition of obj in the `Py_buffer` struct is as a PyObject* https://github.com/python/cpython/blob/ec091bd47e2f968b0d1631b9a8104283a7beeb1b/Include/pybuffer.h#L22 PyMemoryView_GET_BASE returns `.obj` - thus its return type should be a PyObject* (or at least a void*). It definitely doesn't return `Py_buffer` --- Doc/c-api/buffer.rst | 2 +- Doc/c-api/memoryview.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 05e131d06b909d1..a04062fb2a68f1d 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -99,7 +99,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. For :term:`contiguous` arrays, the value points to the beginning of the memory block. - .. c:member:: void *obj + .. c:member:: PyObject *obj A new reference to the exporting object. The reference is owned by the consumer and automatically decremented and set to ``NULL`` by diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index 4d94b3f545f3279..ebd5c7760437bf3 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -55,7 +55,7 @@ any other object. *mview* **must** be a memoryview instance; this macro doesn't check its type, you must do it yourself or you will risk crashes. -.. c:function:: Py_buffer *PyMemoryView_GET_BASE(PyObject *mview) +.. c:function:: PyObject *PyMemoryView_GET_BASE(PyObject *mview) Return either a pointer to the exporting object that the memoryview is based on or ``NULL`` if the memoryview has been created by one of the functions From fc342c62e0debb194d60e79b37e346bf8d940d7a Mon Sep 17 00:00:00 2001 From: Tiger Date: Sun, 9 Oct 2022 20:31:53 -0500 Subject: [PATCH 17/28] gh-98083: Fix URLs in `README.rst` (#98082) --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 4ae5dfd1bb2525d..8227ae748d39341 100644 --- a/README.rst +++ b/README.rst @@ -65,7 +65,7 @@ Building a complete Python installation requires the use of various additional third-party libraries, depending on your build platform and configure options. Not all standard library modules are buildable or useable on all platforms. Refer to the -`Install dependencies `_ +`Install dependencies `_ section of the `Developer Guide`_ for current detailed information on dependencies for various Linux distributions and macOS. @@ -135,7 +135,7 @@ What's New We have a comprehensive overview of the changes in the `What's New in Python 3.12 `_ document. For a more detailed change log, read `Misc/NEWS -`_, but a full +`_, but a full accounting of changes can only be gleaned from the `commit history `_. @@ -189,7 +189,7 @@ your environment, you can `file a bug report `_ and include relevant output from that command to show the issue. -See `Running & Writing Tests `_ +See `Running & Writing Tests `_ for more on running tests. Installing multiple versions From ad817cd5c44416da3752ebf9baf16d650703275c Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Mon, 10 Oct 2022 03:59:07 +0200 Subject: [PATCH 18/28] bpo-43564: preserve original exception in args of FTP URLError (#24938) * bpo-43564: preserve original error in args of FTP URLError * Add NEWS blurb Co-authored-by: Carl Meyer --- Lib/urllib/request.py | 2 +- .../next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index e2d5b8c5c59a3c9..278aa3a14bfeea0 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1582,7 +1582,7 @@ def ftp_open(self, req): headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) except ftplib.all_errors as exp: - raise URLError(f'ftp error: {exp}') from exp + raise URLError(exp) from exp def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, diff --git a/Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst b/Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst new file mode 100644 index 000000000000000..6c63fa4928c62e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-09-12-12-38.gh-issue-87730.ClgP3f.rst @@ -0,0 +1,3 @@ +Wrap network errors consistently in urllib FTP support, so the test suite +doesn't fail when a network is available but the public internet is not +reachable. From 571e23d99157ed7ad67ca2334a396fc9ddbe07ec Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Mon, 10 Oct 2022 09:01:16 +0200 Subject: [PATCH 19/28] doc: remove a misleading statement. (GH-98093) --- Doc/tutorial/introduction.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index ba0f47705297830..558b1c3eec60ede 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -70,8 +70,8 @@ the ones with a fractional part (e.g. ``5.0``, ``1.6``) have type :class:`float`. We will see more about numeric types later in the tutorial. Division (``/``) always returns a float. To do :term:`floor division` and -get an integer result (discarding any fractional result) you can use the ``//`` -operator; to calculate the remainder you can use ``%``:: +get an integer result you can use the ``//`` operator; to calculate +the remainder you can use ``%``:: >>> 17 / 3 # classic division returns a float 5.666666666666667 From 187e853690908ca2af19a0701ca7529b43d05df9 Mon Sep 17 00:00:00 2001 From: Stanley <46876382+slateny@users.noreply.github.com> Date: Mon, 10 Oct 2022 06:43:01 -0700 Subject: [PATCH 20/28] gh-83940: os docs: Improve wording for getenv/getenvb (#98113) --- Doc/library/os.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 8727f811def1c09..43066fa1e13d74b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -304,8 +304,8 @@ process and user. .. function:: getenv(key, default=None) - Return the value of the environment variable *key* if it exists, or - *default* if it doesn't. *key*, *default* and the result are str. Note that + Return the value of the environment variable *key* as a string if it exists, or + *default* if it doesn't. *key* is a string. Note that since :func:`getenv` uses :data:`os.environ`, the mapping of :func:`getenv` is similarly also captured on import, and the function may not reflect future environment changes. @@ -319,8 +319,8 @@ process and user. .. function:: getenvb(key, default=None) - Return the value of the environment variable *key* if it exists, or - *default* if it doesn't. *key*, *default* and the result are bytes. Note that + Return the value of the environment variable *key* as bytes if it exists, or + *default* if it doesn't. *key* must be bytes. Note that since :func:`getenvb` uses :data:`os.environb`, the mapping of :func:`getenvb` is similarly also captured on import, and the function may not reflect future environment changes. From dfcdee4a18ece6f7c4fe1aa5830ee861f8b15b24 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 10 Oct 2022 11:28:41 -0400 Subject: [PATCH 21/28] gh-94808: Add coverage for bytesarray_setitem (#95802) --- Lib/test/test_bytes.py | 111 ++++++++++++++++++++++++-------------- Modules/_testcapimodule.c | 15 ++++++ 2 files changed, 85 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 53ba1ad6f5911f6..7c62b722059d124 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1225,6 +1225,8 @@ class SubBytes(bytes): class ByteArrayTest(BaseBytesTest, unittest.TestCase): type2test = bytearray + _testcapi = import_helper.import_module('_testcapi') + def test_getitem_error(self): b = bytearray(b'python') msg = "bytearray indices must be integers or slices" @@ -1317,47 +1319,73 @@ def by(s): self.assertEqual(re.findall(br"\w+", b), [by("Hello"), by("world")]) def test_setitem(self): - b = bytearray([1, 2, 3]) - b[1] = 100 - self.assertEqual(b, bytearray([1, 100, 3])) - b[-1] = 200 - self.assertEqual(b, bytearray([1, 100, 200])) - b[0] = Indexable(10) - self.assertEqual(b, bytearray([10, 100, 200])) - try: - b[3] = 0 - self.fail("Didn't raise IndexError") - except IndexError: - pass - try: - b[-10] = 0 - self.fail("Didn't raise IndexError") - except IndexError: - pass - try: - b[0] = 256 - self.fail("Didn't raise ValueError") - except ValueError: - pass - try: - b[0] = Indexable(-1) - self.fail("Didn't raise ValueError") - except ValueError: - pass - try: - b[0] = None - self.fail("Didn't raise TypeError") - except TypeError: - pass + def setitem_as_mapping(b, i, val): + b[i] = val + + def setitem_as_sequence(b, i, val): + self._testcapi.sequence_setitem(b, i, val) + + def do_tests(setitem): + b = bytearray([1, 2, 3]) + setitem(b, 1, 100) + self.assertEqual(b, bytearray([1, 100, 3])) + setitem(b, -1, 200) + self.assertEqual(b, bytearray([1, 100, 200])) + setitem(b, 0, Indexable(10)) + self.assertEqual(b, bytearray([10, 100, 200])) + try: + setitem(b, 3, 0) + self.fail("Didn't raise IndexError") + except IndexError: + pass + try: + setitem(b, -10, 0) + self.fail("Didn't raise IndexError") + except IndexError: + pass + try: + setitem(b, 0, 256) + self.fail("Didn't raise ValueError") + except ValueError: + pass + try: + setitem(b, 0, Indexable(-1)) + self.fail("Didn't raise ValueError") + except ValueError: + pass + try: + setitem(b, 0, None) + self.fail("Didn't raise TypeError") + except TypeError: + pass + + with self.subTest("tp_as_mapping"): + do_tests(setitem_as_mapping) + + with self.subTest("tp_as_sequence"): + do_tests(setitem_as_sequence) def test_delitem(self): - b = bytearray(range(10)) - del b[0] - self.assertEqual(b, bytearray(range(1, 10))) - del b[-1] - self.assertEqual(b, bytearray(range(1, 9))) - del b[4] - self.assertEqual(b, bytearray([1, 2, 3, 4, 6, 7, 8])) + def del_as_mapping(b, i): + del b[i] + + def del_as_sequence(b, i): + self._testcapi.sequence_delitem(b, i) + + def do_tests(delete): + b = bytearray(range(10)) + delete(b, 0) + self.assertEqual(b, bytearray(range(1, 10))) + delete(b, -1) + self.assertEqual(b, bytearray(range(1, 9))) + delete(b, 4) + self.assertEqual(b, bytearray([1, 2, 3, 4, 6, 7, 8])) + + with self.subTest("tp_as_mapping"): + do_tests(del_as_mapping) + + with self.subTest("tp_as_sequence"): + do_tests(del_as_sequence) def test_setslice(self): b = bytearray(range(10)) @@ -1729,6 +1757,8 @@ def test_repeat_after_setslice(self): self.assertEqual(b3, b'xcxcxc') def test_mutating_index(self): + # See gh-91153 + class Boom: def __index__(self): b.clear() @@ -1740,10 +1770,9 @@ def __index__(self): b[0] = Boom() with self.subTest("tp_as_sequence"): - _testcapi = import_helper.import_module('_testcapi') b = bytearray(b'Now you see me...') with self.assertRaises(IndexError): - _testcapi.sequence_setitem(b, 0, Boom()) + self._testcapi.sequence_setitem(b, 0, Boom()) class AssortedBytesTest(unittest.TestCase): diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 173d7c2cb805308..95c67fc95e891a8 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4846,6 +4846,20 @@ sequence_setitem(PyObject *self, PyObject *args) } +static PyObject * +sequence_delitem(PyObject *self, PyObject *args) +{ + Py_ssize_t i; + PyObject *seq; + if (!PyArg_ParseTuple(args, "On", &seq, &i)) { + return NULL; + } + if (PySequence_DelItem(seq, i)) { + return NULL; + } + Py_RETURN_NONE; +} + static PyObject * hasattr_string(PyObject *self, PyObject* args) { @@ -5885,6 +5899,7 @@ static PyMethodDef TestMethods[] = { {"write_unraisable_exc", test_write_unraisable_exc, METH_VARARGS}, {"sequence_getitem", sequence_getitem, METH_VARARGS}, {"sequence_setitem", sequence_setitem, METH_VARARGS}, + {"sequence_delitem", sequence_delitem, METH_VARARGS}, {"hasattr_string", hasattr_string, METH_VARARGS}, {"meth_varargs", meth_varargs, METH_VARARGS}, {"meth_varargs_keywords", _PyCFunction_CAST(meth_varargs_keywords), METH_VARARGS|METH_KEYWORDS}, From 553d3c10172254b190078c50eb9f8e60522c8f41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20G=C3=B6rgens?= Date: Tue, 11 Oct 2022 00:12:29 +0800 Subject: [PATCH 22/28] gh-96821: Fix undefined behaviour in `audioop.c` (#96923) * gh-96821: Fix undefined behaviour in `audioop.c` Left-shifting negative numbers is undefined behaviour. Fortunately, multiplication works just as well, is defined behaviour, and gets compiled to the same machine code as before by optimizing compilers. Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- ...2-09-19-03-35-01.gh-issue-96821.izK6JA.rst | 1 + Modules/audioop.c | 27 ++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-09-19-03-35-01.gh-issue-96821.izK6JA.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-09-19-03-35-01.gh-issue-96821.izK6JA.rst b/Misc/NEWS.d/next/Core and Builtins/2022-09-19-03-35-01.gh-issue-96821.izK6JA.rst new file mode 100644 index 000000000000000..73d0c76f0297cab --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-09-19-03-35-01.gh-issue-96821.izK6JA.rst @@ -0,0 +1 @@ +Fix undefined behaviour in ``audioop.c``. diff --git a/Modules/audioop.c b/Modules/audioop.c index d74e634ac44889c..c29a3e8df4099db 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -59,6 +59,8 @@ static const int16_t seg_uend[8] = { static int16_t search(int16_t val, const int16_t *table, int size) { + assert(0 <= size); + assert(size < INT16_MAX); int i; for (i = 0; i < size; i++) { @@ -170,6 +172,7 @@ st_14linear2ulaw(int16_t pcm_val) /* 2's complement (14-bit range) */ if (seg >= 8) /* out of range, return maximum value. */ return (unsigned char) (0x7F ^ mask); else { + assert(seg >= 0); uval = (unsigned char) (seg << 4) | ((pcm_val >> (seg + 1)) & 0xF); return (uval ^ mask); } @@ -300,13 +303,13 @@ static const int stepsizeTable[89] = { #ifdef WORDS_BIGENDIAN #define GETINT24(cp, i) ( \ ((unsigned char *)(cp) + (i))[2] + \ - (((unsigned char *)(cp) + (i))[1] << 8) + \ - (((signed char *)(cp) + (i))[0] << 16) ) + (((unsigned char *)(cp) + (i))[1] * (1 << 8)) + \ + (((signed char *)(cp) + (i))[0] * (1 << 16)) ) #else #define GETINT24(cp, i) ( \ ((unsigned char *)(cp) + (i))[0] + \ - (((unsigned char *)(cp) + (i))[1] << 8) + \ - (((signed char *)(cp) + (i))[2] << 16) ) + (((unsigned char *)(cp) + (i))[1] * (1 << 8)) + \ + (((signed char *)(cp) + (i))[2] * (1 << 16)) ) #endif @@ -347,10 +350,10 @@ static const int stepsizeTable[89] = { } while(0) -#define GETSAMPLE32(size, cp, i) ( \ - (size == 1) ? (int)GETINT8((cp), (i)) << 24 : \ - (size == 2) ? (int)GETINT16((cp), (i)) << 16 : \ - (size == 3) ? (int)GETINT24((cp), (i)) << 8 : \ +#define GETSAMPLE32(size, cp, i) ( \ + (size == 1) ? (int)GETINT8((cp), (i)) * (1 << 24) : \ + (size == 2) ? (int)GETINT16((cp), (i)) * (1 << 16) : \ + (size == 3) ? (int)GETINT24((cp), (i)) * (1 << 8) : \ (int)GETINT32((cp), (i))) #define SETSAMPLE32(size, cp, i, val) do { \ @@ -1558,7 +1561,7 @@ audioop_ulaw2lin_impl(PyObject *module, Py_buffer *fragment, int width) cp = fragment->buf; for (i = 0; i < fragment->len*width; i += width) { - int val = st_ulaw2linear16(*cp++) << 16; + int val = st_ulaw2linear16(*cp++) * (1 << 16); SETSAMPLE32(width, ncp, i, val); } return rv; @@ -1632,7 +1635,7 @@ audioop_alaw2lin_impl(PyObject *module, Py_buffer *fragment, int width) cp = fragment->buf; for (i = 0; i < fragment->len*width; i += width) { - val = st_alaw2linear16(*cp++) << 16; + val = st_alaw2linear16(*cp++) * (1 << 16); SETSAMPLE32(width, ncp, i, val); } return rv; @@ -1757,7 +1760,7 @@ audioop_lin2adpcm_impl(PyObject *module, Py_buffer *fragment, int width, /* Step 6 - Output value */ if ( bufferstep ) { - outputbuffer = (delta << 4) & 0xf0; + outputbuffer = (delta * (1 << 4)) & 0xf0; } else { *ncp++ = (delta & 0x0f) | outputbuffer; } @@ -1875,7 +1878,7 @@ audioop_adpcm2lin_impl(PyObject *module, Py_buffer *fragment, int width, step = stepsizeTable[index]; /* Step 6 - Output value */ - SETSAMPLE32(width, ncp, i, valpred << 16); + SETSAMPLE32(width, ncp, i, valpred * (1 << 16)); } rv = Py_BuildValue("(O(ii))", str, valpred, index); From 6a757da080121c4add61931ae46389b3f3e990a1 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Mon, 10 Oct 2022 19:27:52 +0100 Subject: [PATCH 23/28] gh-88452: Add a warning about non-portability of environments. (GH-98155) --- Doc/library/venv.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 06810612acba548..40eccdec16b39ae 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -85,6 +85,19 @@ Creating virtual environments without there needing to be any reference to its virtual environment in ``PATH``. +.. warning:: Because scripts installed in environments should not expect the + environment to be activated, their shebang lines contain the absolute paths + to their environment's interpreters. Because of this, environments are + inherently non-portable, in the general case. You should always have a + simple means of recreating an environment (for example, if you have a + requirements file ``requirements.txt``, you can invoke ``pip install -r + requirements.txt`` using the environment's ``pip`` to install all of the + packages needed by the environment). If for any reason you need to move the + environment to a new location, you should recreate it at the desired + location and delete the one at the old location. If you move an environment + because you moved a parent directory of it, you should recreate the + environment in its new location. Otherwise, software installed into the + environment may not work as expected. .. _venv-api: From f871e9a7bb4eb823da481b99052636763ec8e710 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 10 Oct 2022 15:14:31 -0700 Subject: [PATCH 24/28] gh-44098: Release the GIL during mmap on Unix (GH-98146) This seems pretty straightforward. The issue mentions other calls in mmapmodule that we could release the GIL on, but those are in methods where we'd need to be careful to ensure that something sensible happens if those are called concurrently. In prior art, note that #12073 released the GIL for munmap. In a toy benchmark, I see the speedup you'd expect from doing this. Automerge-Triggered-By: GH:gvanrossum --- .../Library/2022-10-10-09-52-21.gh-issue-44098.okcqJt.rst | 1 + Modules/mmapmodule.c | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-10-09-52-21.gh-issue-44098.okcqJt.rst diff --git a/Misc/NEWS.d/next/Library/2022-10-10-09-52-21.gh-issue-44098.okcqJt.rst b/Misc/NEWS.d/next/Library/2022-10-10-09-52-21.gh-issue-44098.okcqJt.rst new file mode 100644 index 000000000000000..4efea4a7c4fccc9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-10-09-52-21.gh-issue-44098.okcqJt.rst @@ -0,0 +1 @@ +Release the GIL when creating :class:`mmap.mmap` objects on Unix. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 5c05aa738ef564e..fdce783fdec5e2d 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1318,9 +1318,9 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) } } - m_obj->data = mmap(NULL, map_size, - prot, flags, - fd, offset); + Py_BEGIN_ALLOW_THREADS + m_obj->data = mmap(NULL, map_size, prot, flags, fd, offset); + Py_END_ALLOW_THREADS if (devzero != -1) { close(devzero); From b399115ef18945f7526492225d72a512048ad20d Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Mon, 10 Oct 2022 20:26:08 -0700 Subject: [PATCH 25/28] gh-95756: Lazily created cached co_* attrs (GH-97791) --- Include/cpython/code.h | 9 ++- ...2-10-03-20-33-24.gh-issue-95756.SSmXlG.rst | 1 + Objects/codeobject.c | 77 ++++++++++++++++--- Objects/frameobject.c | 7 +- Tools/scripts/deepfreeze.py | 2 +- 5 files changed, 82 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2022-10-03-20-33-24.gh-issue-95756.SSmXlG.rst diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 7ce69022557af07..893ff934d3d8792 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -32,6 +32,13 @@ typedef uint16_t _Py_CODEUNIT; #define _Py_SET_OPCODE(word, opcode) \ do { ((unsigned char *)&(word))[0] = (opcode); } while (0) +typedef struct { + PyObject *_co_code; + PyObject *_co_varnames; + PyObject *_co_cellvars; + PyObject *_co_freevars; +} _PyCoCached; + // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are // defined in this macro: #define _PyCode_DEF(SIZE) { \ @@ -90,7 +97,7 @@ typedef uint16_t _Py_CODEUNIT; PyObject *co_qualname; /* unicode (qualname, for reference) */ \ PyObject *co_linetable; /* bytes object that holds location info */ \ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ - PyObject *_co_code; /* cached co_code object/attribute */ \ + _PyCoCached *_co_cached; /* cached co_* attributes */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ diff --git a/Misc/NEWS.d/next/C API/2022-10-03-20-33-24.gh-issue-95756.SSmXlG.rst b/Misc/NEWS.d/next/C API/2022-10-03-20-33-24.gh-issue-95756.SSmXlG.rst new file mode 100644 index 000000000000000..b5c78c16e86d18e --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-10-03-20-33-24.gh-issue-95756.SSmXlG.rst @@ -0,0 +1 @@ +Lazily create and cache ``co_`` attributes for better performance for code getters. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 14d1d00684aedfd..7a0080c08c7b547 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -150,7 +150,22 @@ validate_and_copy_tuple(PyObject *tup) return newtuple; } +static int +init_co_cached(PyCodeObject *self) { + if (self->_co_cached == NULL) { + self->_co_cached = PyMem_New(_PyCoCached, 1); + if (self->_co_cached == NULL) { + PyErr_NoMemory(); + return -1; + } + self->_co_cached->_co_code = NULL; + self->_co_cached->_co_cellvars = NULL; + self->_co_cached->_co_freevars = NULL; + self->_co_cached->_co_varnames = NULL; + } + return 0; +} /****************** * _PyCode_New() ******************/ @@ -336,7 +351,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) /* not set */ co->co_weakreflist = NULL; co->co_extra = NULL; - co->_co_code = NULL; + co->_co_cached = NULL; co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; co->_co_linearray_entry_size = 0; @@ -1384,10 +1399,31 @@ _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) * other PyCodeObject accessor functions ******************/ +static PyObject * +get_cached_locals(PyCodeObject *co, PyObject **cached_field, + _PyLocals_Kind kind, int num) +{ + assert(cached_field != NULL); + assert(co->_co_cached != NULL); + if (*cached_field != NULL) { + return Py_NewRef(*cached_field); + } + assert(*cached_field == NULL); + PyObject *varnames = get_localsplus_names(co, kind, num); + if (varnames == NULL) { + return NULL; + } + *cached_field = Py_NewRef(varnames); + return varnames; +} + PyObject * _PyCode_GetVarnames(PyCodeObject *co) { - return get_localsplus_names(co, CO_FAST_LOCAL, co->co_nlocals); + if (init_co_cached(co)) { + return NULL; + } + return get_cached_locals(co, &co->_co_cached->_co_varnames, CO_FAST_LOCAL, co->co_nlocals); } PyObject * @@ -1399,7 +1435,10 @@ PyCode_GetVarnames(PyCodeObject *code) PyObject * _PyCode_GetCellvars(PyCodeObject *co) { - return get_localsplus_names(co, CO_FAST_CELL, co->co_ncellvars); + if (init_co_cached(co)) { + return NULL; + } + return get_cached_locals(co, &co->_co_cached->_co_cellvars, CO_FAST_CELL, co->co_ncellvars); } PyObject * @@ -1411,7 +1450,10 @@ PyCode_GetCellvars(PyCodeObject *code) PyObject * _PyCode_GetFreevars(PyCodeObject *co) { - return get_localsplus_names(co, CO_FAST_FREE, co->co_nfreevars); + if (init_co_cached(co)) { + return NULL; + } + return get_cached_locals(co, &co->_co_cached->_co_freevars, CO_FAST_FREE, co->co_nfreevars); } PyObject * @@ -1437,8 +1479,11 @@ deopt_code(_Py_CODEUNIT *instructions, Py_ssize_t len) PyObject * _PyCode_GetCode(PyCodeObject *co) { - if (co->_co_code != NULL) { - return Py_NewRef(co->_co_code); + if (init_co_cached(co)) { + return NULL; + } + if (co->_co_cached->_co_code != NULL) { + return Py_NewRef(co->_co_cached->_co_code); } PyObject *code = PyBytes_FromStringAndSize((const char *)_PyCode_CODE(co), _PyCode_NBYTES(co)); @@ -1446,8 +1491,8 @@ _PyCode_GetCode(PyCodeObject *co) return NULL; } deopt_code((_Py_CODEUNIT *)PyBytes_AS_STRING(code), Py_SIZE(co)); - assert(co->_co_code == NULL); - co->_co_code = Py_NewRef(code); + assert(co->_co_cached->_co_code == NULL); + co->_co_cached->_co_code = Py_NewRef(code); return code; } @@ -1606,7 +1651,13 @@ code_dealloc(PyCodeObject *co) Py_XDECREF(co->co_qualname); Py_XDECREF(co->co_linetable); Py_XDECREF(co->co_exceptiontable); - Py_XDECREF(co->_co_code); + if (co->_co_cached != NULL) { + Py_XDECREF(co->_co_cached->_co_code); + Py_XDECREF(co->_co_cached->_co_cellvars); + Py_XDECREF(co->_co_cached->_co_freevars); + Py_XDECREF(co->_co_cached->_co_varnames); + PyMem_Free(co->_co_cached); + } if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject*)co); } @@ -2181,7 +2232,13 @@ _PyStaticCode_Dealloc(PyCodeObject *co) deopt_code(_PyCode_CODE(co), Py_SIZE(co)); co->co_warmup = QUICKENING_INITIAL_WARMUP_VALUE; PyMem_Free(co->co_extra); - Py_CLEAR(co->_co_code); + if (co->_co_cached != NULL) { + Py_CLEAR(co->_co_cached->_co_code); + Py_CLEAR(co->_co_cached->_co_cellvars); + Py_CLEAR(co->_co_cached->_co_freevars); + Py_CLEAR(co->_co_cached->_co_varnames); + PyMem_Free(co->_co_cached); + } co->co_extra = NULL; if (co->co_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *)co); diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 6a51a946ef35f0d..bd1608e0d75a6e3 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -645,9 +645,12 @@ add_load_fast_null_checks(PyCodeObject *co) } i += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; } - if (changed) { + if (changed && co->_co_cached != NULL) { // invalidate cached co_code object - Py_CLEAR(co->_co_code); + Py_CLEAR(co->_co_cached->_co_code); + Py_CLEAR(co->_co_cached->_co_cellvars); + Py_CLEAR(co->_co_cached->_co_freevars); + Py_CLEAR(co->_co_cached->_co_varnames); } } diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index d9c6030fc17c070..28ac2b12f92fa78 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -276,7 +276,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_name = {co_name},") self.write(f".co_qualname = {co_qualname},") self.write(f".co_linetable = {co_linetable},") - self.write(f"._co_code = NULL,") + self.write(f"._co_cached = NULL,") self.write("._co_linearray = NULL,") self.write(f".co_code_adaptive = {co_code_adaptive},") for i, op in enumerate(code.co_code[::2]): From e0ae9ddffe0a708d0d3f5b8cc10488d466fc43c4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Oct 2022 10:07:57 +0200 Subject: [PATCH 26/28] gh-97669: Remove outdated example scripts (#97675) (#98167) Remove outdated example scripts of the Tools/scripts/ directory: * gprof2html.py * md5sum.py * nm2def.py * pathfix.py * win_add2path.py Remove test_gprof2html, test_md5sum and test_pathfix of test_tools. --- Lib/test/test_tools/test_gprof2html.py | 35 ---- Lib/test/test_tools/test_md5sum.py | 78 --------- Lib/test/test_tools/test_pathfix.py | 131 -------------- Lib/test/test_tools/test_sundry.py | 11 +- PCbuild/lib.pyproj | 2 - Tools/scripts/README | 9 - Tools/scripts/gprof2html.py | 87 ---------- Tools/scripts/md5sum.py | 93 ---------- Tools/scripts/nm2def.py | 104 ------------ Tools/scripts/pathfix.py | 225 ------------------------- Tools/scripts/win_add2path.py | 58 ------- 11 files changed, 2 insertions(+), 831 deletions(-) delete mode 100644 Lib/test/test_tools/test_gprof2html.py delete mode 100644 Lib/test/test_tools/test_md5sum.py delete mode 100644 Lib/test/test_tools/test_pathfix.py delete mode 100755 Tools/scripts/gprof2html.py delete mode 100755 Tools/scripts/md5sum.py delete mode 100755 Tools/scripts/nm2def.py delete mode 100755 Tools/scripts/pathfix.py delete mode 100644 Tools/scripts/win_add2path.py diff --git a/Lib/test/test_tools/test_gprof2html.py b/Lib/test/test_tools/test_gprof2html.py deleted file mode 100644 index 7cceb8faf8e5f4a..000000000000000 --- a/Lib/test/test_tools/test_gprof2html.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Tests for the gprof2html script in the Tools directory.""" - -import os -import sys -import unittest -from unittest import mock -import tempfile - -from test.test_tools import skip_if_missing, import_tool - -skip_if_missing() - -class Gprof2htmlTests(unittest.TestCase): - - def setUp(self): - self.gprof = import_tool('gprof2html') - oldargv = sys.argv - def fixup(): - sys.argv = oldargv - self.addCleanup(fixup) - sys.argv = [] - - def test_gprof(self): - # Issue #14508: this used to fail with a NameError. - with mock.patch.object(self.gprof, 'webbrowser') as wmock, \ - tempfile.TemporaryDirectory() as tmpdir: - fn = os.path.join(tmpdir, 'abc') - open(fn, 'wb').close() - sys.argv = ['gprof2html', fn] - self.gprof.main() - self.assertTrue(wmock.open.called) - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_tools/test_md5sum.py b/Lib/test/test_tools/test_md5sum.py deleted file mode 100644 index 710461f8819754a..000000000000000 --- a/Lib/test/test_tools/test_md5sum.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Tests for the md5sum script in the Tools directory.""" - -import os -import unittest -from test.support import os_helper -from test.support import hashlib_helper -from test.support.script_helper import assert_python_ok, assert_python_failure - -from test.test_tools import scriptsdir, skip_if_missing - -skip_if_missing() - -@hashlib_helper.requires_hashdigest('md5', openssl=True) -class MD5SumTests(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.script = os.path.join(scriptsdir, 'md5sum.py') - os.mkdir(os_helper.TESTFN_ASCII) - cls.fodder = os.path.join(os_helper.TESTFN_ASCII, 'md5sum.fodder') - with open(cls.fodder, 'wb') as f: - f.write(b'md5sum\r\ntest file\r\n') - cls.fodder_md5 = b'd38dae2eb1ab346a292ef6850f9e1a0d' - cls.fodder_textmode_md5 = b'a8b07894e2ca3f2a4c3094065fa6e0a5' - - @classmethod - def tearDownClass(cls): - os_helper.rmtree(os_helper.TESTFN_ASCII) - - def test_noargs(self): - rc, out, err = assert_python_ok(self.script) - self.assertEqual(rc, 0) - self.assertTrue( - out.startswith(b'd41d8cd98f00b204e9800998ecf8427e ')) - self.assertFalse(err) - - def test_checksum_fodder(self): - rc, out, err = assert_python_ok(self.script, self.fodder) - self.assertEqual(rc, 0) - self.assertTrue(out.startswith(self.fodder_md5)) - for part in self.fodder.split(os.path.sep): - self.assertIn(part.encode(), out) - self.assertFalse(err) - - def test_dash_l(self): - rc, out, err = assert_python_ok(self.script, '-l', self.fodder) - self.assertEqual(rc, 0) - self.assertIn(self.fodder_md5, out) - parts = self.fodder.split(os.path.sep) - self.assertIn(parts[-1].encode(), out) - self.assertNotIn(parts[-2].encode(), out) - - def test_dash_t(self): - rc, out, err = assert_python_ok(self.script, '-t', self.fodder) - self.assertEqual(rc, 0) - self.assertTrue(out.startswith(self.fodder_textmode_md5)) - self.assertNotIn(self.fodder_md5, out) - - def test_dash_s(self): - rc, out, err = assert_python_ok(self.script, '-s', '512', self.fodder) - self.assertEqual(rc, 0) - self.assertIn(self.fodder_md5, out) - - def test_multiple_files(self): - rc, out, err = assert_python_ok(self.script, self.fodder, self.fodder) - self.assertEqual(rc, 0) - lines = out.splitlines() - self.assertEqual(len(lines), 2) - self.assertEqual(*lines) - - def test_usage(self): - rc, out, err = assert_python_failure(self.script, '-h') - self.assertEqual(rc, 2) - self.assertEqual(out, b'') - self.assertGreater(err, b'') - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py deleted file mode 100644 index aa754bc479b4682..000000000000000 --- a/Lib/test/test_tools/test_pathfix.py +++ /dev/null @@ -1,131 +0,0 @@ -import os -import subprocess -import sys -import unittest -from test.support import os_helper -from test.test_tools import scriptsdir, skip_if_missing - - -# need Tools/script/ directory: skip if run on Python installed on the system -skip_if_missing() - - -class TestPathfixFunctional(unittest.TestCase): - script = os.path.join(scriptsdir, 'pathfix.py') - - def setUp(self): - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - - def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr='', - directory=''): - if directory: - # bpo-38347: Test filename should contain lowercase, uppercase, - # "-", "_" and digits. - filename = os.path.join(directory, 'script-A_1.py') - pathfix_arg = directory - else: - filename = os_helper.TESTFN - pathfix_arg = filename - - with open(filename, 'w', encoding='utf8') as f: - f.write(f'{shebang}\n' + 'print("Hello world")\n') - - encoding = sys.getfilesystemencoding() - proc = subprocess.run( - [sys.executable, self.script, - *pathfix_flags, '-n', pathfix_arg], - env={**os.environ, 'PYTHONIOENCODING': encoding}, - capture_output=True) - - if stdout == '' and proc.returncode == 0: - stdout = f'{filename}: updating\n' - self.assertEqual(proc.returncode, exitcode, proc) - self.assertEqual(proc.stdout.decode(encoding), stdout.replace('\n', os.linesep), proc) - self.assertEqual(proc.stderr.decode(encoding), stderr.replace('\n', os.linesep), proc) - - with open(filename, 'r', encoding='utf8') as f: - output = f.read() - - lines = output.split('\n') - self.assertEqual(lines[1:], ['print("Hello world")', '']) - new_shebang = lines[0] - - if proc.returncode != 0: - self.assertEqual(shebang, new_shebang) - - return new_shebang - - def test_recursive(self): - tmpdir = os_helper.TESTFN + '.d' - self.addCleanup(os_helper.rmtree, tmpdir) - os.mkdir(tmpdir) - expected_stderr = f"recursedown('{os.path.basename(tmpdir)}')\n" - self.assertEqual( - self.pathfix( - '#! /usr/bin/env python', - ['-i', '/usr/bin/python3'], - directory=tmpdir, - stderr=expected_stderr), - '#! /usr/bin/python3') - - def test_pathfix(self): - self.assertEqual( - self.pathfix( - '#! /usr/bin/env python', - ['-i', '/usr/bin/python3']), - '#! /usr/bin/python3') - self.assertEqual( - self.pathfix( - '#! /usr/bin/env python -R', - ['-i', '/usr/bin/python3']), - '#! /usr/bin/python3') - - def test_pathfix_keeping_flags(self): - self.assertEqual( - self.pathfix( - '#! /usr/bin/env python -R', - ['-i', '/usr/bin/python3', '-k']), - '#! /usr/bin/python3 -R') - self.assertEqual( - self.pathfix( - '#! /usr/bin/env python', - ['-i', '/usr/bin/python3', '-k']), - '#! /usr/bin/python3') - - def test_pathfix_adding_flag(self): - self.assertEqual( - self.pathfix( - '#! /usr/bin/env python', - ['-i', '/usr/bin/python3', '-a', 's']), - '#! /usr/bin/python3 -s') - self.assertEqual( - self.pathfix( - '#! /usr/bin/env python -S', - ['-i', '/usr/bin/python3', '-a', 's']), - '#! /usr/bin/python3 -s') - self.assertEqual( - self.pathfix( - '#! /usr/bin/env python -V', - ['-i', '/usr/bin/python3', '-a', 'v', '-k']), - '#! /usr/bin/python3 -vV') - self.assertEqual( - self.pathfix( - '#! /usr/bin/env python', - ['-i', '/usr/bin/python3', '-a', 'Rs']), - '#! /usr/bin/python3 -Rs') - self.assertEqual( - self.pathfix( - '#! /usr/bin/env python -W default', - ['-i', '/usr/bin/python3', '-a', 's', '-k']), - '#! /usr/bin/python3 -sW default') - - def test_pathfix_adding_errors(self): - self.pathfix( - '#! /usr/bin/env python -E', - ['-i', '/usr/bin/python3', '-a', 'W default', '-k'], - exitcode=2, - stderr="-a option doesn't support whitespaces") - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py index ed03c2f75a3aa5d..04e38acdb3484e7 100644 --- a/Lib/test/test_tools/test_sundry.py +++ b/Lib/test/test_tools/test_sundry.py @@ -19,15 +19,13 @@ class TestSundryScripts(unittest.TestCase): # added for a script it should be added to the allowlist below. # scripts that have independent tests. - allowlist = ['reindent', 'pdeps', 'gprof2html', 'md5sum'] + allowlist = ['reindent'] # scripts that can't be imported without running denylist = ['make_ctype'] - # scripts that use windows-only modules - windows_only = ['win_add2path'] # denylisted for other reasons other = ['2to3'] - skiplist = denylist + allowlist + windows_only + other + skiplist = denylist + allowlist + other def test_sundry(self): old_modules = import_helper.modules_setup() @@ -45,11 +43,6 @@ def test_sundry(self): # Unload all modules loaded in this test import_helper.modules_cleanup(*old_modules) - @unittest.skipIf(sys.platform != "win32", "Windows-only test") - def test_sundry_windows(self): - for name in self.windows_only: - import_tool(name) - if __name__ == '__main__': unittest.main() diff --git a/PCbuild/lib.pyproj b/PCbuild/lib.pyproj index 9934dc577bf36a3..daa2021191027d2 100644 --- a/PCbuild/lib.pyproj +++ b/PCbuild/lib.pyproj @@ -1334,9 +1334,7 @@ - - diff --git a/Tools/scripts/README b/Tools/scripts/README index 70ea5f4cd0feaef..2fcceccc075e79e 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -4,20 +4,11 @@ useful while building, extending or managing Python. 2to3 Main script for running the 2to3 conversion tool abitype.py Converts a C file to use the PEP 384 type definition API combinerefs.py A helper for analyzing PYTHONDUMPREFS output -diff.py Print file diffs in context, unified, or ndiff formats -gprof2html.py Transform gprof(1) output into useful HTML idle3 Main program to start IDLE -md5sum.py Print MD5 checksums of argument files -ndiff.py Intelligent diff between text files (Tim Peters) -nm2def.py Create a template for PC/python_nt.def (Marc Lemburg) -parseentities.py Utility for parsing HTML entity definitions parse_html5_entities.py Utility for parsing HTML5 entity definitions patchcheck.py Perform common checks and cleanup before committing -pathfix.py Change #!/usr/local/bin/python into something else -ptags.py Create vi tags file for Python modules pydoc3 Python documentation browser reindent.py Change .py files to use 4-space indents run_tests.py Run the test suite with more sensible default options stable_abi.py Stable ABI checks and file generators. untabify.py Replace tabs with spaces in argument files -win_add2path.py Add Python to the search path on Windows diff --git a/Tools/scripts/gprof2html.py b/Tools/scripts/gprof2html.py deleted file mode 100755 index bf0530ef3e4337a..000000000000000 --- a/Tools/scripts/gprof2html.py +++ /dev/null @@ -1,87 +0,0 @@ -#! /usr/bin/env python3 - -"""Transform gprof(1) output into useful HTML.""" - -import html -import os -import re -import sys -import webbrowser - -header = """\ - - - gprof output (%s) - - -
-"""
-
-trailer = """\
-
- - -""" - -def add_escapes(filename): - with open(filename, encoding="utf-8") as fp: - for line in fp: - yield html.escape(line) - -def gprof2html(input, output, filename): - output.write(header % filename) - for line in input: - output.write(line) - if line.startswith(" time"): - break - labels = {} - for line in input: - m = re.match(r"(.* )(\w+)\n", line) - if not m: - output.write(line) - break - stuff, fname = m.group(1, 2) - labels[fname] = fname - output.write('%s%s\n' % - (stuff, fname, fname, fname)) - for line in input: - output.write(line) - if line.startswith("index % time"): - break - for line in input: - m = re.match(r"(.* )(\w+)(( <cycle.*>)? \[\d+\])\n", line) - if not m: - output.write(line) - if line.startswith("Index by function name"): - break - continue - prefix, fname, suffix = m.group(1, 2, 3) - if fname not in labels: - output.write(line) - continue - if line.startswith("["): - output.write('%s%s%s\n' % - (prefix, fname, fname, fname, suffix)) - else: - output.write('%s%s%s\n' % - (prefix, fname, fname, suffix)) - for line in input: - for part in re.findall(r"(\w+(?:\.c)?|\W+)", line): - if part in labels: - part = '%s' % (part, part) - output.write(part) - output.write(trailer) - - -def main(): - filename = "gprof.out" - if sys.argv[1:]: - filename = sys.argv[1] - outputfilename = filename + ".html" - input = add_escapes(filename) - with open(outputfilename, "w", encoding="utf-8") as output: - gprof2html(input, output, filename) - webbrowser.open("file:" + os.path.abspath(outputfilename)) - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/md5sum.py b/Tools/scripts/md5sum.py deleted file mode 100755 index f910576377aa3d0..000000000000000 --- a/Tools/scripts/md5sum.py +++ /dev/null @@ -1,93 +0,0 @@ -#! /usr/bin/env python3 - -"""Python utility to print MD5 checksums of argument files. -""" - - -bufsize = 8096 -fnfilter = None -rmode = 'rb' - -usage = """ -usage: md5sum.py [-b] [-t] [-l] [-s bufsize] [file ...] --b : read files in binary mode (default) --t : read files in text mode (you almost certainly don't want this!) --l : print last pathname component only --s bufsize: read buffer size (default %d) -file ... : files to sum; '-' or no files means stdin -""" % bufsize - -import io -import sys -import os -import getopt -from hashlib import md5 - -def sum(*files): - sts = 0 - if files and isinstance(files[-1], io.IOBase): - out, files = files[-1], files[:-1] - else: - out = sys.stdout - if len(files) == 1 and not isinstance(files[0], str): - files = files[0] - for f in files: - if isinstance(f, str): - if f == '-': - sts = printsumfp(sys.stdin, '', out) or sts - else: - sts = printsum(f, out) or sts - else: - sts = sum(f, out) or sts - return sts - -def printsum(filename, out=sys.stdout): - try: - fp = open(filename, rmode) - except IOError as msg: - sys.stderr.write('%s: Can\'t open: %s\n' % (filename, msg)) - return 1 - with fp: - if fnfilter: - filename = fnfilter(filename) - sts = printsumfp(fp, filename, out) - return sts - -def printsumfp(fp, filename, out=sys.stdout): - m = md5() - try: - while 1: - data = fp.read(bufsize) - if not data: - break - if isinstance(data, str): - data = data.encode(fp.encoding) - m.update(data) - except IOError as msg: - sys.stderr.write('%s: I/O error: %s\n' % (filename, msg)) - return 1 - out.write('%s %s\n' % (m.hexdigest(), filename)) - return 0 - -def main(args = sys.argv[1:], out=sys.stdout): - global fnfilter, rmode, bufsize - try: - opts, args = getopt.getopt(args, 'blts:') - except getopt.error as msg: - sys.stderr.write('%s: %s\n%s' % (sys.argv[0], msg, usage)) - return 2 - for o, a in opts: - if o == '-l': - fnfilter = os.path.basename - elif o == '-b': - rmode = 'rb' - elif o == '-t': - rmode = 'r' - elif o == '-s': - bufsize = int(a) - if not args: - args = ['-'] - return sum(args, out) - -if __name__ == '__main__' or __name__ == sys.argv[0]: - sys.exit(main(sys.argv[1:], sys.stdout)) diff --git a/Tools/scripts/nm2def.py b/Tools/scripts/nm2def.py deleted file mode 100755 index a885ebd6fecc2ff..000000000000000 --- a/Tools/scripts/nm2def.py +++ /dev/null @@ -1,104 +0,0 @@ -#! /usr/bin/env python3 -"""nm2def.py - -Helpers to extract symbols from Unix libs and auto-generate -Windows definition files from them. Depends on nm(1). Tested -on Linux and Solaris only (-p option to nm is for Solaris only). - -By Marc-Andre Lemburg, Aug 1998. - -Additional notes: the output of nm is supposed to look like this: - -acceler.o: -000001fd T PyGrammar_AddAccelerators - U PyGrammar_FindDFA -00000237 T PyGrammar_RemoveAccelerators - U _IO_stderr_ - U exit - U fprintf - U free - U malloc - U printf - -grammar1.o: -00000000 T PyGrammar_FindDFA -00000034 T PyGrammar_LabelRepr - U _PyParser_TokenNames - U abort - U printf - U sprintf - -... - -Even if this isn't the default output of your nm, there is generally an -option to produce this format (since it is the original v7 Unix format). - -""" -import os, sys - -PYTHONLIB = 'libpython%d.%d.a' % sys.version_info[:2] -PC_PYTHONLIB = 'Python%d%d.dll' % sys.version_info[:2] -NM = 'nm -p -g %s' # For Linux, use "nm -g %s" - -def symbols(lib=PYTHONLIB,types=('T','C','D')): - - with os.popen(NM % lib) as pipe: - lines = pipe.readlines() - lines = [s.strip() for s in lines] - symbols = {} - for line in lines: - if len(line) == 0 or ':' in line: - continue - items = line.split() - if len(items) != 3: - continue - address, type, name = items - if type not in types: - continue - symbols[name] = address,type - return symbols - -def export_list(symbols): - - data = [] - code = [] - for name,(addr,type) in symbols.items(): - if type in ('C','D'): - data.append('\t'+name) - else: - code.append('\t'+name) - data.sort() - data.append('') - code.sort() - return ' DATA\n'.join(data)+'\n'+'\n'.join(code) - -# Definition file template -DEF_TEMPLATE = """\ -EXPORTS -%s -""" - -# Special symbols that have to be included even though they don't -# pass the filter -SPECIALS = ( - ) - -def filter_Python(symbols,specials=SPECIALS): - - for name in list(symbols.keys()): - if name[:2] == 'Py' or name[:3] == '_Py': - pass - elif name not in specials: - del symbols[name] - -def main(): - - s = symbols(PYTHONLIB) - filter_Python(s) - exports = export_list(s) - f = sys.stdout # open('PC/python_nt.def','w') - f.write(DEF_TEMPLATE % (exports)) - # f.close() - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py deleted file mode 100755 index f957b11547179d4..000000000000000 --- a/Tools/scripts/pathfix.py +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env python3 - -# Change the #! line (shebang) occurring in Python scripts. The new interpreter -# pathname must be given with a -i option. -# -# Command line arguments are files or directories to be processed. -# Directories are searched recursively for files whose name looks -# like a python module. -# Symbolic links are always ignored (except as explicit directory -# arguments). -# The original file is kept as a back-up (with a "~" attached to its name), -# -n flag can be used to disable this. - -# Sometimes you may find shebangs with flags such as `#! /usr/bin/env python -si`. -# Normally, pathfix overwrites the entire line, including the flags. -# To change interpreter and keep flags from the original shebang line, use -k. -# If you want to keep flags and add to them one single literal flag, use option -a. - - -# Undoubtedly you can do this using find and sed or perl, but this is -# a nice example of Python code that recurses down a directory tree -# and uses regular expressions. Also note several subtleties like -# preserving the file's mode and avoiding to even write a temp file -# when no changes are needed for a file. -# -# NB: by changing only the function fixfile() you can turn this -# into a program for a different change to Python programs... - -import sys -import os -from stat import * -import getopt - -err = sys.stderr.write -dbg = err -rep = sys.stdout.write - -new_interpreter = None -preserve_timestamps = False -create_backup = True -keep_flags = False -add_flags = b'' - - -def main(): - global new_interpreter - global preserve_timestamps - global create_backup - global keep_flags - global add_flags - - usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' % - sys.argv[0]) - try: - opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn') - except getopt.error as msg: - err(str(msg) + '\n') - err(usage) - sys.exit(2) - for o, a in opts: - if o == '-i': - new_interpreter = a.encode() - if o == '-p': - preserve_timestamps = True - if o == '-n': - create_backup = False - if o == '-k': - keep_flags = True - if o == '-a': - add_flags = a.encode() - if b' ' in add_flags: - err("-a option doesn't support whitespaces") - sys.exit(2) - if not new_interpreter or not new_interpreter.startswith(b'/') or \ - not args: - err('-i option or file-or-directory missing\n') - err(usage) - sys.exit(2) - bad = 0 - for arg in args: - if os.path.isdir(arg): - if recursedown(arg): bad = 1 - elif os.path.islink(arg): - err(arg + ': will not process symbolic links\n') - bad = 1 - else: - if fix(arg): bad = 1 - sys.exit(bad) - - -def ispython(name): - return name.endswith('.py') - - -def recursedown(dirname): - dbg('recursedown(%r)\n' % (dirname,)) - bad = 0 - try: - names = os.listdir(dirname) - except OSError as msg: - err('%s: cannot list directory: %r\n' % (dirname, msg)) - return 1 - names.sort() - subdirs = [] - for name in names: - if name in (os.curdir, os.pardir): continue - fullname = os.path.join(dirname, name) - if os.path.islink(fullname): pass - elif os.path.isdir(fullname): - subdirs.append(fullname) - elif ispython(name): - if fix(fullname): bad = 1 - for fullname in subdirs: - if recursedown(fullname): bad = 1 - return bad - - -def fix(filename): -## dbg('fix(%r)\n' % (filename,)) - try: - f = open(filename, 'rb') - except IOError as msg: - err('%s: cannot open: %r\n' % (filename, msg)) - return 1 - with f: - line = f.readline() - fixed = fixline(line) - if line == fixed: - rep(filename+': no change\n') - return - head, tail = os.path.split(filename) - tempname = os.path.join(head, '@' + tail) - try: - g = open(tempname, 'wb') - except IOError as msg: - err('%s: cannot create: %r\n' % (tempname, msg)) - return 1 - with g: - rep(filename + ': updating\n') - g.write(fixed) - BUFSIZE = 8*1024 - while 1: - buf = f.read(BUFSIZE) - if not buf: break - g.write(buf) - - # Finishing touch -- move files - - mtime = None - atime = None - # First copy the file's mode to the temp file - try: - statbuf = os.stat(filename) - mtime = statbuf.st_mtime - atime = statbuf.st_atime - os.chmod(tempname, statbuf[ST_MODE] & 0o7777) - except OSError as msg: - err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) - # Then make a backup of the original file as filename~ - if create_backup: - try: - os.rename(filename, filename + '~') - except OSError as msg: - err('%s: warning: backup failed (%r)\n' % (filename, msg)) - else: - try: - os.remove(filename) - except OSError as msg: - err('%s: warning: removing failed (%r)\n' % (filename, msg)) - # Now move the temp file to the original file - try: - os.rename(tempname, filename) - except OSError as msg: - err('%s: rename failed (%r)\n' % (filename, msg)) - return 1 - if preserve_timestamps: - if atime and mtime: - try: - os.utime(filename, (atime, mtime)) - except OSError as msg: - err('%s: reset of timestamp failed (%r)\n' % (filename, msg)) - return 1 - # Return success - return 0 - - -def parse_shebang(shebangline): - shebangline = shebangline.rstrip(b'\n') - start = shebangline.find(b' -') - if start == -1: - return b'' - return shebangline[start:] - - -def populate_flags(shebangline): - old_flags = b'' - if keep_flags: - old_flags = parse_shebang(shebangline) - if old_flags: - old_flags = old_flags[2:] - if not (old_flags or add_flags): - return b'' - # On Linux, the entire string following the interpreter name - # is passed as a single argument to the interpreter. - # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s" - # so shebang should have single '-' where flags are given and - # flag might need argument for that reasons adding new flags is - # between '-' and original flags - # e.g. #! /usr/bin/python3 -sW Error - return b' -' + add_flags + old_flags - - -def fixline(line): - if not line.startswith(b'#!'): - return line - - if b"python" not in line: - return line - - flags = populate_flags(line) - return b'#! ' + new_interpreter + flags + b'\n' - - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/win_add2path.py b/Tools/scripts/win_add2path.py deleted file mode 100644 index 1c9aedc5ed8dcae..000000000000000 --- a/Tools/scripts/win_add2path.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Add Python to the search path on Windows - -This is a simple script to add Python to the Windows search path. It -modifies the current user (HKCU) tree of the registry. - -Copyright (c) 2008 by Christian Heimes -Licensed to PSF under a Contributor Agreement. -""" - -import sys -import site -import os -import winreg - -HKCU = winreg.HKEY_CURRENT_USER -ENV = "Environment" -PATH = "PATH" -DEFAULT = "%PATH%" - -def modify(): - pythonpath = os.path.dirname(os.path.normpath(sys.executable)) - scripts = os.path.join(pythonpath, "Scripts") - appdata = os.environ["APPDATA"] - if hasattr(site, "USER_SITE"): - usersite = site.USER_SITE.replace(appdata, "%APPDATA%") - userpath = os.path.dirname(usersite) - userscripts = os.path.join(userpath, "Scripts") - else: - userscripts = None - - with winreg.CreateKey(HKCU, ENV) as key: - try: - envpath = winreg.QueryValueEx(key, PATH)[0] - except OSError: - envpath = DEFAULT - - paths = [envpath] - for path in (pythonpath, scripts, userscripts): - if path and path not in envpath and os.path.isdir(path): - paths.append(path) - - envpath = os.pathsep.join(paths) - winreg.SetValueEx(key, PATH, 0, winreg.REG_EXPAND_SZ, envpath) - return paths, envpath - -def main(): - paths, envpath = modify() - if len(paths) > 1: - print("Path(s) added:") - print('\n'.join(paths[1:])) - else: - print("No path was added") - print("\nPATH is now:\n%s\n" % envpath) - print("Expanded:") - print(winreg.ExpandEnvironmentStrings(envpath)) - -if __name__ == '__main__': - main() From f0a680007f345a46bcd187b952a929c7068c1da8 Mon Sep 17 00:00:00 2001 From: Stanley <46876382+slateny@users.noreply.github.com> Date: Tue, 11 Oct 2022 02:27:49 -0700 Subject: [PATCH 27/28] gh-71616: Add note to warn against general translation of saxutils.escape() (#93450) * Add note to warn against general translation of saxutils.escape() * Use more direct wording --- Doc/library/xml.sax.utils.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index e46fefdf997510d..ab4606bcf9fe6ca 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -25,6 +25,11 @@ or as base classes. replaced with its corresponding value. The characters ``'&'``, ``'<'`` and ``'>'`` are always escaped, even if *entities* is provided. + .. note:: + + This function should only be used to escape characters that + can't be used directly in XML. Do not use this function as a general + string translation function. .. function:: unescape(data, entities={}) From 454a6d61bc569b45cad163601d745bb304b53bde Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Oct 2022 13:18:53 +0200 Subject: [PATCH 28/28] gh-97669: Remove abitype.py and pep384_macrocheck.py (#98165) Remove abitype.py and pep384_macrocheck.py scripts of Tools/scripts/. --- Tools/scripts/README | 1 - Tools/scripts/abitype.py | 202 ----------------------------- Tools/scripts/pep384_macrocheck.py | 148 --------------------- 3 files changed, 351 deletions(-) delete mode 100755 Tools/scripts/abitype.py delete mode 100644 Tools/scripts/pep384_macrocheck.py diff --git a/Tools/scripts/README b/Tools/scripts/README index 2fcceccc075e79e..9943d4c42fc4280 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -2,7 +2,6 @@ This directory contains a collection of executable Python scripts that are useful while building, extending or managing Python. 2to3 Main script for running the 2to3 conversion tool -abitype.py Converts a C file to use the PEP 384 type definition API combinerefs.py A helper for analyzing PYTHONDUMPREFS output idle3 Main program to start IDLE parse_html5_entities.py Utility for parsing HTML5 entity definitions diff --git a/Tools/scripts/abitype.py b/Tools/scripts/abitype.py deleted file mode 100755 index d6a74a1fe64117c..000000000000000 --- a/Tools/scripts/abitype.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python3 -# This script converts a C file to use the PEP 384 type definition API -# Usage: abitype.py < old_code > new_code -import re, sys - -###### Replacement of PyTypeObject static instances ############## - -# classify each token, giving it a one-letter code: -# S: static -# T: PyTypeObject -# I: ident -# W: whitespace -# =, {, }, ; : themselves -def classify(): - res = [] - for t,v in tokens: - if t == 'other' and v in "={};": - res.append(v) - elif t == 'ident': - if v == 'PyTypeObject': - res.append('T') - elif v == 'static': - res.append('S') - else: - res.append('I') - elif t == 'ws': - res.append('W') - else: - res.append('.') - return ''.join(res) - -# Obtain a list of fields of a PyTypeObject, in declaration order, -# skipping ob_base -# All comments are dropped from the variable (which are typically -# just the slot names, anyway), and information is discarded whether -# the original type was static. -def get_fields(start, real_end): - pos = start - # static? - if tokens[pos][1] == 'static': - pos += 2 - # PyTypeObject - pos += 2 - # name - name = tokens[pos][1] - pos += 1 - while tokens[pos][1] != '{': - pos += 1 - pos += 1 - # PyVarObject_HEAD_INIT - while tokens[pos][0] in ('ws', 'comment'): - pos += 1 - if tokens[pos][1] != 'PyVarObject_HEAD_INIT': - raise Exception('%s has no PyVarObject_HEAD_INIT' % name) - while tokens[pos][1] != ')': - pos += 1 - pos += 1 - # field definitions: various tokens, comma-separated - fields = [] - while True: - while tokens[pos][0] in ('ws', 'comment'): - pos += 1 - end = pos - while tokens[end][1] not in ',}': - if tokens[end][1] == '(': - nesting = 1 - while nesting: - end += 1 - if tokens[end][1] == '(': nesting+=1 - if tokens[end][1] == ')': nesting-=1 - end += 1 - assert end < real_end - # join field, excluding separator and trailing ws - end1 = end-1 - while tokens[end1][0] in ('ws', 'comment'): - end1 -= 1 - fields.append(''.join(t[1] for t in tokens[pos:end1+1])) - if tokens[end][1] == '}': - break - pos = end+1 - return name, fields - -# List of type slots as of Python 3.2, omitting ob_base -typeslots = [ - 'tp_name', - 'tp_basicsize', - 'tp_itemsize', - 'tp_dealloc', - 'tp_print', - 'tp_getattr', - 'tp_setattr', - 'tp_reserved', - 'tp_repr', - 'tp_as_number', - 'tp_as_sequence', - 'tp_as_mapping', - 'tp_hash', - 'tp_call', - 'tp_str', - 'tp_getattro', - 'tp_setattro', - 'tp_as_buffer', - 'tp_flags', - 'tp_doc', - 'tp_traverse', - 'tp_clear', - 'tp_richcompare', - 'tp_weaklistoffset', - 'tp_iter', - 'iternextfunc', - 'tp_methods', - 'tp_members', - 'tp_getset', - 'tp_base', - 'tp_dict', - 'tp_descr_get', - 'tp_descr_set', - 'tp_dictoffset', - 'tp_init', - 'tp_alloc', - 'tp_new', - 'tp_free', - 'tp_is_gc', - 'tp_bases', - 'tp_mro', - 'tp_cache', - 'tp_subclasses', - 'tp_weaklist', - 'tp_del', - 'tp_version_tag', -] - -# Generate a PyType_Spec definition -def make_slots(name, fields): - res = [] - res.append('static PyType_Slot %s_slots[] = {' % name) - # defaults for spec - spec = { 'tp_itemsize':'0' } - for i, val in enumerate(fields): - if val.endswith('0'): - continue - if typeslots[i] in ('tp_name', 'tp_doc', 'tp_basicsize', - 'tp_itemsize', 'tp_flags'): - spec[typeslots[i]] = val - continue - res.append(' {Py_%s, %s},' % (typeslots[i], val)) - res.append('};') - res.append('static PyType_Spec %s_spec = {' % name) - res.append(' %s,' % spec['tp_name']) - res.append(' %s,' % spec['tp_basicsize']) - res.append(' %s,' % spec['tp_itemsize']) - res.append(' %s,' % spec['tp_flags']) - res.append(' %s_slots,' % name) - res.append('};\n') - return '\n'.join(res) - - -if __name__ == '__main__': - - ############ Simplistic C scanner ################################## - tokenizer = re.compile( - r"(?P#.*\n)" - r"|(?P/\*.*?\*/)" - r"|(?P[a-zA-Z_][a-zA-Z0-9_]*)" - r"|(?P[ \t\n]+)" - r"|(?P.)", - re.MULTILINE) - - tokens = [] - source = sys.stdin.read() - pos = 0 - while pos != len(source): - m = tokenizer.match(source, pos) - tokens.append([m.lastgroup, m.group()]) - pos += len(tokens[-1][1]) - if tokens[-1][0] == 'preproc': - # continuation lines are considered - # only in preprocess statements - while tokens[-1][1].endswith('\\\n'): - nl = source.find('\n', pos) - if nl == -1: - line = source[pos:] - else: - line = source[pos:nl+1] - tokens[-1][1] += line - pos += len(line) - - # Main loop: replace all static PyTypeObjects until - # there are none left. - while 1: - c = classify() - m = re.search('(SW)?TWIW?=W?{.*?};', c) - if not m: - break - start = m.start() - end = m.end() - name, fields = get_fields(start, end) - tokens[start:end] = [('',make_slots(name, fields))] - - # Output result to stdout - for t, v in tokens: - sys.stdout.write(v) diff --git a/Tools/scripts/pep384_macrocheck.py b/Tools/scripts/pep384_macrocheck.py deleted file mode 100644 index ab9dd7c972aab5c..000000000000000 --- a/Tools/scripts/pep384_macrocheck.py +++ /dev/null @@ -1,148 +0,0 @@ -""" -pep384_macrocheck.py - -This program tries to locate errors in the relevant Python header -files where macros access type fields when they are reachable from -the limited API. - -The idea is to search macros with the string "->tp_" in it. -When the macro name does not begin with an underscore, -then we have found a dormant error. - -Christian Tismer -2018-06-02 -""" - -import sys -import os -import re - - -DEBUG = False - -def dprint(*args, **kw): - if DEBUG: - print(*args, **kw) - -def parse_headerfiles(startpath): - """ - Scan all header files which are reachable fronm Python.h - """ - search = "Python.h" - name = os.path.join(startpath, search) - if not os.path.exists(name): - raise ValueError("file {} was not found in {}\n" - "Please give the path to Python's include directory." - .format(search, startpath)) - errors = 0 - with open(name) as python_h: - while True: - line = python_h.readline() - if not line: - break - found = re.match(r'^\s*#\s*include\s*"(\w+\.h)"', line) - if not found: - continue - include = found.group(1) - dprint("Scanning", include) - name = os.path.join(startpath, include) - if not os.path.exists(name): - name = os.path.join(startpath, "../PC", include) - errors += parse_file(name) - return errors - -def ifdef_level_gen(): - """ - Scan lines for #ifdef and track the level. - """ - level = 0 - ifdef_pattern = r"^\s*#\s*if" # covers ifdef and ifndef as well - endif_pattern = r"^\s*#\s*endif" - while True: - line = yield level - if re.match(ifdef_pattern, line): - level += 1 - elif re.match(endif_pattern, line): - level -= 1 - -def limited_gen(): - """ - Scan lines for Py_LIMITED_API yes(1) no(-1) or nothing (0) - """ - limited = [0] # nothing - unlimited_pattern = r"^\s*#\s*ifndef\s+Py_LIMITED_API" - limited_pattern = "|".join([ - r"^\s*#\s*ifdef\s+Py_LIMITED_API", - r"^\s*#\s*(el)?if\s+!\s*defined\s*\(\s*Py_LIMITED_API\s*\)\s*\|\|", - r"^\s*#\s*(el)?if\s+defined\s*\(\s*Py_LIMITED_API" - ]) - else_pattern = r"^\s*#\s*else" - ifdef_level = ifdef_level_gen() - status = next(ifdef_level) - wait_for = -1 - while True: - line = yield limited[-1] - new_status = ifdef_level.send(line) - dir = new_status - status - status = new_status - if dir == 1: - if re.match(unlimited_pattern, line): - limited.append(-1) - wait_for = status - 1 - elif re.match(limited_pattern, line): - limited.append(1) - wait_for = status - 1 - elif dir == -1: - # this must have been an endif - if status == wait_for: - limited.pop() - wait_for = -1 - else: - # it could be that we have an elif - if re.match(limited_pattern, line): - limited.append(1) - wait_for = status - 1 - elif re.match(else_pattern, line): - limited.append(-limited.pop()) # negate top - -def parse_file(fname): - errors = 0 - with open(fname) as f: - lines = f.readlines() - type_pattern = r"^.*?->\s*tp_" - define_pattern = r"^\s*#\s*define\s+(\w+)" - limited = limited_gen() - status = next(limited) - for nr, line in enumerate(lines): - status = limited.send(line) - line = line.rstrip() - dprint(fname, nr, status, line) - if status != -1: - if re.match(define_pattern, line): - name = re.match(define_pattern, line).group(1) - if not name.startswith("_"): - # found a candidate, check it! - macro = line + "\n" - idx = nr - while line.endswith("\\"): - idx += 1 - line = lines[idx].rstrip() - macro += line + "\n" - if re.match(type_pattern, macro, re.DOTALL): - # this type field can reach the limited API - report(fname, nr + 1, macro) - errors += 1 - return errors - -def report(fname, nr, macro): - f = sys.stderr - print(fname + ":" + str(nr), file=f) - print(macro, file=f) - -if __name__ == "__main__": - p = sys.argv[1] if sys.argv[1:] else "../../Include" - errors = parse_headerfiles(p) - if errors: - # somehow it makes sense to raise a TypeError :-) - raise TypeError("These {} locations contradict the limited API." - .format(errors))