From 83aa496f81f86b46caf249a8ff7e168e6b27622d Mon Sep 17 00:00:00 2001 From: Olga Matoula Date: Fri, 28 Apr 2023 13:10:26 -0600 Subject: [PATCH 01/26] gh-101100: Add reference doc for __post_init__ (#103818) Signed-off-by: Olga Matoula --- Doc/library/dataclasses.rst | 45 +++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 85a7d9026d1d30..a5b20149921042 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -437,11 +437,11 @@ Module contents The newly returned object is created by calling the :meth:`~object.__init__` method of the dataclass. This ensures that - :ref:`__post_init__ `, if present, is also called. + :meth:`__post_init__`, if present, is also called. Init-only variables without default values, if any exist, must be specified on the call to :func:`replace` so that they can be passed to - :meth:`~object.__init__` and :ref:`__post_init__ `. + :meth:`~object.__init__` and :meth:`__post_init__`. It is an error for ``changes`` to contain any fields that are defined as having ``init=False``. A :exc:`ValueError` will be raised @@ -449,7 +449,7 @@ Module contents Be forewarned about how ``init=False`` fields work during a call to :func:`replace`. They are not copied from the source object, but - rather are initialized in :ref:`__post_init__ `, if they're + rather are initialized in :meth:`__post_init__`, if they're initialized at all. It is expected that ``init=False`` fields will be rarely and judiciously used. If they are used, it might be wise to have alternate class constructors, or perhaps a custom @@ -510,30 +510,31 @@ Module contents Post-init processing -------------------- -The generated :meth:`~object.__init__` code will call a method named -:meth:`!__post_init__`, if :meth:`!__post_init__` is defined on the -class. It will normally be called as ``self.__post_init__()``. -However, if any ``InitVar`` fields are defined, they will also be -passed to :meth:`!__post_init__` in the order they were defined in the -class. If no :meth:`~object.__init__` method is generated, then -:meth:`!__post_init__` will not automatically be called. +.. function:: __post_init__() -Among other uses, this allows for initializing field values that -depend on one or more other fields. For example:: + When defined on the class, it will be called by the generated + :meth:`~object.__init__`, normally as ``self.__post_init__()``. + However, if any ``InitVar`` fields are defined, they will also be + passed to :meth:`__post_init__` in the order they were defined in the + class. If no :meth:`~object.__init__` method is generated, then + :meth:`__post_init__` will not automatically be called. - @dataclass - class C: - a: float - b: float - c: float = field(init=False) + Among other uses, this allows for initializing field values that + depend on one or more other fields. For example:: - def __post_init__(self): - self.c = self.a + self.b + @dataclass + class C: + a: float + b: float + c: float = field(init=False) + + def __post_init__(self): + self.c = self.a + self.b The :meth:`~object.__init__` method generated by :func:`dataclass` does not call base class :meth:`~object.__init__` methods. If the base class has an :meth:`~object.__init__` method that has to be called, it is common to call this method in a -:meth:`!__post_init__` method:: +:meth:`__post_init__` method:: @dataclass class Rectangle: @@ -552,7 +553,7 @@ don't need to be called, since the derived dataclass will take care of initializing all fields of any base class that is a dataclass itself. See the section below on init-only variables for ways to pass -parameters to :meth:`!__post_init__`. Also see the warning about how +parameters to :meth:`__post_init__`. Also see the warning about how :func:`replace` handles ``init=False`` fields. Class variables @@ -576,7 +577,7 @@ is an ``InitVar``, it is considered a pseudo-field called an init-only field. As it is not a true field, it is not returned by the module-level :func:`fields` function. Init-only fields are added as parameters to the generated :meth:`~object.__init__` method, and are passed to -the optional :ref:`__post_init__ ` method. They are not otherwise used +the optional :meth:`__post_init__` method. They are not otherwise used by dataclasses. For example, suppose a field will be initialized from a database, if a From ebf97c50f25d61e15671a4658f5718f214c35a98 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 28 Apr 2023 13:20:50 -0600 Subject: [PATCH 02/26] gh-103978: avoid using 'class' as an identifier (#103979) --- Include/internal/pycore_code.h | 2 +- Python/specialize.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 7d5d5e03de9e41..86fd48b63ef8e4 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -226,7 +226,7 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); /* Specialization functions */ -extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *class, PyObject *self, +extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, PyObject *self, _Py_CODEUNIT *instr, PyObject *name, int load_method); extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name); diff --git a/Python/specialize.c b/Python/specialize.c index 33a3c4561c7ca2..fbdb435082cece 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -515,7 +515,7 @@ specialize_module_load_attr( /* Attribute specialization */ void -_Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *class, PyObject *self, +_Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *cls, PyObject *self, _Py_CODEUNIT *instr, PyObject *name, int load_method) { assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); @@ -528,11 +528,11 @@ _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *class, PyObject * SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED); goto fail; } - if (!PyType_Check(class)) { + if (!PyType_Check(cls)) { SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS); goto fail; } - PyTypeObject *tp = (PyTypeObject *)class; + PyTypeObject *tp = (PyTypeObject *)cls; PyObject *res = _PySuper_LookupDescr(tp, self, name); if (res == NULL) { SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_ERROR_OR_NOT_FOUND); From 689723a4abdc1e61a9f71db8ff40886ae1b1704d Mon Sep 17 00:00:00 2001 From: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Date: Fri, 28 Apr 2023 13:44:13 -0600 Subject: [PATCH 03/26] GH-103944: Check error status when raising DeprecationWarning (#103949) --- Modules/_datetimemodule.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index d392d384c3eee6..8f86fc91966205 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5144,13 +5144,13 @@ datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz) static PyObject * datetime_utcnow(PyObject *cls, PyObject *dummy) { - PyErr_WarnEx( - PyExc_DeprecationWarning, - "datetime.utcnow() is deprecated and scheduled for removal in a future " - "version. Use timezone-aware objects to represent datetimes in UTC: " - "datetime.now(datetime.UTC).", - 2 - ); + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "datetime.utcnow() is deprecated and scheduled for removal in a " + "future version. Use timezone-aware objects to represent datetimes " + "in UTC: datetime.now(datetime.UTC).", 2)) + { + return NULL; + } return datetime_best_possible(cls, _PyTime_gmtime, Py_None); } @@ -5187,13 +5187,13 @@ datetime_fromtimestamp(PyObject *cls, PyObject *args, PyObject *kw) static PyObject * datetime_utcfromtimestamp(PyObject *cls, PyObject *args) { - PyErr_WarnEx( - PyExc_DeprecationWarning, + if (PyErr_WarnEx(PyExc_DeprecationWarning, "datetime.utcfromtimestamp() is deprecated and scheduled for removal " "in a future version. Use timezone-aware objects to represent " - "datetimes in UTC: datetime.now(datetime.UTC).", - 2 - ); + "datetimes in UTC: datetime.now(datetime.UTC).", 2)) + { + return NULL; + } PyObject *timestamp; PyObject *result = NULL; From 79b9db9295a5a1607a0b4b10a8b4b72567eaf1ef Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 28 Apr 2023 13:25:48 -0700 Subject: [PATCH 04/26] GH-103971: Forward-port test from GH-103980 (GH-103984) --- Lib/test/test_patma.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 0ed54079c99b30..3dbd19dfffd318 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -3165,6 +3165,19 @@ def f(command): # 0 self.assertListEqual(self._trace(f, "go x"), [1, 2, 3]) self.assertListEqual(self._trace(f, "spam"), [1, 2, 3]) + def test_unreachable_code(self): + def f(command): # 0 + match command: # 1 + case 1: # 2 + if False: # 3 + return 1 # 4 + case _: # 5 + if False: # 6 + return 0 # 7 + + self.assertListEqual(self._trace(f, 1), [1, 2, 3]) + self.assertListEqual(self._trace(f, 0), [1, 2, 5, 6]) + def test_parser_deeply_nested_patterns(self): # Deeply nested patterns can cause exponential backtracking when parsing. # See gh-93671 for more information. From e1f14643dc0e6024f8df9ae975c3b05912a3cb28 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 28 Apr 2023 16:17:58 -0700 Subject: [PATCH 05/26] gh-98040: Remove just the `imp` module (#98573) --- Doc/c-api/import.rst | 2 + Doc/library/functions.rst | 1 - Doc/library/imp.rst | 411 -------------- Doc/library/superseded.rst | 1 - Doc/library/sys.rst | 4 - Doc/reference/import.rst | 3 +- Doc/tools/.nitignore | 1 - Doc/whatsnew/3.12.rst | 5 +- Lib/imp.py | 346 ------------ Lib/pkgutil.py | 183 +----- Lib/pydoc.py | 2 +- Lib/runpy.py | 7 +- Lib/test/test_imp.py | 524 ------------------ Lib/test/test_importlib/util.py | 5 +- Lib/test/test_pkgutil.py | 8 - ...2-10-21-17-20-57.gh-issue-98040.3btbmA.rst | 1 + Modules/_testmultiphase.c | 12 - Python/import.c | 8 +- Python/makeopcodetargets.py | 30 +- Python/pylifecycle.c | 7 +- Python/stdlib_module_names.h | 1 - Tools/build/generate_stdlib_module_names.py | 2 +- Tools/c-analyzer/TODO | 1 - Tools/importbench/importbench.py | 14 +- 24 files changed, 41 insertions(+), 1538 deletions(-) delete mode 100644 Doc/library/imp.rst delete mode 100644 Lib/imp.py delete mode 100644 Lib/test/test_imp.py create mode 100644 Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index a51619db6d3d97..474a64800044d0 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -188,6 +188,8 @@ Importing Modules .. versionchanged:: 3.3 Uses :func:`imp.source_from_cache()` in calculating the source path if only the bytecode path is provided. + .. versionchanged:: 3.12 + No longer uses the removed ``imp`` module. .. c:function:: long PyImport_GetMagicNumber() diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 7792e598c1155c..a5e86ef0f9eb59 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1987,7 +1987,6 @@ are always available. They are listed here in alphabetical order. .. index:: statement: import - module: imp .. note:: diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst deleted file mode 100644 index 000793a7e66cae..00000000000000 --- a/Doc/library/imp.rst +++ /dev/null @@ -1,411 +0,0 @@ -:mod:`imp` --- Access the :ref:`import ` internals -================================================================ - -.. module:: imp - :synopsis: Access the implementation of the import statement. - :deprecated: - -**Source code:** :source:`Lib/imp.py` - -.. deprecated-removed:: 3.4 3.12 - The :mod:`imp` module is deprecated in favor of :mod:`importlib`. - -.. index:: statement: import - --------------- - -This module provides an interface to the mechanisms used to implement the -:keyword:`import` statement. It defines the following constants and functions: - - -.. function:: get_magic() - - .. index:: pair: file; byte-code - - Return the magic string value used to recognize byte-compiled code files - (:file:`.pyc` files). (This value may be different for each Python version.) - - .. deprecated:: 3.4 - Use :attr:`importlib.util.MAGIC_NUMBER` instead. - - -.. function:: get_suffixes() - - Return a list of 3-element tuples, each describing a particular type of - module. Each triple has the form ``(suffix, mode, type)``, where *suffix* is - a string to be appended to the module name to form the filename to search - for, *mode* is the mode string to pass to the built-in :func:`open` function - to open the file (this can be ``'r'`` for text files or ``'rb'`` for binary - files), and *type* is the file type, which has one of the values - :const:`PY_SOURCE`, :const:`PY_COMPILED`, or :const:`C_EXTENSION`, described - below. - - .. deprecated:: 3.3 - Use the constants defined on :mod:`importlib.machinery` instead. - - -.. function:: find_module(name[, path]) - - Try to find the module *name*. If *path* is omitted or ``None``, the list of - directory names given by ``sys.path`` is searched, but first a few special - places are searched: the function tries to find a built-in module with the - given name (:const:`C_BUILTIN`), then a frozen module (:const:`PY_FROZEN`), - and on some systems some other places are looked in as well (on Windows, it - looks in the registry which may point to a specific file). - - Otherwise, *path* must be a list of directory names; each directory is - searched for files with any of the suffixes returned by :func:`get_suffixes` - above. Invalid names in the list are silently ignored (but all list items - must be strings). - - If search is successful, the return value is a 3-element tuple ``(file, - pathname, description)``: - - *file* is an open :term:`file object` positioned at the beginning, *pathname* - is the pathname of the file found, and *description* is a 3-element tuple as - contained in the list returned by :func:`get_suffixes` describing the kind of - module found. - - If the module is built-in or frozen then *file* and *pathname* are both ``None`` - and the *description* tuple contains empty strings for its suffix and mode; - the module type is indicated as given in parentheses above. If the search - is unsuccessful, :exc:`ImportError` is raised. Other exceptions indicate - problems with the arguments or environment. - - If the module is a package, *file* is ``None``, *pathname* is the package - path and the last item in the *description* tuple is :const:`PKG_DIRECTORY`. - - This function does not handle hierarchical module names (names containing - dots). In order to find *P.M*, that is, submodule *M* of package *P*, use - :func:`find_module` and :func:`load_module` to find and load package *P*, and - then use :func:`find_module` with the *path* argument set to ``P.__path__``. - When *P* itself has a dotted name, apply this recipe recursively. - - .. deprecated:: 3.3 - Use :func:`importlib.util.find_spec` instead unless Python 3.3 - compatibility is required, in which case use - :func:`importlib.find_loader`. For example usage of the former case, - see the :ref:`importlib-examples` section of the :mod:`importlib` - documentation. - - -.. function:: load_module(name, file, pathname, description) - - Load a module that was previously found by :func:`find_module` (or by an - otherwise conducted search yielding compatible results). This function does - more than importing the module: if the module was already imported, it will - reload the module! The *name* argument indicates the full - module name (including the package name, if this is a submodule of a - package). The *file* argument is an open file, and *pathname* is the - corresponding file name; these can be ``None`` and ``''``, respectively, when - the module is a package or not being loaded from a file. The *description* - argument is a tuple, as would be returned by :func:`get_suffixes`, describing - what kind of module must be loaded. - - If the load is successful, the return value is the module object; otherwise, - an exception (usually :exc:`ImportError`) is raised. - - **Important:** the caller is responsible for closing the *file* argument, if - it was not ``None``, even when an exception is raised. This is best done - using a :keyword:`try` ... :keyword:`finally` statement. - - .. deprecated:: 3.3 - If previously used in conjunction with :func:`imp.find_module` then - consider using :func:`importlib.import_module`, otherwise use the loader - returned by the replacement you chose for :func:`imp.find_module`. If you - called :func:`imp.load_module` and related functions directly with file - path arguments then use a combination of - :func:`importlib.util.spec_from_file_location` and - :func:`importlib.util.module_from_spec`. See the :ref:`importlib-examples` - section of the :mod:`importlib` documentation for details of the various - approaches. - - -.. function:: new_module(name) - - Return a new empty module object called *name*. This object is *not* inserted - in ``sys.modules``. - - .. deprecated:: 3.4 - Use :func:`importlib.util.module_from_spec` instead. - - -.. function:: reload(module) - - Reload a previously imported *module*. The argument must be a module object, so - it must have been successfully imported before. This is useful if you have - edited the module source file using an external editor and want to try out the - new version without leaving the Python interpreter. The return value is the - module object (the same as the *module* argument). - - When ``reload(module)`` is executed: - - * Python modules' code is recompiled and the module-level code reexecuted, - defining a new set of objects which are bound to names in the module's - dictionary. The ``init`` function of extension modules is not called a second - time. - - * As with all other objects in Python the old objects are only reclaimed after - their reference counts drop to zero. - - * The names in the module namespace are updated to point to any new or changed - objects. - - * Other references to the old objects (such as names external to the module) are - not rebound to refer to the new objects and must be updated in each namespace - where they occur if that is desired. - - There are a number of other caveats: - - When a module is reloaded, its dictionary (containing the module's global - variables) is retained. Redefinitions of names will override the old - definitions, so this is generally not a problem. If the new version of a module - does not define a name that was defined by the old version, the old definition - remains. This feature can be used to the module's advantage if it maintains a - global table or cache of objects --- with a :keyword:`try` statement it can test - for the table's presence and skip its initialization if desired:: - - try: - cache - except NameError: - cache = {} - - It is legal though generally not very useful to reload built-in or dynamically - loaded modules, except for :mod:`sys`, :mod:`__main__` and :mod:`builtins`. - In many cases, however, extension modules are not designed to be initialized - more than once, and may fail in arbitrary ways when reloaded. - - If a module imports objects from another module using :keyword:`from` ... - :keyword:`import` ..., calling :func:`reload` for the other module does not - redefine the objects imported from it --- one way around this is to re-execute - the :keyword:`!from` statement, another is to use :keyword:`!import` and qualified - names (*module*.*name*) instead. - - If a module instantiates instances of a class, reloading the module that defines - the class does not affect the method definitions of the instances --- they - continue to use the old class definition. The same is true for derived classes. - - .. versionchanged:: 3.3 - Relies on both ``__name__`` and ``__loader__`` being defined on the module - being reloaded instead of just ``__name__``. - - .. deprecated:: 3.4 - Use :func:`importlib.reload` instead. - - -The following functions are conveniences for handling :pep:`3147` byte-compiled -file paths. - -.. versionadded:: 3.2 - -.. function:: cache_from_source(path, debug_override=None) - - Return the :pep:`3147` path to the byte-compiled file associated with the - source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return - value would be ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. - The ``cpython-32`` string comes from the current magic tag (see - :func:`get_tag`; if :attr:`sys.implementation.cache_tag` is not defined then - :exc:`NotImplementedError` will be raised). By passing in ``True`` or - ``False`` for *debug_override* you can override the system's value for - ``__debug__``, leading to optimized bytecode. - - *path* need not exist. - - .. versionchanged:: 3.3 - If :attr:`sys.implementation.cache_tag` is ``None``, then - :exc:`NotImplementedError` is raised. - - .. deprecated:: 3.4 - Use :func:`importlib.util.cache_from_source` instead. - - .. versionchanged:: 3.5 - The *debug_override* parameter no longer creates a ``.pyo`` file. - - -.. function:: source_from_cache(path) - - Given the *path* to a :pep:`3147` file name, return the associated source code - file path. For example, if *path* is - ``/foo/bar/__pycache__/baz.cpython-32.pyc`` the returned path would be - ``/foo/bar/baz.py``. *path* need not exist, however if it does not conform - to :pep:`3147` format, a :exc:`ValueError` is raised. If - :attr:`sys.implementation.cache_tag` is not defined, - :exc:`NotImplementedError` is raised. - - .. versionchanged:: 3.3 - Raise :exc:`NotImplementedError` when - :attr:`sys.implementation.cache_tag` is not defined. - - .. deprecated:: 3.4 - Use :func:`importlib.util.source_from_cache` instead. - - -.. function:: get_tag() - - Return the :pep:`3147` magic tag string matching this version of Python's - magic number, as returned by :func:`get_magic`. - - .. deprecated:: 3.4 - Use :attr:`sys.implementation.cache_tag` directly starting - in Python 3.3. - - -The following functions help interact with the import system's internal -locking mechanism. Locking semantics of imports are an implementation -detail which may vary from release to release. However, Python ensures -that circular imports work without any deadlocks. - - -.. function:: lock_held() - - Return ``True`` if the global import lock is currently held, else - ``False``. On platforms without threads, always return ``False``. - - On platforms with threads, a thread executing an import first holds a - global import lock, then sets up a per-module lock for the rest of the - import. This blocks other threads from importing the same module until - the original import completes, preventing other threads from seeing - incomplete module objects constructed by the original thread. An - exception is made for circular imports, which by construction have to - expose an incomplete module object at some point. - - .. versionchanged:: 3.3 - The locking scheme has changed to per-module locks for - the most part. A global import lock is kept for some critical tasks, - such as initializing the per-module locks. - - .. deprecated:: 3.4 - - -.. function:: acquire_lock() - - Acquire the interpreter's global import lock for the current thread. - This lock should be used by import hooks to ensure thread-safety when - importing modules. - - Once a thread has acquired the import lock, the same thread may acquire it - again without blocking; the thread must release it once for each time it has - acquired it. - - On platforms without threads, this function does nothing. - - .. versionchanged:: 3.3 - The locking scheme has changed to per-module locks for - the most part. A global import lock is kept for some critical tasks, - such as initializing the per-module locks. - - .. deprecated:: 3.4 - - -.. function:: release_lock() - - Release the interpreter's global import lock. On platforms without - threads, this function does nothing. - - .. versionchanged:: 3.3 - The locking scheme has changed to per-module locks for - the most part. A global import lock is kept for some critical tasks, - such as initializing the per-module locks. - - .. deprecated:: 3.4 - - -The following constants with integer values, defined in this module, are used -to indicate the search result of :func:`find_module`. - - -.. data:: PY_SOURCE - - The module was found as a source file. - - .. deprecated:: 3.3 - - -.. data:: PY_COMPILED - - The module was found as a compiled code object file. - - .. deprecated:: 3.3 - - -.. data:: C_EXTENSION - - The module was found as dynamically loadable shared library. - - .. deprecated:: 3.3 - - -.. data:: PKG_DIRECTORY - - The module was found as a package directory. - - .. deprecated:: 3.3 - - -.. data:: C_BUILTIN - - The module was found as a built-in module. - - .. deprecated:: 3.3 - - -.. data:: PY_FROZEN - - The module was found as a frozen module. - - .. deprecated:: 3.3 - - -.. class:: NullImporter(path_string) - - The :class:`NullImporter` type is a :pep:`302` import hook that handles - non-directory path strings by failing to find any modules. Calling this type - with an existing directory or empty string raises :exc:`ImportError`. - Otherwise, a :class:`NullImporter` instance is returned. - - Instances have only one method: - - .. method:: NullImporter.find_module(fullname [, path]) - - This method always returns ``None``, indicating that the requested module could - not be found. - - .. versionchanged:: 3.3 - ``None`` is inserted into ``sys.path_importer_cache`` instead of an - instance of :class:`NullImporter`. - - .. deprecated:: 3.4 - Insert ``None`` into ``sys.path_importer_cache`` instead. - - -.. _examples-imp: - -Examples --------- - -The following function emulates what was the standard import statement up to -Python 1.4 (no hierarchical module names). (This *implementation* wouldn't work -in that version, since :func:`find_module` has been extended and -:func:`load_module` has been added in 1.4.) :: - - import imp - import sys - - def __import__(name, globals=None, locals=None, fromlist=None): - # Fast path: see if the module has already been imported. - try: - return sys.modules[name] - except KeyError: - pass - - # If any of the following calls raises an exception, - # there's a problem we can't handle -- let the caller handle it. - - fp, pathname, description = imp.find_module(name) - - try: - return imp.load_module(name, fp, pathname, description) - finally: - # Since we may exit via an exception, close fp explicitly. - if fp: - fp.close() diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst index 8786e227be9182..aaf66ea121d39c 100644 --- a/Doc/library/superseded.rst +++ b/Doc/library/superseded.rst @@ -17,7 +17,6 @@ backwards compatibility. They have been superseded by other modules. chunk.rst crypt.rst imghdr.rst - imp.rst mailcap.rst msilib.rst nis.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 7324f3113e0a08..7c0e85142e7716 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1253,10 +1253,6 @@ always available. Originally specified in :pep:`302`. - .. versionchanged:: 3.3 - ``None`` is stored instead of :class:`imp.NullImporter` when no finder - is found. - .. data:: platform diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index b22b5251f1de46..57eb5403243eef 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -1077,4 +1077,5 @@ methods to finders and loaders. .. [#fnpic] In legacy code, it is possible to find instances of :class:`imp.NullImporter` in the :data:`sys.path_importer_cache`. It is recommended that code be changed to use ``None`` instead. See - :ref:`portingpythoncode` for more details. + :ref:`portingpythoncode` for more details. Note that the ``imp`` module + was removed in Python 3.12. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f3350174f931aa..1d3503bf06f085 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -149,7 +149,6 @@ Doc/library/http.cookiejar.rst Doc/library/http.cookies.rst Doc/library/http.server.rst Doc/library/idle.rst -Doc/library/imp.rst Doc/library/importlib.resources.abc.rst Doc/library/importlib.resources.rst Doc/library/importlib.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 291500532493c6..a75e88c2615aca 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -961,11 +961,14 @@ Removed completed: * References to, and support for ``module_repr()`` has been eradicated. - + (Contributed by Barry Warsaw in :gh:`97850`.) * ``importlib.util.set_package`` has been removed. (Contributed by Brett Cannon in :gh:`65961`.) +* The ``imp`` module has been removed. (Contributed by Barry Warsaw in + :gh:`98040`.) + * Removed the ``suspicious`` rule from the documentation Makefile, and removed ``Doc/tools/rstlint.py``, both in favor of `sphinx-lint `_. diff --git a/Lib/imp.py b/Lib/imp.py deleted file mode 100644 index fe850f6a001814..00000000000000 --- a/Lib/imp.py +++ /dev/null @@ -1,346 +0,0 @@ -"""This module provides the components needed to build your own __import__ -function. Undocumented functions are obsolete. - -In most cases it is preferred you consider using the importlib module's -functionality over this module. - -""" -# (Probably) need to stay in _imp -from _imp import (lock_held, acquire_lock, release_lock, - get_frozen_object, is_frozen_package, - init_frozen, is_builtin, is_frozen, - _fix_co_filename, _frozen_module_names) -try: - from _imp import create_dynamic -except ImportError: - # Platform doesn't support dynamic loading. - create_dynamic = None - -from importlib._bootstrap import _ERR_MSG, _exec, _load, _builtin_from_name -from importlib._bootstrap_external import SourcelessFileLoader - -from importlib import machinery -from importlib import util -import importlib -import os -import sys -import tokenize -import types -import warnings - -warnings.warn("the imp module is deprecated in favour of importlib and slated " - "for removal in Python 3.12; " - "see the module's documentation for alternative uses", - DeprecationWarning, stacklevel=2) - -# DEPRECATED -SEARCH_ERROR = 0 -PY_SOURCE = 1 -PY_COMPILED = 2 -C_EXTENSION = 3 -PY_RESOURCE = 4 -PKG_DIRECTORY = 5 -C_BUILTIN = 6 -PY_FROZEN = 7 -PY_CODERESOURCE = 8 -IMP_HOOK = 9 - - -def new_module(name): - """**DEPRECATED** - - Create a new module. - - The module is not entered into sys.modules. - - """ - return types.ModuleType(name) - - -def get_magic(): - """**DEPRECATED** - - Return the magic number for .pyc files. - """ - return util.MAGIC_NUMBER - - -def get_tag(): - """Return the magic tag for .pyc files.""" - return sys.implementation.cache_tag - - -def cache_from_source(path, debug_override=None): - """**DEPRECATED** - - Given the path to a .py file, return the path to its .pyc file. - - The .py file does not need to exist; this simply returns the path to the - .pyc file calculated as if the .py file were imported. - - If debug_override is not None, then it must be a boolean and is used in - place of sys.flags.optimize. - - If sys.implementation.cache_tag is None then NotImplementedError is raised. - - """ - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - return util.cache_from_source(path, debug_override) - - -def source_from_cache(path): - """**DEPRECATED** - - Given the path to a .pyc. file, return the path to its .py file. - - The .pyc file does not need to exist; this simply returns the path to - the .py file calculated to correspond to the .pyc file. If path does - not conform to PEP 3147 format, ValueError will be raised. If - sys.implementation.cache_tag is None then NotImplementedError is raised. - - """ - return util.source_from_cache(path) - - -def get_suffixes(): - """**DEPRECATED**""" - extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES] - source = [(s, 'r', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES] - bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES] - - return extensions + source + bytecode - - -class NullImporter: - - """**DEPRECATED** - - Null import object. - - """ - - def __init__(self, path): - if path == '': - raise ImportError('empty pathname', path='') - elif os.path.isdir(path): - raise ImportError('existing directory', path=path) - - def find_module(self, fullname): - """Always returns None.""" - return None - - -class _HackedGetData: - - """Compatibility support for 'file' arguments of various load_*() - functions.""" - - def __init__(self, fullname, path, file=None): - super().__init__(fullname, path) - self.file = file - - def get_data(self, path): - """Gross hack to contort loader to deal w/ load_*()'s bad API.""" - if self.file and path == self.path: - # The contract of get_data() requires us to return bytes. Reopen the - # file in binary mode if needed. - if not self.file.closed: - file = self.file - if 'b' not in file.mode: - file.close() - if self.file.closed: - self.file = file = open(self.path, 'rb') - - with file: - return file.read() - else: - return super().get_data(path) - - -class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader): - - """Compatibility support for implementing load_source().""" - - -def load_source(name, pathname, file=None): - loader = _LoadSourceCompatibility(name, pathname, file) - spec = util.spec_from_file_location(name, pathname, loader=loader) - if name in sys.modules: - module = _exec(spec, sys.modules[name]) - else: - module = _load(spec) - # To allow reloading to potentially work, use a non-hacked loader which - # won't rely on a now-closed file object. - module.__loader__ = machinery.SourceFileLoader(name, pathname) - module.__spec__.loader = module.__loader__ - return module - - -class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader): - - """Compatibility support for implementing load_compiled().""" - - -def load_compiled(name, pathname, file=None): - """**DEPRECATED**""" - loader = _LoadCompiledCompatibility(name, pathname, file) - spec = util.spec_from_file_location(name, pathname, loader=loader) - if name in sys.modules: - module = _exec(spec, sys.modules[name]) - else: - module = _load(spec) - # To allow reloading to potentially work, use a non-hacked loader which - # won't rely on a now-closed file object. - module.__loader__ = SourcelessFileLoader(name, pathname) - module.__spec__.loader = module.__loader__ - return module - - -def load_package(name, path): - """**DEPRECATED**""" - if os.path.isdir(path): - extensions = (machinery.SOURCE_SUFFIXES[:] + - machinery.BYTECODE_SUFFIXES[:]) - for extension in extensions: - init_path = os.path.join(path, '__init__' + extension) - if os.path.exists(init_path): - path = init_path - break - else: - raise ValueError('{!r} is not a package'.format(path)) - spec = util.spec_from_file_location(name, path, - submodule_search_locations=[]) - if name in sys.modules: - return _exec(spec, sys.modules[name]) - else: - return _load(spec) - - -def load_module(name, file, filename, details): - """**DEPRECATED** - - Load a module, given information returned by find_module(). - - The module name must include the full package name, if any. - - """ - suffix, mode, type_ = details - if mode and (not mode.startswith('r') or '+' in mode): - raise ValueError('invalid file open mode {!r}'.format(mode)) - elif file is None and type_ in {PY_SOURCE, PY_COMPILED}: - msg = 'file object required for import (type code {})'.format(type_) - raise ValueError(msg) - elif type_ == PY_SOURCE: - return load_source(name, filename, file) - elif type_ == PY_COMPILED: - return load_compiled(name, filename, file) - elif type_ == C_EXTENSION and load_dynamic is not None: - if file is None: - with open(filename, 'rb') as opened_file: - return load_dynamic(name, filename, opened_file) - else: - return load_dynamic(name, filename, file) - elif type_ == PKG_DIRECTORY: - return load_package(name, filename) - elif type_ == C_BUILTIN: - return init_builtin(name) - elif type_ == PY_FROZEN: - return init_frozen(name) - else: - msg = "Don't know how to import {} (type code {})".format(name, type_) - raise ImportError(msg, name=name) - - -def find_module(name, path=None): - """**DEPRECATED** - - Search for a module. - - If path is omitted or None, search for a built-in, frozen or special - module and continue search in sys.path. The module name cannot - contain '.'; to search for a submodule of a package, pass the - submodule name and the package's __path__. - - """ - if not isinstance(name, str): - raise TypeError("'name' must be a str, not {}".format(type(name))) - elif not isinstance(path, (type(None), list)): - # Backwards-compatibility - raise RuntimeError("'path' must be None or a list, " - "not {}".format(type(path))) - - if path is None: - if is_builtin(name): - return None, None, ('', '', C_BUILTIN) - elif is_frozen(name): - return None, None, ('', '', PY_FROZEN) - else: - path = sys.path - - for entry in path: - package_directory = os.path.join(entry, name) - for suffix in ['.py', machinery.BYTECODE_SUFFIXES[0]]: - package_file_name = '__init__' + suffix - file_path = os.path.join(package_directory, package_file_name) - if os.path.isfile(file_path): - return None, package_directory, ('', '', PKG_DIRECTORY) - for suffix, mode, type_ in get_suffixes(): - file_name = name + suffix - file_path = os.path.join(entry, file_name) - if os.path.isfile(file_path): - break - else: - continue - break # Break out of outer loop when breaking out of inner loop. - else: - raise ImportError(_ERR_MSG.format(name), name=name) - - encoding = None - if 'b' not in mode: - with open(file_path, 'rb') as file: - encoding = tokenize.detect_encoding(file.readline)[0] - file = open(file_path, mode, encoding=encoding) - return file, file_path, (suffix, mode, type_) - - -def reload(module): - """**DEPRECATED** - - Reload the module and return it. - - The module must have been successfully imported before. - - """ - return importlib.reload(module) - - -def init_builtin(name): - """**DEPRECATED** - - Load and return a built-in module by name, or None is such module doesn't - exist - """ - try: - return _builtin_from_name(name) - except ImportError: - return None - - -if create_dynamic: - def load_dynamic(name, path, file=None): - """**DEPRECATED** - - Load an extension module. - """ - import importlib.machinery - loader = importlib.machinery.ExtensionFileLoader(name, path) - - # Issue #24748: Skip the sys.modules check in _load_module_shim; - # always load new extension - spec = importlib.util.spec_from_file_location( - name, path, loader=loader) - return _load(spec) - -else: - load_dynamic = None diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 56731de64af494..fb977eaaa05767 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -14,7 +14,7 @@ __all__ = [ 'get_importer', 'iter_importers', 'get_loader', 'find_loader', 'walk_packages', 'iter_modules', 'get_data', - 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', + 'read_code', 'extend_path', 'ModuleInfo', ] @@ -185,187 +185,6 @@ def _iter_file_finder_modules(importer, prefix=''): importlib.machinery.FileFinder, _iter_file_finder_modules) -def _import_imp(): - global imp - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - imp = importlib.import_module('imp') - -class ImpImporter: - """PEP 302 Finder that wraps Python's "classic" import algorithm - - ImpImporter(dirname) produces a PEP 302 finder that searches that - directory. ImpImporter(None) produces a PEP 302 finder that searches - the current sys.path, plus any modules that are frozen or built-in. - - Note that ImpImporter does not currently support being used by placement - on sys.meta_path. - """ - - def __init__(self, path=None): - global imp - warnings.warn("This emulation is deprecated and slated for removal " - "in Python 3.12; use 'importlib' instead", - DeprecationWarning) - _import_imp() - self.path = path - - def find_module(self, fullname, path=None): - # Note: we ignore 'path' argument since it is only used via meta_path - subname = fullname.split(".")[-1] - if subname != fullname and self.path is None: - return None - if self.path is None: - path = None - else: - path = [os.path.realpath(self.path)] - try: - file, filename, etc = imp.find_module(subname, path) - except ImportError: - return None - return ImpLoader(fullname, file, filename, etc) - - def iter_modules(self, prefix=''): - if self.path is None or not os.path.isdir(self.path): - return - - yielded = {} - import inspect - try: - filenames = os.listdir(self.path) - except OSError: - # ignore unreadable directories like import does - filenames = [] - filenames.sort() # handle packages before same-named modules - - for fn in filenames: - modname = inspect.getmodulename(fn) - if modname=='__init__' or modname in yielded: - continue - - path = os.path.join(self.path, fn) - ispkg = False - - if not modname and os.path.isdir(path) and '.' not in fn: - modname = fn - try: - dircontents = os.listdir(path) - except OSError: - # ignore unreadable directories like import does - dircontents = [] - for fn in dircontents: - subname = inspect.getmodulename(fn) - if subname=='__init__': - ispkg = True - break - else: - continue # not a package - - if modname and '.' not in modname: - yielded[modname] = 1 - yield prefix + modname, ispkg - - -class ImpLoader: - """PEP 302 Loader that wraps Python's "classic" import algorithm - """ - code = source = None - - def __init__(self, fullname, file, filename, etc): - warnings.warn("This emulation is deprecated and slated for removal in " - "Python 3.12; use 'importlib' instead", - DeprecationWarning) - _import_imp() - self.file = file - self.filename = filename - self.fullname = fullname - self.etc = etc - - def load_module(self, fullname): - self._reopen() - try: - mod = imp.load_module(fullname, self.file, self.filename, self.etc) - finally: - if self.file: - self.file.close() - # Note: we don't set __loader__ because we want the module to look - # normal; i.e. this is just a wrapper for standard import machinery - return mod - - def get_data(self, pathname): - with open(pathname, "rb") as file: - return file.read() - - def _reopen(self): - if self.file and self.file.closed: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - self.file = open(self.filename, 'r') - elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): - self.file = open(self.filename, 'rb') - - def _fix_name(self, fullname): - if fullname is None: - fullname = self.fullname - elif fullname != self.fullname: - raise ImportError("Loader for module %s cannot handle " - "module %s" % (self.fullname, fullname)) - return fullname - - def is_package(self, fullname): - fullname = self._fix_name(fullname) - return self.etc[2]==imp.PKG_DIRECTORY - - def get_code(self, fullname=None): - fullname = self._fix_name(fullname) - if self.code is None: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - source = self.get_source(fullname) - self.code = compile(source, self.filename, 'exec') - elif mod_type==imp.PY_COMPILED: - self._reopen() - try: - self.code = read_code(self.file) - finally: - self.file.close() - elif mod_type==imp.PKG_DIRECTORY: - self.code = self._get_delegate().get_code() - return self.code - - def get_source(self, fullname=None): - fullname = self._fix_name(fullname) - if self.source is None: - mod_type = self.etc[2] - if mod_type==imp.PY_SOURCE: - self._reopen() - try: - self.source = self.file.read() - finally: - self.file.close() - elif mod_type==imp.PY_COMPILED: - if os.path.exists(self.filename[:-1]): - with open(self.filename[:-1], 'r') as f: - self.source = f.read() - elif mod_type==imp.PKG_DIRECTORY: - self.source = self._get_delegate().get_source() - return self.source - - def _get_delegate(self): - finder = ImpImporter(self.filename) - spec = _get_spec(finder, '__init__') - return spec.loader - - def get_filename(self, fullname=None): - fullname = self._fix_name(fullname) - mod_type = self.etc[2] - if mod_type==imp.PKG_DIRECTORY: - return self._get_delegate().get_filename() - elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): - return self.filename - return None - - try: import zipimport from zipimport import zipimporter diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 78d8fd5357f72a..1c3443fa8469f7 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -512,7 +512,7 @@ def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): basedir = os.path.normcase(basedir) if (isinstance(object, type(os)) and - (object.__name__ in ('errno', 'exceptions', 'gc', 'imp', + (object.__name__ in ('errno', 'exceptions', 'gc', 'marshal', 'posix', 'signal', 'sys', '_thread', 'zipimport') or (file.startswith(basedir) and diff --git a/Lib/runpy.py b/Lib/runpy.py index 54fc136d4074f2..42f896c9cd5094 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -279,12 +279,7 @@ def run_path(path_name, init_globals=None, run_name=None): pkg_name = run_name.rpartition(".")[0] from pkgutil import get_importer importer = get_importer(path_name) - # Trying to avoid importing imp so as to not consume the deprecation warning. - is_NullImporter = False - if type(importer).__module__ == 'imp': - if type(importer).__name__ == 'NullImporter': - is_NullImporter = True - if isinstance(importer, type(None)) or is_NullImporter: + if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files code, fname = _get_code_from_file(run_name, path_name) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py deleted file mode 100644 index 80abc720c3251a..00000000000000 --- a/Lib/test/test_imp.py +++ /dev/null @@ -1,524 +0,0 @@ -import gc -import importlib -import importlib.util -import os -import os.path -import py_compile -import sys -from test import support -from test.support import import_helper -from test.support import os_helper -from test.support import script_helper -from test.support import warnings_helper -import unittest -import warnings -imp = warnings_helper.import_deprecated('imp') -import _imp - - -OS_PATH_NAME = os.path.__name__ - - -def requires_load_dynamic(meth): - """Decorator to skip a test if not running under CPython or lacking - imp.load_dynamic().""" - meth = support.cpython_only(meth) - return unittest.skipIf(getattr(imp, 'load_dynamic', None) is None, - 'imp.load_dynamic() required')(meth) - - -class LockTests(unittest.TestCase): - - """Very basic test of import lock functions.""" - - def verify_lock_state(self, expected): - self.assertEqual(imp.lock_held(), expected, - "expected imp.lock_held() to be %r" % expected) - def testLock(self): - LOOPS = 50 - - # The import lock may already be held, e.g. if the test suite is run - # via "import test.autotest". - lock_held_at_start = imp.lock_held() - self.verify_lock_state(lock_held_at_start) - - for i in range(LOOPS): - imp.acquire_lock() - self.verify_lock_state(True) - - for i in range(LOOPS): - imp.release_lock() - - # The original state should be restored now. - self.verify_lock_state(lock_held_at_start) - - if not lock_held_at_start: - try: - imp.release_lock() - except RuntimeError: - pass - else: - self.fail("release_lock() without lock should raise " - "RuntimeError") - -class ImportTests(unittest.TestCase): - def setUp(self): - mod = importlib.import_module('test.encoded_modules') - self.test_strings = mod.test_strings - self.test_path = mod.__path__ - - # test_import_encoded_module moved to test_source_encoding.py - - def test_find_module_encoding(self): - for mod, encoding, _ in self.test_strings: - with imp.find_module('module_' + mod, self.test_path)[0] as fd: - self.assertEqual(fd.encoding, encoding) - - path = [os.path.dirname(__file__)] - with self.assertRaises(SyntaxError): - imp.find_module('badsyntax_pep3120', path) - - def test_issue1267(self): - for mod, encoding, _ in self.test_strings: - fp, filename, info = imp.find_module('module_' + mod, - self.test_path) - with fp: - self.assertNotEqual(fp, None) - self.assertEqual(fp.encoding, encoding) - self.assertEqual(fp.tell(), 0) - self.assertEqual(fp.readline(), '# test %s encoding\n' - % encoding) - - fp, filename, info = imp.find_module("tokenize") - with fp: - self.assertNotEqual(fp, None) - self.assertEqual(fp.encoding, "utf-8") - self.assertEqual(fp.tell(), 0) - self.assertEqual(fp.readline(), - '"""Tokenization help for Python programs.\n') - - def test_issue3594(self): - temp_mod_name = 'test_imp_helper' - sys.path.insert(0, '.') - try: - with open(temp_mod_name + '.py', 'w', encoding="latin-1") as file: - file.write("# coding: cp1252\nu = 'test.test_imp'\n") - file, filename, info = imp.find_module(temp_mod_name) - file.close() - self.assertEqual(file.encoding, 'cp1252') - finally: - del sys.path[0] - os_helper.unlink(temp_mod_name + '.py') - os_helper.unlink(temp_mod_name + '.pyc') - - def test_issue5604(self): - # Test cannot cover imp.load_compiled function. - # Martin von Loewis note what shared library cannot have non-ascii - # character because init_xxx function cannot be compiled - # and issue never happens for dynamic modules. - # But sources modified to follow generic way for processing paths. - - # the return encoding could be uppercase or None - fs_encoding = sys.getfilesystemencoding() - - # covers utf-8 and Windows ANSI code pages - # one non-space symbol from every page - # (http://en.wikipedia.org/wiki/Code_page) - known_locales = { - 'utf-8' : b'\xc3\xa4', - 'cp1250' : b'\x8C', - 'cp1251' : b'\xc0', - 'cp1252' : b'\xc0', - 'cp1253' : b'\xc1', - 'cp1254' : b'\xc0', - 'cp1255' : b'\xe0', - 'cp1256' : b'\xe0', - 'cp1257' : b'\xc0', - 'cp1258' : b'\xc0', - } - - if sys.platform == 'darwin': - self.assertEqual(fs_encoding, 'utf-8') - # Mac OS X uses the Normal Form D decomposition - # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html - special_char = b'a\xcc\x88' - else: - special_char = known_locales.get(fs_encoding) - - if not special_char: - self.skipTest("can't run this test with %s as filesystem encoding" - % fs_encoding) - decoded_char = special_char.decode(fs_encoding) - temp_mod_name = 'test_imp_helper_' + decoded_char - test_package_name = 'test_imp_helper_package_' + decoded_char - init_file_name = os.path.join(test_package_name, '__init__.py') - try: - # if the curdir is not in sys.path the test fails when run with - # ./python ./Lib/test/regrtest.py test_imp - sys.path.insert(0, os.curdir) - with open(temp_mod_name + '.py', 'w', encoding="utf-8") as file: - file.write('a = 1\n') - file, filename, info = imp.find_module(temp_mod_name) - with file: - self.assertIsNotNone(file) - self.assertTrue(filename[:-3].endswith(temp_mod_name)) - self.assertEqual(info[0], '.py') - self.assertEqual(info[1], 'r') - self.assertEqual(info[2], imp.PY_SOURCE) - - mod = imp.load_module(temp_mod_name, file, filename, info) - self.assertEqual(mod.a, 1) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - mod = imp.load_source(temp_mod_name, temp_mod_name + '.py') - self.assertEqual(mod.a, 1) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - if not sys.dont_write_bytecode: - mod = imp.load_compiled( - temp_mod_name, - imp.cache_from_source(temp_mod_name + '.py')) - self.assertEqual(mod.a, 1) - - if not os.path.exists(test_package_name): - os.mkdir(test_package_name) - with open(init_file_name, 'w', encoding="utf-8") as file: - file.write('b = 2\n') - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - package = imp.load_package(test_package_name, test_package_name) - self.assertEqual(package.b, 2) - finally: - del sys.path[0] - for ext in ('.py', '.pyc'): - os_helper.unlink(temp_mod_name + ext) - os_helper.unlink(init_file_name + ext) - os_helper.rmtree(test_package_name) - os_helper.rmtree('__pycache__') - - def test_issue9319(self): - path = os.path.dirname(__file__) - self.assertRaises(SyntaxError, - imp.find_module, "badsyntax_pep3120", [path]) - - def test_load_from_source(self): - # Verify that the imp module can correctly load and find .py files - # XXX (ncoghlan): It would be nice to use import_helper.CleanImport - # here, but that breaks because the os module registers some - # handlers in copy_reg on import. Since CleanImport doesn't - # revert that registration, the module is left in a broken - # state after reversion. Reinitialising the module contents - # and just reverting os.environ to its previous state is an OK - # workaround - with import_helper.CleanImport('os', 'os.path', OS_PATH_NAME): - import os - orig_path = os.path - orig_getenv = os.getenv - with os_helper.EnvironmentVarGuard(): - x = imp.find_module("os") - self.addCleanup(x[0].close) - new_os = imp.load_module("os", *x) - self.assertIs(os, new_os) - self.assertIs(orig_path, new_os.path) - self.assertIsNot(orig_getenv, new_os.getenv) - - @requires_load_dynamic - def test_issue15828_load_extensions(self): - # Issue 15828 picked up that the adapter between the old imp API - # and importlib couldn't handle C extensions - example = "_heapq" - x = imp.find_module(example) - file_ = x[0] - if file_ is not None: - self.addCleanup(file_.close) - mod = imp.load_module(example, *x) - self.assertEqual(mod.__name__, example) - - @requires_load_dynamic - def test_issue16421_multiple_modules_in_one_dll(self): - # Issue 16421: loading several modules from the same compiled file fails - m = '_testimportmultiple' - fileobj, pathname, description = imp.find_module(m) - fileobj.close() - mod0 = imp.load_dynamic(m, pathname) - mod1 = imp.load_dynamic('_testimportmultiple_foo', pathname) - mod2 = imp.load_dynamic('_testimportmultiple_bar', pathname) - self.assertEqual(mod0.__name__, m) - self.assertEqual(mod1.__name__, '_testimportmultiple_foo') - self.assertEqual(mod2.__name__, '_testimportmultiple_bar') - with self.assertRaises(ImportError): - imp.load_dynamic('nonexistent', pathname) - - @requires_load_dynamic - def test_load_dynamic_ImportError_path(self): - # Issue #1559549 added `name` and `path` attributes to ImportError - # in order to provide better detail. Issue #10854 implemented those - # attributes on import failures of extensions on Windows. - path = 'bogus file path' - name = 'extension' - with self.assertRaises(ImportError) as err: - imp.load_dynamic(name, path) - self.assertIn(path, err.exception.path) - self.assertEqual(name, err.exception.name) - - @requires_load_dynamic - def test_load_module_extension_file_is_None(self): - # When loading an extension module and the file is None, open one - # on the behalf of imp.load_dynamic(). - # Issue #15902 - name = '_testimportmultiple' - found = imp.find_module(name) - if found[0] is not None: - found[0].close() - if found[2][2] != imp.C_EXTENSION: - self.skipTest("found module doesn't appear to be a C extension") - imp.load_module(name, None, *found[1:]) - - @requires_load_dynamic - def test_issue24748_load_module_skips_sys_modules_check(self): - name = 'test.imp_dummy' - try: - del sys.modules[name] - except KeyError: - pass - try: - module = importlib.import_module(name) - spec = importlib.util.find_spec('_testmultiphase') - module = imp.load_dynamic(name, spec.origin) - self.assertEqual(module.__name__, name) - self.assertEqual(module.__spec__.name, name) - self.assertEqual(module.__spec__.origin, spec.origin) - self.assertRaises(AttributeError, getattr, module, 'dummy_name') - self.assertEqual(module.int_const, 1969) - self.assertIs(sys.modules[name], module) - finally: - try: - del sys.modules[name] - except KeyError: - pass - - @unittest.skipIf(sys.dont_write_bytecode, - "test meaningful only when writing bytecode") - def test_bug7732(self): - with os_helper.temp_cwd(): - source = os_helper.TESTFN + '.py' - os.mkdir(source) - self.assertRaisesRegex(ImportError, '^No module', - imp.find_module, os_helper.TESTFN, ["."]) - - def test_multiple_calls_to_get_data(self): - # Issue #18755: make sure multiple calls to get_data() can succeed. - loader = imp._LoadSourceCompatibility('imp', imp.__file__, - open(imp.__file__, encoding="utf-8")) - loader.get_data(imp.__file__) # File should be closed - loader.get_data(imp.__file__) # Will need to create a newly opened file - - def test_load_source(self): - # Create a temporary module since load_source(name) modifies - # sys.modules[name] attributes like __loader___ - modname = f"tmp{__name__}" - mod = type(sys.modules[__name__])(modname) - with support.swap_item(sys.modules, modname, mod): - with self.assertRaisesRegex(ValueError, 'embedded null'): - imp.load_source(modname, __file__ + "\0") - - @support.cpython_only - def test_issue31315(self): - # There shouldn't be an assertion failure in imp.create_dynamic(), - # when spec.name is not a string. - create_dynamic = support.get_attribute(imp, 'create_dynamic') - class BadSpec: - name = None - origin = 'foo' - with self.assertRaises(TypeError): - create_dynamic(BadSpec()) - - def test_issue_35321(self): - # Both _frozen_importlib and _frozen_importlib_external - # should have a spec origin of "frozen" and - # no need to clean up imports in this case. - - import _frozen_importlib_external - self.assertEqual(_frozen_importlib_external.__spec__.origin, "frozen") - - import _frozen_importlib - self.assertEqual(_frozen_importlib.__spec__.origin, "frozen") - - def test_source_hash(self): - self.assertEqual(_imp.source_hash(42, b'hi'), b'\xfb\xd9G\x05\xaf$\x9b~') - self.assertEqual(_imp.source_hash(43, b'hi'), b'\xd0/\x87C\xccC\xff\xe2') - - def test_pyc_invalidation_mode_from_cmdline(self): - cases = [ - ([], "default"), - (["--check-hash-based-pycs", "default"], "default"), - (["--check-hash-based-pycs", "always"], "always"), - (["--check-hash-based-pycs", "never"], "never"), - ] - for interp_args, expected in cases: - args = interp_args + [ - "-c", - "import _imp; print(_imp.check_hash_based_pycs)", - ] - res = script_helper.assert_python_ok(*args) - self.assertEqual(res.out.strip().decode('utf-8'), expected) - - def test_find_and_load_checked_pyc(self): - # issue 34056 - with os_helper.temp_cwd(): - with open('mymod.py', 'wb') as fp: - fp.write(b'x = 42\n') - py_compile.compile( - 'mymod.py', - doraise=True, - invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, - ) - file, path, description = imp.find_module('mymod', path=['.']) - mod = imp.load_module('mymod', file, path, description) - self.assertEqual(mod.x, 42) - - def test_issue98354(self): - # _imp.create_builtin should raise TypeError - # if 'name' attribute of 'spec' argument is not a 'str' instance - - create_builtin = support.get_attribute(_imp, "create_builtin") - - class FakeSpec: - def __init__(self, name): - self.name = self - spec = FakeSpec("time") - with self.assertRaises(TypeError): - create_builtin(spec) - - class FakeSpec2: - name = [1, 2, 3, 4] - spec = FakeSpec2() - with self.assertRaises(TypeError): - create_builtin(spec) - - import builtins - class UnicodeSubclass(str): - pass - class GoodSpec: - name = UnicodeSubclass("builtins") - spec = GoodSpec() - bltin = create_builtin(spec) - self.assertEqual(bltin, builtins) - - class UnicodeSubclassFakeSpec(str): - def __init__(self, name): - self.name = self - spec = UnicodeSubclassFakeSpec("builtins") - bltin = create_builtin(spec) - self.assertEqual(bltin, builtins) - - @support.cpython_only - def test_create_builtin_subinterp(self): - # gh-99578: create_builtin() behavior changes after the creation of the - # first sub-interpreter. Test both code paths, before and after the - # creation of a sub-interpreter. Previously, create_builtin() had - # a reference leak after the creation of the first sub-interpreter. - - import builtins - create_builtin = support.get_attribute(_imp, "create_builtin") - class Spec: - name = "builtins" - spec = Spec() - - def check_get_builtins(): - refcnt = sys.getrefcount(builtins) - mod = _imp.create_builtin(spec) - self.assertIs(mod, builtins) - self.assertEqual(sys.getrefcount(builtins), refcnt + 1) - # Check that a GC collection doesn't crash - gc.collect() - - check_get_builtins() - - ret = support.run_in_subinterp("import builtins") - self.assertEqual(ret, 0) - - check_get_builtins() - - -class ReloadTests(unittest.TestCase): - - """Very basic tests to make sure that imp.reload() operates just like - reload().""" - - def test_source(self): - # XXX (ncoghlan): It would be nice to use test.import_helper.CleanImport - # here, but that breaks because the os module registers some - # handlers in copy_reg on import. Since CleanImport doesn't - # revert that registration, the module is left in a broken - # state after reversion. Reinitialising the module contents - # and just reverting os.environ to its previous state is an OK - # workaround - with os_helper.EnvironmentVarGuard(): - import os - imp.reload(os) - - def test_extension(self): - with import_helper.CleanImport('time'): - import time - imp.reload(time) - - def test_builtin(self): - with import_helper.CleanImport('marshal'): - import marshal - imp.reload(marshal) - - def test_with_deleted_parent(self): - # see #18681 - from html import parser - html = sys.modules.pop('html') - def cleanup(): - sys.modules['html'] = html - self.addCleanup(cleanup) - with self.assertRaisesRegex(ImportError, 'html'): - imp.reload(parser) - - -class PEP3147Tests(unittest.TestCase): - """Tests of PEP 3147.""" - - tag = imp.get_tag() - - @unittest.skipUnless(sys.implementation.cache_tag is not None, - 'requires sys.implementation.cache_tag not be None') - def test_cache_from_source(self): - # Given the path to a .py file, return the path to its PEP 3147 - # defined .pyc file (i.e. under __pycache__). - path = os.path.join('foo', 'bar', 'baz', 'qux.py') - expect = os.path.join('foo', 'bar', 'baz', '__pycache__', - 'qux.{}.pyc'.format(self.tag)) - self.assertEqual(imp.cache_from_source(path, True), expect) - - @unittest.skipUnless(sys.implementation.cache_tag is not None, - 'requires sys.implementation.cache_tag to not be ' - 'None') - def test_source_from_cache(self): - # Given the path to a PEP 3147 defined .pyc file, return the path to - # its source. This tests the good path. - path = os.path.join('foo', 'bar', 'baz', '__pycache__', - 'qux.{}.pyc'.format(self.tag)) - expect = os.path.join('foo', 'bar', 'baz', 'qux.py') - self.assertEqual(imp.source_from_cache(path), expect) - - -class NullImporterTests(unittest.TestCase): - @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, - "Need an undecodeable filename") - def test_unencodeable(self): - name = os_helper.TESTFN_UNENCODABLE - os.mkdir(name) - try: - self.assertRaises(ImportError, imp.NullImporter, name) - finally: - os.rmdir(name) - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 9032fd18d3f95b..e348733f6ce3c3 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -131,9 +131,8 @@ def uncache(*names): """ for name in names: - if name in ('sys', 'marshal', 'imp'): - raise ValueError( - "cannot uncache {0}".format(name)) + if name in ('sys', 'marshal'): + raise ValueError("cannot uncache {}".format(name)) try: del sys.modules[name] except KeyError: diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 0cc99e0cc22763..4d9f5db3c6b3cf 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -541,14 +541,6 @@ def check_deprecated(self): "Python 3.12; use 'importlib' instead", DeprecationWarning)) - def test_importer_deprecated(self): - with self.check_deprecated(): - pkgutil.ImpImporter("") - - def test_loader_deprecated(self): - with self.check_deprecated(): - pkgutil.ImpLoader("", "", "", "") - def test_get_loader_avoids_emulation(self): with check_warnings() as w: self.assertIsNotNone(pkgutil.get_loader("sys")) diff --git a/Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst b/Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst new file mode 100644 index 00000000000000..f67bffcb0ddc6c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-21-17-20-57.gh-issue-98040.3btbmA.rst @@ -0,0 +1 @@ +Remove the long-deprecated ``imp`` module. diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index e34854f7025798..cf8990a2df0a9b 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -884,15 +884,3 @@ PyInit__test_module_state_shared(void) } return module; } - - -/*** Helper for imp test ***/ - -static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods); - -PyMODINIT_FUNC -PyInit_imp_dummy(void) -{ - return PyModuleDef_Init(&imp_dummy_def); -} - diff --git a/Python/import.c b/Python/import.c index daec64ea4adaad..0bf107b28d3990 100644 --- a/Python/import.c +++ b/Python/import.c @@ -594,11 +594,11 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) /* It may help to have a big picture view of what happens when an extension is loaded. This includes when it is imported - for the first time or via imp.load_dynamic(). + for the first time. - Here's a summary, using imp.load_dynamic() as the starting point: + Here's a summary, using importlib._boostrap._load() as a starting point. - 1. imp.load_dynamic() -> importlib._bootstrap._load() + 1. importlib._bootstrap._load() 2. _load(): acquire import lock 3. _load() -> importlib._bootstrap._load_unlocked() 4. _load_unlocked() -> importlib._bootstrap.module_from_spec() @@ -3794,7 +3794,7 @@ _imp_source_hash_impl(PyObject *module, long key, Py_buffer *source) PyDoc_STRVAR(doc_imp, -"(Extremely) low-level import machinery bits as used by importlib and imp."); +"(Extremely) low-level import machinery bits as used by importlib."); static PyMethodDef imp_methods[] = { _IMP_EXTENSION_SUFFIXES_METHODDEF diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 5aa31803397ce4..2b402ae0b6a031 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -7,24 +7,18 @@ import sys -try: - from importlib.machinery import SourceFileLoader -except ImportError: - import imp - - def find_module(modname): - """Finds and returns a module in the local dist/checkout. - """ - modpath = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "Lib") - return imp.load_module(modname, *imp.find_module(modname, [modpath])) -else: - def find_module(modname): - """Finds and returns a module in the local dist/checkout. - """ - modpath = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "Lib", modname + ".py") - return SourceFileLoader(modname, modpath).load_module() +# 2023-04-27(warsaw): Pre-Python 3.12, this would catch ImportErrors and try to +# import imp, and then use imp.load_module(). The imp module was removed in +# Python 3.12 (and long deprecated before that), and it's unclear under what +# conditions this import will now fail, so the fallback was simply removed. +from importlib.machinery import SourceFileLoader + +def find_module(modname): + """Finds and returns a module in the local dist/checkout. + """ + modpath = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "Lib", modname + ".py") + return SourceFileLoader(modname, modpath).load_module() def write_contents(f): diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d525fb1075c466..ba248d208e425a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2173,10 +2173,9 @@ add_main_module(PyInterpreterState *interp) Py_DECREF(bimod); } - /* Main is a little special - imp.is_builtin("__main__") will return - * False, but BuiltinImporter is still the most appropriate initial - * setting for its __loader__ attribute. A more suitable value will - * be set if __main__ gets further initialized later in the startup + /* Main is a little special - BuiltinImporter is the most appropriate + * initial setting for its __loader__ attribute. A more suitable value + * will be set if __main__ gets further initialized later in the startup * process. */ loader = _PyDict_GetItemStringWithError(d, "__loader__"); diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index e9f0061a59d3ba..27f42e5202e571 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -164,7 +164,6 @@ static const char* _Py_stdlib_module_names[] = { "idlelib", "imaplib", "imghdr", -"imp", "importlib", "inspect", "io", diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index d15e5e2d5450d7..7e0e9602a10765 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -1,5 +1,5 @@ # This script lists the names of standard library modules -# to update Python/stdlib_mod_names.h +# to update Python/stdlib_module_names.h import _imp import os.path import re diff --git a/Tools/c-analyzer/TODO b/Tools/c-analyzer/TODO index 43760369b1980e..27a535814ea52b 100644 --- a/Tools/c-analyzer/TODO +++ b/Tools/c-analyzer/TODO @@ -495,7 +495,6 @@ Python/import.c:PyImport_ImportModuleLevelObject():PyId___path__ _Py_IDENTIFIER( Python/import.c:PyImport_ImportModuleLevelObject():PyId___spec__ _Py_IDENTIFIER(__spec__) Python/import.c:PyImport_ImportModuleLevelObject():PyId__handle_fromlist _Py_IDENTIFIER(_handle_fromlist) Python/import.c:PyImport_ImportModuleLevelObject():PyId__lock_unlock_module _Py_IDENTIFIER(_lock_unlock_module) -Python/import.c:PyImport_ReloadModule():PyId_imp _Py_IDENTIFIER(imp) Python/import.c:PyImport_ReloadModule():PyId_reload _Py_IDENTIFIER(reload) Python/import.c:_PyImportZip_Init():PyId_zipimporter _Py_IDENTIFIER(zipimporter) Python/import.c:import_find_and_load():PyId__find_and_load _Py_IDENTIFIER(_find_and_load) diff --git a/Tools/importbench/importbench.py b/Tools/importbench/importbench.py index 6c4a537ad86e6c..619263b553c081 100644 --- a/Tools/importbench/importbench.py +++ b/Tools/importbench/importbench.py @@ -6,7 +6,7 @@ """ from test.test_importlib import util import decimal -import imp +from importlib.util import cache_from_source import importlib import importlib.machinery import json @@ -65,7 +65,7 @@ def source_wo_bytecode(seconds, repeat): name = '__importlib_test_benchmark__' # Clears out sys.modules and puts an entry at the front of sys.path. with util.create_modules(name) as mapping: - assert not os.path.exists(imp.cache_from_source(mapping[name])) + assert not os.path.exists(cache_from_source(mapping[name])) sys.meta_path.append(importlib.machinery.PathFinder) loader = (importlib.machinery.SourceFileLoader, importlib.machinery.SOURCE_SUFFIXES) @@ -80,7 +80,7 @@ def _wo_bytecode(module): name = module.__name__ def benchmark_wo_bytecode(seconds, repeat): """Source w/o bytecode: {}""" - bytecode_path = imp.cache_from_source(module.__file__) + bytecode_path = cache_from_source(module.__file__) if os.path.exists(bytecode_path): os.unlink(bytecode_path) sys.dont_write_bytecode = True @@ -108,9 +108,9 @@ def source_writing_bytecode(seconds, repeat): sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) def cleanup(): sys.modules.pop(name) - os.unlink(imp.cache_from_source(mapping[name])) + os.unlink(cache_from_source(mapping[name])) for result in bench(name, cleanup, repeat=repeat, seconds=seconds): - assert not os.path.exists(imp.cache_from_source(mapping[name])) + assert not os.path.exists(cache_from_source(mapping[name])) yield result @@ -121,7 +121,7 @@ def writing_bytecode_benchmark(seconds, repeat): assert not sys.dont_write_bytecode def cleanup(): sys.modules.pop(name) - os.unlink(imp.cache_from_source(module.__file__)) + os.unlink(cache_from_source(module.__file__)) yield from bench(name, cleanup, repeat=repeat, seconds=seconds) writing_bytecode_benchmark.__doc__ = ( @@ -141,7 +141,7 @@ def source_using_bytecode(seconds, repeat): importlib.machinery.SOURCE_SUFFIXES) sys.path_hooks.append(importlib.machinery.FileFinder.path_hook(loader)) py_compile.compile(mapping[name]) - assert os.path.exists(imp.cache_from_source(mapping[name])) + assert os.path.exists(cache_from_source(mapping[name])) yield from bench(name, lambda: sys.modules.pop(name), repeat=repeat, seconds=seconds) From 738c226786997262b76557d2dadd2beb89ea3fd1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 29 Apr 2023 05:51:55 +0100 Subject: [PATCH 06/26] GH-103082: Code cleanup in instrumentation code (#103474) --- Include/internal/pycore_frame.h | 6 +-- Python/instrumentation.c | 65 +++++++++++++++++---------------- Python/pystate.c | 8 ++-- Python/specialize.c | 2 +- 4 files changed, 42 insertions(+), 39 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 20d48d20362571..d8d7fe9ef2ebde 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -145,9 +145,9 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) } /* Fetches the stack pointer, and sets stacktop to -1. - Having stacktop <= 0 ensures that invalid - values are not visible to the cycle GC. - We choose -1 rather than 0 to assist debugging. */ + Having stacktop <= 0 ensures that invalid + values are not visible to the cycle GC. + We choose -1 rather than 0 to assist debugging. */ static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 8334f596eb3e19..c5bbbdacbb851e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -113,13 +113,17 @@ static const uint8_t INSTRUMENTED_OPCODES[256] = { }; static inline bool -opcode_has_event(int opcode) { - return opcode < INSTRUMENTED_LINE && - INSTRUMENTED_OPCODES[opcode] > 0; +opcode_has_event(int opcode) +{ + return ( + opcode < INSTRUMENTED_LINE && + INSTRUMENTED_OPCODES[opcode] > 0 + ); } static inline bool -is_instrumented(int opcode) { +is_instrumented(int opcode) +{ assert(opcode != 0); assert(opcode != RESERVED); return opcode >= MIN_INSTRUMENTED_OPCODE; @@ -339,7 +343,8 @@ dump_monitors(const char *prefix, _Py_Monitors monitors, FILE*out) /* Like _Py_GetBaseOpcode but without asserts. * Does its best to give the right answer, but won't abort * if something is wrong */ -int get_base_opcode_best_attempt(PyCodeObject *code, int offset) +static int +get_base_opcode_best_attempt(PyCodeObject *code, int offset) { int opcode = _Py_OPCODE(_PyCode_CODE(code)[offset]); if (INSTRUMENTED_OPCODES[opcode] != opcode) { @@ -418,13 +423,15 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) assert(test); \ } while (0) -bool valid_opcode(int opcode) { +static bool +valid_opcode(int opcode) +{ if (opcode > 0 && opcode != RESERVED && opcode < 255 && _PyOpcode_OpName[opcode] && - _PyOpcode_OpName[opcode][0] != '<' - ) { + _PyOpcode_OpName[opcode][0] != '<') + { return true; } return false; @@ -550,11 +557,11 @@ de_instrument(PyCodeObject *code, int i, int event) opcode_ptr = &code->_co_monitoring->lines[i].original_opcode; opcode = *opcode_ptr; } - if (opcode == INSTRUMENTED_INSTRUCTION) { + if (opcode == INSTRUMENTED_INSTRUCTION) { opcode_ptr = &code->_co_monitoring->per_instruction_opcodes[i]; opcode = *opcode_ptr; } - int deinstrumented = DE_INSTRUMENT[opcode]; + int deinstrumented = DE_INSTRUMENT[opcode]; if (deinstrumented == 0) { return; } @@ -781,8 +788,7 @@ add_line_tools(PyCodeObject * code, int offset, int tools) { assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_LINE, tools)); assert(code->_co_monitoring); - if (code->_co_monitoring->line_tools - ) { + if (code->_co_monitoring->line_tools) { code->_co_monitoring->line_tools[offset] |= tools; } else { @@ -798,8 +804,7 @@ add_per_instruction_tools(PyCodeObject * code, int offset, int tools) { assert(tools_is_subset_for_event(code, PY_MONITORING_EVENT_INSTRUCTION, tools)); assert(code->_co_monitoring); - if (code->_co_monitoring->per_instruction_tools - ) { + if (code->_co_monitoring->per_instruction_tools) { code->_co_monitoring->per_instruction_tools[offset] |= tools; } else { @@ -814,11 +819,10 @@ static void remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) { assert(code->_co_monitoring); - if (code->_co_monitoring->per_instruction_tools) - { + if (code->_co_monitoring->per_instruction_tools) { uint8_t *toolsptr = &code->_co_monitoring->per_instruction_tools[offset]; *toolsptr &= ~tools; - if (*toolsptr == 0 ) { + if (*toolsptr == 0) { de_instrument_per_instruction(code, offset); } } @@ -843,7 +847,7 @@ call_one_instrument( assert(tstate->tracing == 0); PyObject *instrument = interp->monitoring_callables[tool][event]; if (instrument == NULL) { - return 0; + return 0; } int old_what = tstate->what_event; tstate->what_event = event; @@ -865,16 +869,15 @@ static const int8_t MOST_SIGNIFICANT_BITS[16] = { 3, 3, 3, 3, }; -/* We could use _Py_bit_length here, but that is designed for larger (32/64) bit ints, - and can perform relatively poorly on platforms without the necessary intrinsics. */ +/* We could use _Py_bit_length here, but that is designed for larger (32/64) + * bit ints, and can perform relatively poorly on platforms without the + * necessary intrinsics. */ static inline int most_significant_bit(uint8_t bits) { assert(bits != 0); if (bits > 15) { return MOST_SIGNIFICANT_BITS[bits>>4]+4; } - else { - return MOST_SIGNIFICANT_BITS[bits]; - } + return MOST_SIGNIFICANT_BITS[bits]; } static bool @@ -1002,8 +1005,8 @@ _Py_call_instrumentation_2args( int _Py_call_instrumentation_jump( PyThreadState *tstate, int event, - _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target -) { + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target) +{ assert(event == PY_MONITORING_EVENT_JUMP || event == PY_MONITORING_EVENT_BRANCH); assert(frame->prev_instr == instr); @@ -1309,8 +1312,8 @@ initialize_line_tools(PyCodeObject *code, _Py_Monitors *all_events) } } -static -int allocate_instrumentation_data(PyCodeObject *code) +static int +allocate_instrumentation_data(PyCodeObject *code) { if (code->_co_monitoring == NULL) { @@ -1404,7 +1407,7 @@ static const uint8_t super_instructions[256] = { /* Should use instruction metadata for this */ static bool -is_super_instruction(int opcode) { +is_super_instruction(uint8_t opcode) { return super_instructions[opcode] != 0; } @@ -1516,7 +1519,7 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) #define C_RETURN_EVENTS \ ((1 << PY_MONITORING_EVENT_C_RETURN) | \ - (1 << PY_MONITORING_EVENT_C_RAISE)) + (1 << PY_MONITORING_EVENT_C_RAISE)) #define C_CALL_EVENTS \ (C_RETURN_EVENTS | (1 << PY_MONITORING_EVENT_CALL)) @@ -1561,8 +1564,8 @@ static int check_tool(PyInterpreterState *interp, int tool_id) { if (tool_id < PY_MONITORING_SYS_PROFILE_ID && - interp->monitoring_tool_names[tool_id] == NULL - ) { + interp->monitoring_tool_names[tool_id] == NULL) + { PyErr_Format(PyExc_ValueError, "tool %d is not in use", tool_id); return -1; } diff --git a/Python/pystate.c b/Python/pystate.c index ffab301f3171b2..f103a059f0f369 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -686,11 +686,11 @@ init_interpreter(PyInterpreterState *interp, _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); - for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; } for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - for(int e = 0; e < PY_MONITORING_EVENTS; e++) { + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { interp->monitoring_callables[t][e] = NULL; } @@ -834,11 +834,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); - for(int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { + for (int i = 0; i < PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; } for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { - for(int e = 0; e < PY_MONITORING_EVENTS; e++) { + for (int e = 0; e < PY_MONITORING_EVENTS; e++) { Py_CLEAR(interp->monitoring_callables[t][e]); } } diff --git a/Python/specialize.c b/Python/specialize.c index fbdb435082cece..b1cc66124cfa4a 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -148,7 +148,7 @@ print_spec_stats(FILE *out, OpcodeStats *stats) PRIu64 "\n", i, j, val); } } - for(int j = 0; j < 256; j++) { + for (int j = 0; j < 256; j++) { if (stats[i].pair_count[j]) { fprintf(out, "opcode[%d].pair_count[%d] : %" PRIu64 "\n", i, j, stats[i].pair_count[j]); From ed29f524cf7d2fd11d420604f80f1d4cf0079891 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 29 Apr 2023 00:02:21 -0700 Subject: [PATCH 07/26] Various small fixes to dis docs (#103923) - Fix description of MAKE_CELL, which appeared to be inverted from the actual behavior - Fix stray ".:" (sphinx-contrib/sphinx-lint#63) - Fix inconsistent indentation - Add some missing code blocks - Slight style improvements --- Doc/library/dis.rst | 157 ++++++++++++++++++++++---------------------- 1 file changed, 79 insertions(+), 78 deletions(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 3894837127877c..6c3f436ddb1494 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -402,7 +402,7 @@ The Python compiler currently generates the following bytecode instructions. **General instructions** -In the following, We will refer to the interpreter stack as STACK and describe +In the following, We will refer to the interpreter stack as ``STACK`` and describe operations on it as if it was a Python list. The top of the stack corresponds to ``STACK[-1]`` in this language. @@ -414,7 +414,7 @@ operations on it as if it was a Python list. The top of the stack corresponds to .. opcode:: POP_TOP - Removes the top-of-stack item.:: + Removes the top-of-stack item:: STACK.pop() @@ -422,7 +422,7 @@ operations on it as if it was a Python list. The top of the stack corresponds to .. opcode:: END_FOR Removes the top two values from the stack. - Equivalent to POP_TOP; POP_TOP. + Equivalent to ``POP_TOP``; ``POP_TOP``. Used to clean up at the end of loops, hence the name. .. versionadded:: 3.12 @@ -431,7 +431,7 @@ operations on it as if it was a Python list. The top of the stack corresponds to .. opcode:: COPY (i) Push the i-th item to the top of the stack without removing it from its original - location.:: + location:: assert i > 0 STACK.append(STACK[-i]) @@ -441,7 +441,7 @@ operations on it as if it was a Python list. The top of the stack corresponds to .. opcode:: SWAP (i) - Swap the top of the stack with the i-th element.:: + Swap the top of the stack with the i-th element:: STACK[-i], STACK[-1] = stack[-1], STACK[-i] @@ -513,7 +513,7 @@ not have to be) the original ``STACK[-2]``. .. opcode:: BINARY_OP (op) Implements the binary and in-place operators (depending on the value of - *op*).:: + *op*):: rhs = STACK.pop() lhs = STACK.pop() @@ -580,14 +580,14 @@ not have to be) the original ``STACK[-2]``. Implements ``STACK[-1] = get_awaitable(STACK[-1])``, where ``get_awaitable(o)`` returns ``o`` if ``o`` is a coroutine object or a generator object with - the CO_ITERABLE_COROUTINE flag, or resolves + the :data:`~inspect.CO_ITERABLE_COROUTINE` flag, or resolves ``o.__await__``. If the ``where`` operand is nonzero, it indicates where the instruction occurs: - * ``1`` After a call to ``__aenter__`` - * ``2`` After a call to ``__aexit__`` + * ``1``: After a call to ``__aenter__`` + * ``2``: After a call to ``__aexit__`` .. versionadded:: 3.5 @@ -652,6 +652,7 @@ not have to be) the original ``STACK[-2]``. .. opcode:: SET_ADD (i) Implements:: + item = STACK.pop() set.add(STACK[-i], item) @@ -705,11 +706,11 @@ iterations of the loop. Yields ``STACK.pop()`` from a :term:`generator`. - .. versionchanged:: 3.11 - oparg set to be the stack depth. + .. versionchanged:: 3.11 + oparg set to be the stack depth. - .. versionchanged:: 3.12 - oparg set to be the exception block depth, for efficient closing of generators. + .. versionchanged:: 3.12 + oparg set to be the exception block depth, for efficient closing of generators. .. opcode:: SETUP_ANNOTATIONS @@ -726,32 +727,32 @@ iterations of the loop. Pops a value from the stack, which is used to restore the exception state. - .. versionchanged:: 3.11 - Exception representation on the stack now consist of one, not three, items. + .. versionchanged:: 3.11 + Exception representation on the stack now consist of one, not three, items. .. opcode:: RERAISE - Re-raises the exception currently on top of the stack. If oparg is non-zero, - pops an additional value from the stack which is used to set ``f_lasti`` - of the current frame. + Re-raises the exception currently on top of the stack. If oparg is non-zero, + pops an additional value from the stack which is used to set ``f_lasti`` + of the current frame. - .. versionadded:: 3.9 + .. versionadded:: 3.9 - .. versionchanged:: 3.11 - Exception representation on the stack now consist of one, not three, items. + .. versionchanged:: 3.11 + Exception representation on the stack now consist of one, not three, items. .. opcode:: PUSH_EXC_INFO - Pops a value from the stack. Pushes the current exception to the top of the stack. - Pushes the value originally popped back to the stack. - Used in exception handlers. + Pops a value from the stack. Pushes the current exception to the top of the stack. + Pushes the value originally popped back to the stack. + Used in exception handlers. - .. versionadded:: 3.11 + .. versionadded:: 3.11 .. opcode:: CHECK_EXC_MATCH Performs exception matching for ``except``. Tests whether the ``STACK[-2]`` - is an exception matching ``STACK[-1]``. Pops STACK[-1] and pushes the boolean + is an exception matching ``STACK[-1]``. Pops ``STACK[-1]`` and pushes the boolean result of the test. .. versionadded:: 3.11 @@ -770,16 +771,16 @@ iterations of the loop. .. opcode:: WITH_EXCEPT_START - Calls the function in position 4 on the stack with arguments (type, val, tb) - representing the exception at the top of the stack. - Used to implement the call ``context_manager.__exit__(*exc_info())`` when an exception - has occurred in a :keyword:`with` statement. + Calls the function in position 4 on the stack with arguments (type, val, tb) + representing the exception at the top of the stack. + Used to implement the call ``context_manager.__exit__(*exc_info())`` when an exception + has occurred in a :keyword:`with` statement. - .. versionadded:: 3.9 + .. versionadded:: 3.9 - .. versionchanged:: 3.11 - The ``__exit__`` function is in position 4 of the stack rather than 7. - Exception representation on the stack now consist of one, not three, items. + .. versionchanged:: 3.11 + The ``__exit__`` function is in position 4 of the stack rather than 7. + Exception representation on the stack now consist of one, not three, items. .. opcode:: LOAD_ASSERTION_ERROR @@ -863,7 +864,7 @@ iterations of the loop. .. opcode:: UNPACK_SEQUENCE (count) Unpacks ``STACK[-1]`` into *count* individual values, which are put onto the stack - right-to-left.:: + right-to-left:: STACK.extend(STACK.pop()[:count:-1]) @@ -1028,7 +1029,7 @@ iterations of the loop. This bytecode distinguishes two cases: if ``STACK[-1]`` has a method with the correct name, the bytecode pushes the unbound method and ``STACK[-1]``. ``STACK[-1]`` will be used as the first argument (``self``) by :opcode:`CALL` - when calling the unbound method. Otherwise, ``NULL`` and the object return by + when calling the unbound method. Otherwise, ``NULL`` and the object returned by the attribute lookup are pushed. .. versionchanged:: 3.12 @@ -1207,7 +1208,7 @@ iterations of the loop. .. opcode:: MAKE_CELL (i) - Creates a new cell in slot ``i``. If that slot is empty then + Creates a new cell in slot ``i``. If that slot is nonempty then that value is stored into the new cell. .. versionadded:: 3.11 @@ -1332,9 +1333,9 @@ iterations of the loop. .. opcode:: PUSH_NULL - Pushes a ``NULL`` to the stack. - Used in the call sequence to match the ``NULL`` pushed by - :opcode:`LOAD_METHOD` for non-method calls. + Pushes a ``NULL`` to the stack. + Used in the call sequence to match the ``NULL`` pushed by + :opcode:`LOAD_METHOD` for non-method calls. .. versionadded:: 3.11 @@ -1434,38 +1435,38 @@ iterations of the loop. .. opcode:: RESUME (where) - A no-op. Performs internal tracing, debugging and optimization checks. + A no-op. Performs internal tracing, debugging and optimization checks. - The ``where`` operand marks where the ``RESUME`` occurs: + The ``where`` operand marks where the ``RESUME`` occurs: - * ``0`` The start of a function, which is neither a generator, coroutine - nor an async generator - * ``1`` After a ``yield`` expression - * ``2`` After a ``yield from`` expression - * ``3`` After an ``await`` expression + * ``0`` The start of a function, which is neither a generator, coroutine + nor an async generator + * ``1`` After a ``yield`` expression + * ``2`` After a ``yield from`` expression + * ``3`` After an ``await`` expression .. versionadded:: 3.11 .. opcode:: RETURN_GENERATOR - Create a generator, coroutine, or async generator from the current frame. - Used as first opcode of in code object for the above mentioned callables. - Clear the current frame and return the newly created generator. + Create a generator, coroutine, or async generator from the current frame. + Used as first opcode of in code object for the above mentioned callables. + Clear the current frame and return the newly created generator. - .. versionadded:: 3.11 + .. versionadded:: 3.11 .. opcode:: SEND (delta) - Equivalent to ``STACK[-1] = STACK[-2].send(STACK[-1])``. Used in ``yield from`` - and ``await`` statements. + Equivalent to ``STACK[-1] = STACK[-2].send(STACK[-1])``. Used in ``yield from`` + and ``await`` statements. - If the call raises :exc:`StopIteration`, pop both items, push the - exception's ``value`` attribute, and increment the bytecode counter by - *delta*. + If the call raises :exc:`StopIteration`, pop both items, push the + exception's ``value`` attribute, and increment the bytecode counter by + *delta*. - .. versionadded:: 3.11 + .. versionadded:: 3.11 .. opcode:: HAVE_ARGUMENT @@ -1493,15 +1494,15 @@ iterations of the loop. argument and sets ``STACK[-1]`` to the result. Used to implement functionality that is necessary but not performance critical. - The operand determines which intrinsic function is called: + The operand determines which intrinsic function is called: - * ``0`` Not valid - * ``1`` Prints the argument to standard out. Used in the REPL. - * ``2`` Performs ``import *`` for the named module. - * ``3`` Extracts the return value from a ``StopIteration`` exception. - * ``4`` Wraps an aync generator value - * ``5`` Performs the unary ``+`` operation - * ``6`` Converts a list to a tuple + * ``0`` Not valid + * ``1`` Prints the argument to standard out. Used in the REPL. + * ``2`` Performs ``import *`` for the named module. + * ``3`` Extracts the return value from a ``StopIteration`` exception. + * ``4`` Wraps an aync generator value + * ``5`` Performs the unary ``+`` operation + * ``6`` Converts a list to a tuple .. versionadded:: 3.12 @@ -1511,17 +1512,17 @@ iterations of the loop. arguments and sets ``STACK[-1]`` to the result. Used to implement functionality that is necessary but not performance critical. - The operand determines which intrinsic function is called: + The operand determines which intrinsic function is called: - * ``0`` Not valid - * ``1`` Calculates the :exc:`ExceptionGroup` to raise from a ``try-except*``. + * ``0`` Not valid + * ``1`` Calculates the :exc:`ExceptionGroup` to raise from a ``try-except*``. .. versionadded:: 3.12 **Pseudo-instructions** -These opcodes do not appear in python bytecode, they are used by the compiler +These opcodes do not appear in Python bytecode. They are used by the compiler but are replaced by real opcodes or removed before bytecode is generated. .. opcode:: SETUP_FINALLY (target) @@ -1533,7 +1534,7 @@ but are replaced by real opcodes or removed before bytecode is generated. .. opcode:: SETUP_CLEANUP (target) - Like ``SETUP_FINALLY``, but in case of exception also pushes the last + Like ``SETUP_FINALLY``, but in case of an exception also pushes the last instruction (``lasti``) to the stack so that ``RERAISE`` can restore it. If an exception occurs, the value stack level and the last instruction on the frame are restored to their current state, and control is transferred @@ -1542,7 +1543,7 @@ but are replaced by real opcodes or removed before bytecode is generated. .. opcode:: SETUP_WITH (target) - Like ``SETUP_CLEANUP``, but in case of exception one more item is popped + Like ``SETUP_CLEANUP``, but in case of an exception one more item is popped from the stack before control is transferred to the exception handler at ``target``. @@ -1576,9 +1577,9 @@ Opcode collections These collections are provided for automatic introspection of bytecode instructions: - .. versionchanged:: 3.12 - The collections now contain pseudo instructions as well. These are - opcodes with values ``>= MIN_PSEUDO_OPCODE``. +.. versionchanged:: 3.12 + The collections now contain pseudo instructions as well. These are + opcodes with values ``>= MIN_PSEUDO_OPCODE``. .. data:: opname @@ -1599,7 +1600,7 @@ instructions: Sequence of bytecodes that use their argument. - .. versionadded:: 3.12 + .. versionadded:: 3.12 .. data:: hasconst @@ -1609,10 +1610,10 @@ instructions: .. data:: hasfree - Sequence of bytecodes that access a free variable (note that 'free' in this + Sequence of bytecodes that access a free variable. 'free' in this context refers to names in the current scope that are referenced by inner scopes or names in outer scopes that are referenced from this scope. It does - *not* include references to global or builtin scopes). + *not* include references to global or builtin scopes. .. data:: hasname @@ -1643,4 +1644,4 @@ instructions: Sequence of bytecodes that set an exception handler. - .. versionadded:: 3.12 + .. versionadded:: 3.12 From 84e7d0f0c7f9a44d81be2d705ed4d401a6505356 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Sat, 29 Apr 2023 12:46:46 +0530 Subject: [PATCH 08/26] gh-103636: issue warning for deprecated calendar constants (#103833) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo van Kemenade Co-authored-by: Éric Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Doc/library/calendar.rst | 52 +++++++++++++++++++ Doc/whatsnew/3.12.rst | 9 ++++ Lib/calendar.py | 13 +++++ Lib/test/test_calendar.py | 9 ++++ ...-04-26-18-12-13.gh-issue-103636.-KvCgO.rst | 1 + 5 files changed, 84 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 66f59f0e2ced27..07d04a1c7b582a 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -28,6 +28,58 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is 2 BC, and so on. +.. class:: Day + + Enumeration defining the days of the week as integer constants, from 0 to 6. + + .. attribute:: MONDAY + + .. attribute:: TUESDAY + + .. attribute:: WEDNESDAY + + .. attribute:: THURSDAY + + .. attribute:: FRIDAY + + .. attribute:: SATURDAY + + .. attribute:: SUNDAY + + .. versionadded:: 3.12 + + +.. class:: Month + + Enumeration defining months of the year as integer constants, from 1 to 12. + + .. attribute:: JANUARY + + .. attribute:: FEBRUARY + + .. attribute:: MARCH + + .. attribute:: APRIL + + .. attribute:: MAY + + .. attribute:: JUNE + + .. attribute:: JULY + + .. attribute:: AUGUST + + .. attribute:: SEPTEMBER + + .. attribute:: OCTOBER + + .. attribute:: NOVEMBER + + .. attribute:: DECEMBER + + .. versionadded:: 3.12 + + .. class:: Calendar(firstweekday=0) Creates a :class:`Calendar` object. *firstweekday* is an integer specifying the diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a75e88c2615aca..908cf3bb365691 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -300,6 +300,12 @@ asyncio yielding tasks. (Contributed by Kumar Aditya in :gh:`78530`.) +calendar +-------- + +* Add enums :data:`~calendar.Month` and :data:`~calendar.Day`. + (Contributed by Prince Roshan in :gh:`103636`.) + csv --- @@ -692,6 +698,9 @@ Deprecated Python 3.14, when ``'data'`` filter will become the default. See :ref:`tarfile-extraction-filter` for details. +* ``calendar.January`` and ``calendar.February`` constants are deprecated and + replaced by :data:`calendar.Month.JANUARY` and :data:`calendar.Month.FEBRUARY`. + (Contributed by Prince Roshan in :gh:`103636`.) Pending Removal in Python 3.13 ------------------------------ diff --git a/Lib/calendar.py b/Lib/calendar.py index c219f0c218d9fa..bbd4fea3b88ca4 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -10,6 +10,7 @@ from enum import IntEnum, global_enum import locale as _locale from itertools import repeat +import warnings __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", @@ -41,6 +42,18 @@ def __str__(self): return "bad weekday number %r; must be 0 (Monday) to 6 (Sunday)" % self.weekday +def __getattr__(name): + if name in ('January', 'February'): + warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead", + DeprecationWarning, stacklevel=2) + if name == 'January': + return 1 + else: + return 2 + + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") + + # Constants for months @global_enum class Month(IntEnum): diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index ccfbeede0be949..03388e8c55d5a8 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -8,6 +8,7 @@ import sys import datetime import os +import warnings # From https://en.wikipedia.org/wiki/Leap_year_starting_on_Saturday result_0_02_text = """\ @@ -490,6 +491,14 @@ def test_format(self): self.assertEqual(out.getvalue().strip(), "1 2 3") class CalendarTestCase(unittest.TestCase): + + def test_deprecation_warning(self): + with warnings.catch_warnings(record=True) as w: + calendar.January + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, DeprecationWarning) + self.assertIn("The 'January' attribute is deprecated, use 'JANUARY' instead", str(w[0].message)) + def test_isleap(self): # Make sure that the return is right for a few years, and # ensure that the return values are 1 or 0, not just true or diff --git a/Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst b/Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst new file mode 100644 index 00000000000000..a05a6f5cbcdb99 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-26-18-12-13.gh-issue-103636.-KvCgO.rst @@ -0,0 +1 @@ +Module-level attributes ``January`` and ``February`` are deprecated from :mod:`calendar`. From fbf3596c3edadd03b5a8c659e9f27a09e5d1a051 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Sat, 29 Apr 2023 12:06:04 +0100 Subject: [PATCH 09/26] gh-87092: change assembler to use instruction sequence instead of CFG (#103933) --- Include/internal/pycore_compile.h | 13 +++- Include/internal/pycore_flowgraph.h | 4 +- Python/assemble.c | 95 ++++++++++++++--------------- Python/compile.c | 40 ++++++++---- Python/flowgraph.c | 20 +++--- 5 files changed, 92 insertions(+), 80 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index f85240c48a89b0..1a032f652dddaf 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -19,6 +19,7 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( int optimize, struct _arena *arena); +static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; typedef struct { int optimize; @@ -33,15 +34,21 @@ extern int _PyAST_Optimize( struct _arena *arena, _PyASTOptimizeState *state); +typedef struct { + int h_offset; + int h_startdepth; + int h_preserve_lasti; +} _PyCompile_ExceptHandlerInfo; typedef struct { int i_opcode; int i_oparg; _PyCompilerSrcLocation i_loc; -} _PyCompilerInstruction; + _PyCompile_ExceptHandlerInfo i_except_handler_info; +} _PyCompile_Instruction; typedef struct { - _PyCompilerInstruction *s_instrs; + _PyCompile_Instruction *s_instrs; int s_allocated; int s_used; @@ -82,6 +89,8 @@ int _PyCompile_EnsureArrayLargeEnough( int _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj); +int _PyCompile_InstrSize(int opcode, int oparg); + /* Access compiler internals for unit testing */ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index f470dad3aaa459..883334f4b182eb 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -11,7 +11,6 @@ extern "C" { #include "pycore_opcode_utils.h" #include "pycore_compile.h" -static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; typedef struct { int i_opcode; @@ -97,7 +96,6 @@ int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_ int _PyCfg_Stackdepth(_PyCfgBasicblock *entryblock, int code_flags); void _PyCfg_ConvertExceptionHandlersToNops(_PyCfgBasicblock *entryblock); int _PyCfg_ResolveJumps(_PyCfgBuilder *g); -int _PyCfg_InstrSize(_PyCfgInstruction *instruction); static inline int @@ -113,7 +111,7 @@ basicblock_nofallthrough(const _PyCfgBasicblock *b) { PyCodeObject * _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache, - PyObject *consts, int maxdepth, _PyCfgBasicblock *entryblock, + PyObject *consts, int maxdepth, _PyCompile_InstructionSequence *instrs, int nlocalsplus, int code_flags, PyObject *filename); #ifdef __cplusplus diff --git a/Python/assemble.c b/Python/assemble.c index e5a361b230cf1c..369dd8dcde9b9b 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -1,10 +1,10 @@ #include #include "Python.h" -#include "pycore_flowgraph.h" +#include "pycore_code.h" // write_location_entry_start() #include "pycore_compile.h" +#include "pycore_opcode.h" // _PyOpcode_Caches[] and opcode category macros #include "pycore_pymem.h" // _PyMem_IsPtrFreed() -#include "pycore_code.h" // write_location_entry_start() #define DEFAULT_CODE_SIZE 128 @@ -22,8 +22,8 @@ } typedef _PyCompilerSrcLocation location; -typedef _PyCfgInstruction cfg_instr; -typedef _PyCfgBasicblock basicblock; +typedef _PyCompile_Instruction instruction; +typedef _PyCompile_InstructionSequence instr_sequence; static inline bool same_location(location a, location b) @@ -117,7 +117,8 @@ assemble_emit_exception_table_item(struct assembler *a, int value, int msb) #define MAX_SIZE_OF_ENTRY 20 static int -assemble_emit_exception_table_entry(struct assembler *a, int start, int end, basicblock *handler) +assemble_emit_exception_table_entry(struct assembler *a, int start, int end, + _PyCompile_ExceptHandlerInfo *handler) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_except_table); if (a->a_except_table_off + MAX_SIZE_OF_ENTRY >= len) { @@ -125,13 +126,13 @@ assemble_emit_exception_table_entry(struct assembler *a, int start, int end, bas } int size = end-start; assert(end > start); - int target = handler->b_offset; - int depth = handler->b_startdepth - 1; - if (handler->b_preserve_lasti) { + int target = handler->h_offset; + int depth = handler->h_startdepth - 1; + if (handler->h_preserve_lasti) { depth -= 1; } assert(depth >= 0); - int depth_lasti = (depth<<1) | handler->b_preserve_lasti; + int depth_lasti = (depth<<1) | handler->h_preserve_lasti; assemble_emit_exception_table_item(a, start, (1<<7)); assemble_emit_exception_table_item(a, size, 0); assemble_emit_exception_table_item(a, target, 0); @@ -140,29 +141,26 @@ assemble_emit_exception_table_entry(struct assembler *a, int start, int end, bas } static int -assemble_exception_table(struct assembler *a, basicblock *entryblock) +assemble_exception_table(struct assembler *a, instr_sequence *instrs) { - basicblock *b; int ioffset = 0; - basicblock *handler = NULL; + _PyCompile_ExceptHandlerInfo handler; + handler.h_offset = -1; int start = -1; - for (b = entryblock; b != NULL; b = b->b_next) { - ioffset = b->b_offset; - for (int i = 0; i < b->b_iused; i++) { - cfg_instr *instr = &b->b_instr[i]; - if (instr->i_except != handler) { - if (handler != NULL) { - RETURN_IF_ERROR( - assemble_emit_exception_table_entry(a, start, ioffset, handler)); - } - start = ioffset; - handler = instr->i_except; + for (int i = 0; i < instrs->s_used; i++) { + instruction *instr = &instrs->s_instrs[i]; + if (instr->i_except_handler_info.h_offset != handler.h_offset) { + if (handler.h_offset >= 0) { + RETURN_IF_ERROR( + assemble_emit_exception_table_entry(a, start, ioffset, &handler)); } - ioffset += _PyCfg_InstrSize(instr); + start = ioffset; + handler = instr->i_except_handler_info; } + ioffset += _PyCompile_InstrSize(instr->i_opcode, instr->i_oparg); } - if (handler != NULL) { - RETURN_IF_ERROR(assemble_emit_exception_table_entry(a, start, ioffset, handler)); + if (handler.h_offset >= 0) { + RETURN_IF_ERROR(assemble_emit_exception_table_entry(a, start, ioffset, &handler)); } return SUCCESS; } @@ -316,31 +314,31 @@ assemble_emit_location(struct assembler* a, location loc, int isize) } static int -assemble_location_info(struct assembler *a, basicblock *entryblock, int firstlineno) +assemble_location_info(struct assembler *a, instr_sequence *instrs, + int firstlineno) { a->a_lineno = firstlineno; location loc = NO_LOCATION; int size = 0; - for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - for (int j = 0; j < b->b_iused; j++) { - if (!same_location(loc, b->b_instr[j].i_loc)) { + for (int i = 0; i < instrs->s_used; i++) { + instruction *instr = &instrs->s_instrs[i]; + if (!same_location(loc, instr->i_loc)) { RETURN_IF_ERROR(assemble_emit_location(a, loc, size)); - loc = b->b_instr[j].i_loc; + loc = instr->i_loc; size = 0; - } - size += _PyCfg_InstrSize(&b->b_instr[j]); } + size += _PyCompile_InstrSize(instr->i_opcode, instr->i_oparg); } RETURN_IF_ERROR(assemble_emit_location(a, loc, size)); return SUCCESS; } static void -write_instr(_Py_CODEUNIT *codestr, cfg_instr *instruction, int ilen) +write_instr(_Py_CODEUNIT *codestr, instruction *instr, int ilen) { - int opcode = instruction->i_opcode; + int opcode = instr->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); - int oparg = instruction->i_oparg; + int oparg = instr->i_oparg; assert(HAS_ARG(opcode) || oparg == 0); int caches = _PyOpcode_Caches[opcode]; switch (ilen - caches) { @@ -380,12 +378,12 @@ write_instr(_Py_CODEUNIT *codestr, cfg_instr *instruction, int ilen) */ static int -assemble_emit_instr(struct assembler *a, cfg_instr *i) +assemble_emit_instr(struct assembler *a, instruction *instr) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); _Py_CODEUNIT *code; - int size = _PyCfg_InstrSize(i); + int size = _PyCompile_InstrSize(instr->i_opcode, instr->i_oparg); if (a->a_offset + size >= len / (int)sizeof(_Py_CODEUNIT)) { if (len > PY_SSIZE_T_MAX / 2) { return ERROR; @@ -394,25 +392,24 @@ assemble_emit_instr(struct assembler *a, cfg_instr *i) } code = (_Py_CODEUNIT *)PyBytes_AS_STRING(a->a_bytecode) + a->a_offset; a->a_offset += size; - write_instr(code, i, size); + write_instr(code, instr, size); return SUCCESS; } static int -assemble_emit(struct assembler *a, basicblock *entryblock, int first_lineno, - PyObject *const_cache) +assemble_emit(struct assembler *a, instr_sequence *instrs, + int first_lineno, PyObject *const_cache) { RETURN_IF_ERROR(assemble_init(a, first_lineno)); - for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - for (int j = 0; j < b->b_iused; j++) { - RETURN_IF_ERROR(assemble_emit_instr(a, &b->b_instr[j])); - } + for (int i = 0; i < instrs->s_used; i++) { + instruction *instr = &instrs->s_instrs[i]; + RETURN_IF_ERROR(assemble_emit_instr(a, instr)); } - RETURN_IF_ERROR(assemble_location_info(a, entryblock, a->a_lineno)); + RETURN_IF_ERROR(assemble_location_info(a, instrs, a->a_lineno)); - RETURN_IF_ERROR(assemble_exception_table(a, entryblock)); + RETURN_IF_ERROR(assemble_exception_table(a, instrs)); RETURN_IF_ERROR(_PyBytes_Resize(&a->a_except_table, a->a_except_table_off)); RETURN_IF_ERROR(_PyCompile_ConstCacheMergeOne(const_cache, &a->a_except_table)); @@ -586,13 +583,13 @@ makecode(_PyCompile_CodeUnitMetadata *umd, struct assembler *a, PyObject *const_ PyCodeObject * _PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *umd, PyObject *const_cache, - PyObject *consts, int maxdepth, basicblock *entryblock, + PyObject *consts, int maxdepth, instr_sequence *instrs, int nlocalsplus, int code_flags, PyObject *filename) { PyCodeObject *co = NULL; struct assembler a; - int res = assemble_emit(&a, entryblock, umd->u_firstlineno, const_cache); + int res = assemble_emit(&a, instrs, umd->u_firstlineno, const_cache); if (res == SUCCESS) { co = makecode(umd, &a, const_cache, consts, maxdepth, nlocalsplus, code_flags, filename); diff --git a/Python/compile.c b/Python/compile.c index a0ad3687f586d8..e8789def867308 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -149,7 +149,18 @@ enum { COMPILER_SCOPE_COMPREHENSION, }; -typedef _PyCompilerInstruction instruction; + +int +_PyCompile_InstrSize(int opcode, int oparg) +{ + assert(!IS_PSEUDO_OPCODE(opcode)); + assert(HAS_ARG(opcode) || oparg == 0); + int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg); + int caches = _PyOpcode_Caches[opcode]; + return extended_args + 1 + caches; +} + +typedef _PyCompile_Instruction instruction; typedef _PyCompile_InstructionSequence instr_sequence; #define INITIAL_INSTR_SEQUENCE_SIZE 100 @@ -6968,10 +6979,6 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, goto error; } - if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) { - goto error; - } - /** Assembly **/ int nlocalsplus = prepare_localsplus(u, &g, code_flags); if (nlocalsplus < 0) { @@ -6990,15 +6997,15 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, if (_PyCfg_ResolveJumps(&g) < 0) { goto error; } + + /* Can't modify the bytecode after computing jump offsets. */ + if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) { goto error; } - - /* Can't modify the bytecode after computing jump offsets. */ - co = _PyAssemble_MakeCodeObject(&u->u_metadata, const_cache, consts, - maxdepth, g.g_entryblock, nlocalsplus, + maxdepth, &optimized_instrs, nlocalsplus, code_flags, filename); error: @@ -7039,11 +7046,18 @@ cfg_to_instr_sequence(cfg_builder *g, instr_sequence *seq) RETURN_IF_ERROR(instr_sequence_use_label(seq, b->b_label.id)); for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - int arg = HAS_TARGET(instr->i_opcode) ? - instr->i_target->b_label.id : - instr->i_oparg; RETURN_IF_ERROR( - instr_sequence_addop(seq, instr->i_opcode, arg, instr->i_loc)); + instr_sequence_addop(seq, instr->i_opcode, instr->i_oparg, instr->i_loc)); + + _PyCompile_ExceptHandlerInfo *hi = &seq->s_instrs[seq->s_used-1].i_except_handler_info; + if (instr->i_except != NULL) { + hi->h_offset = instr->i_except->b_offset; + hi->h_startdepth = instr->i_except->b_startdepth; + hi->h_preserve_lasti = instr->i_except->b_preserve_lasti; + } + else { + hi->h_offset = -1; + } } } return SUCCESS; diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 67cc5c5e88be10..6f83a910cab392 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -166,16 +166,10 @@ _PyBasicblock_InsertInstruction(basicblock *block, int pos, cfg_instr *instr) { return SUCCESS; } -int -_PyCfg_InstrSize(cfg_instr *instruction) +static int +instr_size(cfg_instr *instruction) { - int opcode = instruction->i_opcode; - assert(!IS_PSEUDO_OPCODE(opcode)); - int oparg = instruction->i_oparg; - assert(HAS_ARG(opcode) || oparg == 0); - int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg); - int caches = _PyOpcode_Caches[opcode]; - return extended_args + 1 + caches; + return _PyCompile_InstrSize(instruction->i_opcode, instruction->i_oparg); } static int @@ -183,7 +177,7 @@ blocksize(basicblock *b) { int size = 0; for (int i = 0; i < b->b_iused; i++) { - size += _PyCfg_InstrSize(&b->b_instr[i]); + size += instr_size(&b->b_instr[i]); } return size; } @@ -492,7 +486,7 @@ resolve_jump_offsets(basicblock *entryblock) bsize = b->b_offset; for (int i = 0; i < b->b_iused; i++) { cfg_instr *instr = &b->b_instr[i]; - int isize = _PyCfg_InstrSize(instr); + int isize = instr_size(instr); /* jump offsets are computed relative to * the instruction pointer after fetching * the jump instruction. @@ -508,7 +502,7 @@ resolve_jump_offsets(basicblock *entryblock) assert(!IS_BACKWARDS_JUMP_OPCODE(instr->i_opcode)); instr->i_oparg -= bsize; } - if (_PyCfg_InstrSize(instr) != isize) { + if (instr_size(instr) != isize) { extended_arg_recompile = 1; } } @@ -520,7 +514,7 @@ resolve_jump_offsets(basicblock *entryblock) with a better solution. The issue is that in the first loop blocksize() is called - which calls _PyCfg_InstrSize() which requires i_oparg be set + which calls instr_size() which requires i_oparg be set appropriately. There is a bootstrap problem because i_oparg is calculated in the second loop above. From 85c7bf5bcec07beea6064976e6199195cd34329d Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Sat, 29 Apr 2023 08:20:09 -0700 Subject: [PATCH 10/26] gh-103793: Defer formatting task name (#103767) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default task name is "Task-" (if no name is passed in during Task creation). This is initialized in `Task.__init__` (C impl) using string formatting, which can be quite slow. Actually using the task name in real world code is not very common, so this is wasted init. Let's defer this string formatting to the first time the name is read (in `get_name` impl), so we don't need to pay the string formatting cost if the task name is never read. We don't change the order in which tasks are assigned numbers (if they are) -- the number is set on task creation, as a PyLong instead of a formatted string. Co-authored-by: Łukasz Langa --- Doc/whatsnew/3.12.rst | 3 +++ Lib/test/test_asyncio/test_tasks.py | 12 ++++++++++++ .../2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst | 3 +++ Modules/_asynciomodule.c | 13 +++++++++++-- 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 908cf3bb365691..f4ee30b0d4d9eb 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -610,6 +610,9 @@ Optimizations replacement strings containing group references by 2--3 times. (Contributed by Serhiy Storchaka in :gh:`91524`.) +* Speed up :class:`asyncio.Task` creation by deferring expensive string formatting. + (Contributed by Itamar O in :gh:`103793`.) + CPython bytecode changes ======================== diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 31622c91470bcb..6e8a51ce2555d5 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -399,6 +399,18 @@ async def notmuch(): self.loop.run_until_complete(t1) self.loop.run_until_complete(t2) + def test_task_set_name_pylong(self): + # test that setting the task name to a PyLong explicitly doesn't + # incorrectly trigger the deferred name formatting logic + async def notmuch(): + return 123 + + t = self.new_task(self.loop, notmuch(), name=987654321) + self.assertEqual(t.get_name(), '987654321') + t.set_name(123456789) + self.assertEqual(t.get_name(), '123456789') + self.loop.run_until_complete(t) + def test_task_repr_name_not_str(self): async def notmuch(): return 123 diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst new file mode 100644 index 00000000000000..c48348798e7142 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-24-14-38-16.gh-issue-103793.kqoH6Q.rst @@ -0,0 +1,3 @@ +Optimized asyncio Task creation by deferring expensive string formatting +(task name generation) from Task creation to the first time ``get_name`` is +called. This makes asyncio benchmarks up to 5% faster. diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 2476dca6f58ebf..82dbc087322aa9 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2069,8 +2069,10 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, Py_XSETREF(self->task_coro, coro); if (name == Py_None) { - name = PyUnicode_FromFormat("Task-%" PRIu64, - ++state->task_name_counter); + // optimization: defer task name formatting + // store the task counter as PyLong in the name + // for deferred formatting in get_name + name = PyLong_FromUnsignedLongLong(++state->task_name_counter); } else if (!PyUnicode_CheckExact(name)) { name = PyObject_Str(name); } else { @@ -2449,6 +2451,13 @@ _asyncio_Task_get_name_impl(TaskObj *self) /*[clinic end generated code: output=0ecf1570c3b37a8f input=a4a6595d12f4f0f8]*/ { if (self->task_name) { + if (PyLong_CheckExact(self->task_name)) { + PyObject *name = PyUnicode_FromFormat("Task-%S", self->task_name); + if (name == NULL) { + return NULL; + } + Py_SETREF(self->task_name, name); + } return Py_NewRef(self->task_name); } From 00e2c5960ea785e4659187335c29ab19df9298a4 Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sat, 29 Apr 2023 23:11:15 +0400 Subject: [PATCH 11/26] Remove non-existing tools from Sundry skiplist (#103991) --- Lib/test/test_tools/test_sundry.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py index 6a3dc12781b2b6..3177fafb84a65b 100644 --- a/Lib/test/test_tools/test_sundry.py +++ b/Lib/test/test_tools/test_sundry.py @@ -1,4 +1,4 @@ -"""Tests for scripts in the Tools directory. +"""Tests for scripts in the Tools/scripts directory. This file contains extremely basic regression tests for the scripts found in the Tools directory of a Python checkout or tarball which don't have separate @@ -17,14 +17,7 @@ class TestSundryScripts(unittest.TestCase): # At least make sure the rest don't have syntax errors. When tests are # added for a script it should be added to the allowlist below. - # scripts that have independent tests. - allowlist = ['reindent'] - # scripts that can't be imported without running - denylist = ['make_ctype'] - # denylisted for other reasons - other = ['2to3'] - - skiplist = denylist + allowlist + other + skiplist = ['2to3'] # import logging registers "atfork" functions which keep indirectly the # logging module dictionary alive. Mock the function to be able to unload From 9e011e7c77dad7d0bbb944c44891531606caeb21 Mon Sep 17 00:00:00 2001 From: Joshua Herman Date: Sat, 29 Apr 2023 20:26:24 -0500 Subject: [PATCH 12/26] gh-82054: allow test runner to split test_asyncio to execute in parallel by sharding. (#103927) This runs test_asyncio sub-tests in parallel using sharding from Cinder. This suite is typically the longest-pole in runs because it is a test package with a lot of further sub-tests otherwise run serially. By breaking out the sub-tests as independent modules we can run a lot more in parallel. After porting we can see the direct impact on a multicore system. Without this change: Running make test is 5 min 26 seconds With this change: Running make test takes 3 min 39 seconds That'll vary based on system and parallelism. On a `-j 4` run similar to what CI and buildbot systems often do, it reduced the overall test suite completion latency by 10%. The drawbacks are that this implementation is hacky and due to the sorting of the tests it obscures when the asyncio tests occur and involves changing CPython test infrastructure but, the wall time saved it is worth it, especially in low-core count CI runs as it pulls a long tail. The win for productivity and reserved CI resource usage is significant. Future tests that deserve to be refactored into split up suites to benefit from are test_concurrent_futures and the way the _test_multiprocessing suite gets run for all start methods. As exposed by passing the -o flag to python -m test to get a list of the 10 longest running tests. --------- Co-authored-by: Carl Meyer Co-authored-by: Gregory P. Smith [Google, LLC] --- Lib/test/libregrtest/runtest.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index e9bb72a7d77ee1..61595277ed6d5a 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -143,6 +143,14 @@ def __str__(self) -> str: # set of tests that we don't want to be executed when using regrtest NOTTESTS = set() +#If these test directories are encountered recurse into them and treat each +# test_ .py or dir as a separate test module. This can increase parallelism. +# Beware this can't generally be done for any directory with sub-tests as the +# __init__.py may do things which alter what tests are to be run. + +SPLITTESTDIRS = { + "test_asyncio", +} # Storage of uncollectable objects FOUND_GARBAGE = [] @@ -158,7 +166,7 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_dirs=SPLITTESTDIRS, base_mod=""): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) names = os.listdir(testdir) @@ -166,8 +174,13 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): others = set(stdtests) | nottests for name in names: mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) + if mod[:5] == "test_" and mod not in others: + if mod in split_test_dirs: + subdir = os.path.join(testdir, mod) + mod = f"{base_mod or 'test'}.{mod}" + tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod)) + elif ext in (".py", ""): + tests.append(f"{base_mod}.{mod}" if base_mod else mod) return stdtests + sorted(tests) From 4b10ecc29f6ae69e599a5475a62d8e96a8711f90 Mon Sep 17 00:00:00 2001 From: Amethyst Reese Date: Sat, 29 Apr 2023 20:21:20 -0700 Subject: [PATCH 13/26] Update name in acknowledgements and add mailmap (#103696) I changed my name last year, and would like to update my name in the acknowledgements and git history accordingly. git-mailmap reference: https://git-scm.com/docs/gitmailmap Co-authored-by: Jelle Zijlstra --- .mailmap | 3 +++ Misc/ACKS | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000000000..013c839ed6b7a4 --- /dev/null +++ b/.mailmap @@ -0,0 +1,3 @@ +# This file sets the canonical name for contributors to the repository. +# Documentation: https://git-scm.com/docs/gitmailmap +Amethyst Reese diff --git a/Misc/ACKS b/Misc/ACKS index 65be5cfc3c7945..42ec059a7c4ec2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1488,7 +1488,7 @@ John Redford Kalyan Reddy Terry J. Reedy Gareth Rees -John Reese +Amethyst Reese Steve Reeves Lennart Regebro John Regehr From accb417c338630ac6e836a5c811a89d54a3cd1d3 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 30 Apr 2023 08:02:03 +0300 Subject: [PATCH 14/26] Replace Netlify with Read the Docs build previews (#103843) Co-authored-by: Oleg Iarygin Co-authored-by: C.A.M. Gerlach --- .github/workflows/documentation-links.yml | 27 +++++++++++++++++++++++ .readthedocs.yml | 18 +++++++++++++++ Doc/conf.py | 11 ++++----- Doc/tools/templates/layout.html | 5 ----- netlify.toml | 11 --------- 5 files changed, 51 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/documentation-links.yml create mode 100644 .readthedocs.yml delete mode 100644 netlify.toml diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml new file mode 100644 index 00000000000000..43a7afec73884e --- /dev/null +++ b/.github/workflows/documentation-links.yml @@ -0,0 +1,27 @@ +name: Read the Docs PR preview +# Automatically edits a pull request's descriptions with a link +# to the documentation's preview on Read the Docs. + +on: + pull_request_target: + types: + - opened + paths: + - 'Doc/**' + - '.github/workflows/doc.yml' + +permissions: + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/actions/preview@v1 + with: + project-slug: "cpython-previews" + single-version: "true" diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000000000..898a9ae89dbb92 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,18 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +# Project page: https://readthedocs.org/projects/cpython-previews/ + +version: 2 + +sphinx: + configuration: Doc/conf.py + +build: + os: ubuntu-22.04 + tools: + python: "3" + + commands: + - make -C Doc venv html + - mkdir _readthedocs + - mv Doc/build/html _readthedocs/html diff --git a/Doc/conf.py b/Doc/conf.py index 42c23bf77c7034..cef2a0e2837f6a 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -114,12 +114,13 @@ # Short title used e.g. for HTML tags. html_short_title = '%s Documentation' % release -# Deployment preview information, from Netlify -# (See netlify.toml and https://docs.netlify.com/configure-builds/environment-variables/#git-metadata) +# Deployment preview information +# (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) +repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL") html_context = { - "is_deployment_preview": os.getenv("IS_DEPLOYMENT_PREVIEW"), - "repository_url": os.getenv("REPOSITORY_URL"), - "pr_id": os.getenv("REVIEW_ID") + "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external", + "repository_url": repository_url.removesuffix(".git") if repository_url else None, + "pr_id": os.getenv("READTHEDOCS_VERSION") } # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 460161cd320223..b91f8138553e62 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -11,11 +11,6 @@ {%- if is_deployment_preview %} <div id="deployment-preview-warning" style="padding: .5em; text-align: center; background-color: #fff2ba; color: #6a580e;"> - <div style="float: right; margin-top: -10px; margin-left: 10px;"> - <a href="https://www.netlify.com"> - <img src="https://www.netlify.com/img/global/badges/netlify-color-accent.svg" alt="Deploys by Netlify" /> - </a> - </div> {% trans %}This is a deploy preview created from a <a href="{{ repository_url }}/pull/{{ pr_id }}">pull request</a>. For authoritative documentation, see the {% endtrans %} <a href="https://docs.python.org/3/{{ pagename }}{{ file_suffix }}">{% trans %} the current stable release{% endtrans %}</a>. diff --git a/netlify.toml b/netlify.toml deleted file mode 100644 index f5790fc5fec74f..00000000000000 --- a/netlify.toml +++ /dev/null @@ -1,11 +0,0 @@ -[build] - base = "Doc/" - command = "make html" - publish = "build/html" - # Do not trigger netlify builds if docs were not changed. - # Changed files should be in sync with `.github/workflows/doc.yml` - ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF . ../netlify.toml" - -[build.environment] - PYTHON_VERSION = "3.8" - IS_DEPLOYMENT_PREVIEW = "true" From ed95e8cbd4cbc813666c7ce7760257cc0f169d03 Mon Sep 17 00:00:00 2001 From: Ken Jin <kenjin@python.org> Date: Sun, 30 Apr 2023 21:08:26 +0800 Subject: [PATCH 15/26] gh-98003: Inline call frames for CALL_FUNCTION_EX (GH-98004) --- Include/internal/pycore_call.h | 10 ++ ...2-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst | 3 + Objects/call.c | 20 ++-- Python/bytecodes.c | 19 ++++ Python/ceval.c | 46 ++++++++ Python/generated_cases.c.h | 105 +++++++++++------- 6 files changed, 148 insertions(+), 55 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 55378e3dfebf24..5d9342b562b002 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -116,6 +116,16 @@ _PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); } +PyObject *const * +_PyStack_UnpackDict(PyThreadState *tstate, + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject **p_kwnames); + +void +_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, + PyObject *kwnames); + +void _PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames); #ifdef __cplusplus } diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst new file mode 100644 index 00000000000000..f9e71bc1344bb3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -0,0 +1,3 @@ +Complex function calls are now faster and consume no C stack +space. + diff --git a/Objects/call.c b/Objects/call.c index bd027e41f8a9a5..cf6e357a990441 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -8,16 +8,6 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() -static PyObject *const * -_PyStack_UnpackDict(PyThreadState *tstate, - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject **p_kwnames); - -static void -_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, - PyObject *kwnames); - - static PyObject * null_error(PyThreadState *tstate) { @@ -965,7 +955,7 @@ _PyStack_AsDict(PyObject *const *values, PyObject *kwnames) The newly allocated argument vector supports PY_VECTORCALL_ARGUMENTS_OFFSET. When done, you must call _PyStack_UnpackDict_Free(stack, nargs, kwnames) */ -static PyObject *const * +PyObject *const * _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames) @@ -1034,7 +1024,7 @@ _PyStack_UnpackDict(PyThreadState *tstate, return stack; } -static void +void _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames) { @@ -1042,6 +1032,12 @@ _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, for (Py_ssize_t i = 0; i < n; i++) { Py_DECREF(stack[i]); } + _PyStack_UnpackDict_FreeNoDecRef(stack, kwnames); +} + +void +_PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames) +{ PyMem_Free((PyObject **)stack - 1); Py_DECREF(kwnames); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9de0d92e382d3d..e83894e8902872 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3103,6 +3103,25 @@ dummy_func( } } else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); + } result = PyObject_Call(func, callargs, kwargs); } DECREF_INPUTS(); diff --git a/Python/ceval.c b/Python/ceval.c index 5d5221b2e40990..958689debc87f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -212,6 +212,9 @@ static _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, PyObject* const* args, size_t argcount, PyObject *kwnames); +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); static void _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); @@ -1501,6 +1504,49 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, return NULL; } +/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. + Steals references to func, callargs and kwargs. +*/ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) +{ + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs; + if (has_dict) { + newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); + if (newargs == NULL) { + Py_DECREF(func); + goto error; + } + } + else { + newargs = &PyTuple_GET_ITEM(callargs, 0); + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + /* No need to decref func here because the reference has been stolen by + _PyEvalFramePushAndInit. + */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return new_frame; +error: + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return NULL; +} + PyObject * _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 864a4f7bcaff0f..069a7ced0a4c25 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4296,16 +4296,35 @@ } } else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); + } result = PyObject_Call(func, callargs, kwargs); } - #line 4302 "Python/generated_cases.c.h" + #line 4321 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3109 "Python/bytecodes.c" + #line 3128 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4309 "Python/generated_cases.c.h" + #line 4328 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4320,7 +4339,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3119 "Python/bytecodes.c" + #line 3138 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4349,14 +4368,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4353 "Python/generated_cases.c.h" + #line 4372 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3150 "Python/bytecodes.c" + #line 3169 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4377,7 +4396,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4381 "Python/generated_cases.c.h" + #line 4400 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4385,15 +4404,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3173 "Python/bytecodes.c" + #line 3192 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4391 "Python/generated_cases.c.h" + #line 4410 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3175 "Python/bytecodes.c" + #line 3194 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4397 "Python/generated_cases.c.h" + #line 4416 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4404,7 +4423,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3179 "Python/bytecodes.c" + #line 3198 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4439,7 +4458,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4443 "Python/generated_cases.c.h" + #line 4462 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4448,10 +4467,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3216 "Python/bytecodes.c" + #line 3235 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4455 "Python/generated_cases.c.h" + #line 4474 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4463,7 +4482,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3221 "Python/bytecodes.c" + #line 3240 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4478,12 +4497,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4482 "Python/generated_cases.c.h" + #line 4501 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3236 "Python/bytecodes.c" + #line 3255 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4487 "Python/generated_cases.c.h" + #line 4506 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4493,16 +4512,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3241 "Python/bytecodes.c" + #line 3260 "Python/bytecodes.c" assert(oparg >= 2); - #line 4499 "Python/generated_cases.c.h" + #line 4518 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3245 "Python/bytecodes.c" + #line 3264 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4522,11 +4541,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4526 "Python/generated_cases.c.h" + #line 4545 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3267 "Python/bytecodes.c" + #line 3286 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4538,26 +4557,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4542 "Python/generated_cases.c.h" + #line 4561 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3281 "Python/bytecodes.c" + #line 3300 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4548 "Python/generated_cases.c.h" + #line 4567 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3285 "Python/bytecodes.c" + #line 3304 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4555 "Python/generated_cases.c.h" + #line 4574 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3290 "Python/bytecodes.c" + #line 3309 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4566,12 +4585,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4570 "Python/generated_cases.c.h" + #line 4589 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3301 "Python/bytecodes.c" + #line 3320 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4580,12 +4599,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4584 "Python/generated_cases.c.h" + #line 4603 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3312 "Python/bytecodes.c" + #line 3331 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4598,12 +4617,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4602 "Python/generated_cases.c.h" + #line 4621 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3327 "Python/bytecodes.c" + #line 3346 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4616,30 +4635,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4620 "Python/generated_cases.c.h" + #line 4639 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3342 "Python/bytecodes.c" + #line 3361 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4631 "Python/generated_cases.c.h" + #line 4650 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3350 "Python/bytecodes.c" + #line 3369 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4638 "Python/generated_cases.c.h" + #line 4657 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3355 "Python/bytecodes.c" + #line 3374 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4645 "Python/generated_cases.c.h" + #line 4664 "Python/generated_cases.c.h" } From f186557dc3e77495ba65b9b7e492ab6ddb7cecc3 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak <pieter.eendebak@gmail.com> Date: Sun, 30 Apr 2023 17:36:19 +0200 Subject: [PATCH 16/26] gh-103977: compile re expressions in platform.py only if required (#103981) Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com> --- Lib/platform.py | 75 ++++++++++--------- ...-04-28-19-08-50.gh-issue-103977.msF70A.rst | 1 + 2 files changed, 40 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst diff --git a/Lib/platform.py b/Lib/platform.py index 790ef860bf106e..7bb222088d5061 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -136,11 +136,11 @@ 'pl': 200, 'p': 200, } -_component_re = re.compile(r'([0-9]+|[._+-])') def _comparable_version(version): + component_re = re.compile(r'([0-9]+|[._+-])') result = [] - for v in _component_re.split(version): + for v in component_re.split(version): if v not in '._+-': try: v = int(v, 10) @@ -152,11 +152,6 @@ def _comparable_version(version): ### Platform specific APIs -_libc_search = re.compile(b'(__libc_init)' - b'|' - b'(GLIBC_([0-9.]+))' - b'|' - br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) def libc_ver(executable=None, lib='', version='', chunksize=16384): @@ -190,6 +185,12 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): # sys.executable is not set. return lib, version + libc_search = re.compile(b'(__libc_init)' + b'|' + b'(GLIBC_([0-9.]+))' + b'|' + br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) + V = _comparable_version # We use os.path.realpath() # here to work around problems with Cygwin not being @@ -200,7 +201,7 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): pos = 0 while pos < len(binary): if b'libc' in binary or b'GLIBC' in binary: - m = _libc_search.search(binary, pos) + m = libc_search.search(binary, pos) else: m = None if not m or m.end() == len(binary): @@ -247,9 +248,6 @@ def _norm_version(version, build=''): version = '.'.join(strings[:3]) return version -_ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' - r'.*' - r'\[.* ([\d.]+)\])') # Examples of VER command output: # @@ -295,9 +293,13 @@ def _syscmd_ver(system='', release='', version='', else: return system, release, version + ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' + r'.*' + r'\[.* ([\d.]+)\])') + # Parse the output info = info.strip() - m = _ver_output.match(info) + m = ver_output.match(info) if m is not None: system, release, version = m.groups() # Strip trailing dots from version and release @@ -1033,18 +1035,6 @@ def processor(): ### Various APIs for extracting information from sys.version -_sys_version_parser = re.compile( - r'([\w.+]+)\s*' # "version<space>" - r'\(#?([^,]+)' # "(#buildno" - r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)<space>" - r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" - -_pypy_sys_version_parser = re.compile( - r'([\w.+]+)\s*' - r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' - r'\[PyPy [^\]]+\]?') - _sys_version_cache = {} def _sys_version(sys_version=None): @@ -1076,10 +1066,17 @@ def _sys_version(sys_version=None): if result is not None: return result + sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version<space>" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)<space>" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" + if sys.platform.startswith('java'): # Jython name = 'Jython' - match = _sys_version_parser.match(sys_version) + match = sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % @@ -1091,8 +1088,13 @@ def _sys_version(sys_version=None): elif "PyPy" in sys_version: # PyPy + pypy_sys_version_parser = re.compile( + r'([\w.+]+)\s*' + r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' + r'\[PyPy [^\]]+\]?') + name = "PyPy" - match = _pypy_sys_version_parser.match(sys_version) + match = pypy_sys_version_parser.match(sys_version) if match is None: raise ValueError("failed to parse PyPy sys.version: %s" % repr(sys_version)) @@ -1101,7 +1103,7 @@ def _sys_version(sys_version=None): else: # CPython - match = _sys_version_parser.match(sys_version) + match = sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % @@ -1290,13 +1292,6 @@ def platform(aliased=False, terse=False): ### freedesktop.org os-release standard # https://www.freedesktop.org/software/systemd/man/os-release.html -# NAME=value with optional quotes (' or "). The regular expression is less -# strict than shell lexer, but that's ok. -_os_release_line = re.compile( - "^(?P<name>[a-zA-Z0-9_]+)=(?P<quote>[\"\']?)(?P<value>.*)(?P=quote)$" -) -# unescape five special characters mentioned in the standard -_os_release_unescape = re.compile(r"\\([\\\$\"\'`])") # /etc takes precedence over /usr/lib _os_release_candidates = ("/etc/os-release", "/usr/lib/os-release") _os_release_cache = None @@ -1311,10 +1306,18 @@ def _parse_os_release(lines): "PRETTY_NAME": "Linux", } + # NAME=value with optional quotes (' or "). The regular expression is less + # strict than shell lexer, but that's ok. + os_release_line = re.compile( + "^(?P<name>[a-zA-Z0-9_]+)=(?P<quote>[\"\']?)(?P<value>.*)(?P=quote)$" + ) + # unescape five special characters mentioned in the standard + os_release_unescape = re.compile(r"\\([\\\$\"\'`])") + for line in lines: - mo = _os_release_line.match(line) + mo = os_release_line.match(line) if mo is not None: - info[mo.group('name')] = _os_release_unescape.sub( + info[mo.group('name')] = os_release_unescape.sub( r"\1", mo.group('value') ) diff --git a/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst b/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst new file mode 100644 index 00000000000000..ff4005774a95d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst @@ -0,0 +1 @@ +Improve import time of :mod:`platform` module. From 7d3931e94a76491111a6e391e111cb066236cff4 Mon Sep 17 00:00:00 2001 From: Alex Waygood <Alex.Waygood@Gmail.com> Date: Sun, 30 Apr 2023 16:51:46 +0100 Subject: [PATCH 17/26] gh-104012: Ensure test_calendar.CalendarTestCase.test_deprecation_warning consistently passes (#104014) --- Lib/test/test_calendar.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 03388e8c55d5a8..24e472b5fee828 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -493,11 +493,11 @@ def test_format(self): class CalendarTestCase(unittest.TestCase): def test_deprecation_warning(self): - with warnings.catch_warnings(record=True) as w: + with self.assertWarnsRegex( + DeprecationWarning, + "The 'January' attribute is deprecated, use 'JANUARY' instead" + ): calendar.January - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, DeprecationWarning) - self.assertIn("The 'January' attribute is deprecated, use 'JANUARY' instead", str(w[0].message)) def test_isleap(self): # Make sure that the return is right for a few years, and From 654d44b3a4d3ee4d92b690668aa5189acf4f9d8f Mon Sep 17 00:00:00 2001 From: Nikita Sobolev <mail@sobolevn.me> Date: Sun, 30 Apr 2023 20:16:55 +0300 Subject: [PATCH 18/26] gh-104015: Fix direct invocation of `test_dataclasses` (#104017) Previously, `python -m test test_dataclasses` passed, but `./python.exe Lib/test/test_dataclasses.py` failed --- Lib/test/test_dataclasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 7dd81a8855f1be..7b48b26f9e7743 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -3671,7 +3671,7 @@ def test_text_annotations(self): ByMakeDataClass = make_dataclass('ByMakeDataClass', [('x', int)]) ManualModuleMakeDataClass = make_dataclass('ManualModuleMakeDataClass', [('x', int)], - module='test.test_dataclasses') + module=__name__) WrongNameMakeDataclass = make_dataclass('Wrong', [('x', int)]) WrongModuleMakeDataclass = make_dataclass('WrongModuleMakeDataclass', [('x', int)], From 74a2b79c6265c92ef381b5ff0dc63903bf0178ac Mon Sep 17 00:00:00 2001 From: Liam Gersten <gerstenliam@gmail.com> Date: Sun, 30 Apr 2023 16:17:36 -0400 Subject: [PATCH 19/26] gh-88773: Added teleport method to Turtle library (#103974) Add a `teleport` method to `turtle` module turtle instances that acts a lot like `goto`, _but_ ensures the pen is up while warping to the new position to and can control shape filling behavior as part of the jump. Based on an educator user feature request. --------- Co-authored-by: Terry Jan Reedy <tjreedy@udel.edu> Co-authored-by: Hugo van Kemenade <hugovk@users.noreply.github.com> Co-authored-by: Gregory P. Smith <greg@krypto.org> --- Doc/library/turtle.rst | 53 +++++++++++---- Lib/test/test_turtle.py | 20 ++++++ Lib/turtle.py | 66 ++++++++++++++++++- ...3-04-28-18-04-23.gh-issue-88773.xXCNJw.rst | 1 + 4 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 05392d04e52263..10138f4f406f85 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -107,6 +107,7 @@ Turtle motion | :func:`right` | :func:`rt` | :func:`left` | :func:`lt` | :func:`goto` | :func:`setpos` | :func:`setposition` + | :func:`teleport` | :func:`setx` | :func:`sety` | :func:`setheading` | :func:`seth` @@ -372,6 +373,44 @@ Turtle motion (0.00,0.00) +.. function:: teleport(x, y=None, *, fill_gap=False) + + :param x: a number or ``None`` + :param y: a number or ``None`` + :param fill_gap: a boolean + + Move turtle to an absolute position. Unlike goto(x, y), a line will not + be drawn. The turtle's orientation does not change. If currently + filling, the polygon(s) teleported from will be filled after leaving, + and filling will begin again after teleporting. This can be disabled + with fill_gap=True, which makes the imaginary line traveled during + teleporting act as a fill barrier like in goto(x, y). + + .. doctest:: + :skipif: _tkinter is None + :hide: + + >>> turtle.goto(0, 0) + + .. doctest:: + :skipif: _tkinter is None + + >>> tp = turtle.pos() + >>> tp + (0.00,0.00) + >>> turtle.teleport(60) + >>> turtle.pos() + (60.00,0.00) + >>> turtle.teleport(y=10) + >>> turtle.pos() + (60.00,10.00) + >>> turtle.teleport(20, 30) + >>> turtle.pos() + (20.00,30.00) + + .. versionadded: 3.12 + + .. function:: setx(x) :param x: a number (integer or float) @@ -537,8 +576,7 @@ Turtle motion :skipif: _tkinter is None >>> turtle.color("blue") - >>> turtle.stamp() - 11 + >>> stamp_id = turtle.stamp() >>> turtle.fd(50) @@ -575,15 +613,8 @@ Turtle motion .. doctest:: >>> for i in range(8): - ... turtle.stamp(); turtle.fd(30) - 13 - 14 - 15 - 16 - 17 - 18 - 19 - 20 + ... unused_stamp_id = turtle.stamp() + ... turtle.fd(30) >>> turtle.clearstamps(2) >>> turtle.clearstamps(-2) >>> turtle.clearstamps() diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 95af84e3779824..3f9f129a3dd200 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -267,6 +267,14 @@ def test_goto(self): self.assertAlmostEqual(self.nav.xcor(), 100) self.assertAlmostEqual(self.nav.ycor(), -100) + def test_teleport(self): + self.nav.teleport(20, -30, fill_gap=True) + self.assertAlmostEqual(self.nav.xcor(), 20) + self.assertAlmostEqual(self.nav.ycor(), -30) + self.nav.teleport(-20, 30, fill_gap=False) + self.assertAlmostEqual(self.nav.xcor(), -20) + self.assertAlmostEqual(self.nav.ycor(), 30) + def test_pos(self): self.assertEqual(self.nav.pos(), self.nav._position) self.nav.goto(100, -100) @@ -440,6 +448,18 @@ def test_showturtle_hideturtle_and_isvisible(self): tpen.showturtle() self.assertTrue(tpen.isvisible()) + def test_teleport(self): + + tpen = turtle.TPen() + + for fill_gap_value in [True, False]: + tpen.penup() + tpen.teleport(100, 100, fill_gap=fill_gap_value) + self.assertFalse(tpen.isdown()) + tpen.pendown() + tpen.teleport(-100, -100, fill_gap=fill_gap_value) + self.assertTrue(tpen.isdown()) + if __name__ == '__main__': unittest.main() diff --git a/Lib/turtle.py b/Lib/turtle.py index 1b369327bc8eff..2de406e0f517af 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -135,7 +135,7 @@ 'pu', 'radians', 'right', 'reset', 'resizemode', 'rt', 'seth', 'setheading', 'setpos', 'setposition', 'settiltangle', 'setundobuffer', 'setx', 'sety', 'shape', 'shapesize', 'shapetransform', 'shearfactor', 'showturtle', - 'speed', 'st', 'stamp', 'tilt', 'tiltangle', 'towards', + 'speed', 'st', 'stamp', 'teleport', 'tilt', 'tiltangle', 'towards', 'turtlesize', 'undo', 'undobufferentries', 'up', 'width', 'write', 'xcor', 'ycor'] _tg_utilities = ['write_docstringdict', 'done'] @@ -1614,6 +1614,13 @@ def _goto(self, end): """move turtle to position end.""" self._position = end + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: + """To be overwritten by child class RawTurtle. + Includes no TPen references.""" + new_x = x if x is not None else self._position[0] + new_y = y if y is not None else self._position[1] + self._position = Vec2D(new_x, new_y) + def forward(self, distance): """Move the turtle forward by the specified distance. @@ -2293,6 +2300,15 @@ def fillcolor(self, *args): else: return self._color(self._fillcolor) + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: + """To be overwritten by child class RawTurtle. + Includes no TNavigator references. + """ + pendown = self.isdown() + if pendown: + self.pen(pendown=False) + self.pen(pendown=pendown) + def showturtle(self): """Makes the turtle visible. @@ -2710,6 +2726,54 @@ def _cc(self, args): if not ((0 <= r <= 255) and (0 <= g <= 255) and (0 <= b <= 255)): raise TurtleGraphicsError("bad color sequence: %s" % str(args)) return "#%02x%02x%02x" % (r, g, b) + + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: + """Instantly move turtle to an absolute position. + + Arguments: + x -- a number or None + y -- a number None + fill_gap -- a boolean This argument must be specified by name. + + call: teleport(x, y) # two coordinates + --or: teleport(x) # teleport to x position, keeping y as is + --or: teleport(y=y) # teleport to y position, keeping x as is + --or: teleport(x, y, fill_gap=True) + # teleport but fill the gap in between + + Move turtle to an absolute position. Unlike goto(x, y), a line will not + be drawn. The turtle's orientation does not change. If currently + filling, the polygon(s) teleported from will be filled after leaving, + and filling will begin again after teleporting. This can be disabled + with fill_gap=True, which makes the imaginary line traveled during + teleporting act as a fill barrier like in goto(x, y). + + Example (for a Turtle instance named turtle): + >>> tp = turtle.pos() + >>> tp + (0.00,0.00) + >>> turtle.teleport(60) + >>> turtle.pos() + (60.00,0.00) + >>> turtle.teleport(y=10) + >>> turtle.pos() + (60.00,10.00) + >>> turtle.teleport(20, 30) + >>> turtle.pos() + (20.00,30.00) + """ + pendown = self.isdown() + was_filling = self.filling() + if pendown: + self.pen(pendown=False) + if was_filling and not fill_gap: + self.end_fill() + new_x = x if x is not None else self._position[0] + new_y = y if y is not None else self._position[1] + self._position = Vec2D(new_x, new_y) + self.pen(pendown=pendown) + if was_filling and not fill_gap: + self.begin_fill() def clone(self): """Create and return a clone of the turtle. diff --git a/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst b/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst new file mode 100644 index 00000000000000..f14c9533f3af87 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst @@ -0,0 +1 @@ +Added :func:`turtle.teleport` to the :mod:`turtle` module to move a turtle to a new point without tracing a line, visible or invisible. Patch by Liam Gersten. From 69bc86cb1aed49db27afc0095e0f4bcd8f1f3983 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 30 Apr 2023 17:16:38 -0700 Subject: [PATCH 20/26] Improve int test coverage (#104024) Following discussion in https://discuss.python.org/t/bug-in-int-42/26360/5 This tests some of the things documented in https://github.com/python/cpython/pull/100436 Co-authored-by: Gregory P. Smith <greg@krypto.org> --- Lib/test/test_int.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 334fea0774be51..5545ee39d8e942 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -155,6 +155,8 @@ def test_basic(self): self.assertEqual(int(' 0O123 ', 0), 83) self.assertEqual(int(' 0X123 ', 0), 291) self.assertEqual(int(' 0B100 ', 0), 4) + with self.assertRaises(ValueError): + int('010', 0) # without base still base 10 self.assertEqual(int('0123'), 123) @@ -221,6 +223,24 @@ def test_basic(self): self.assertEqual(int('2br45qc', 35), 4294967297) self.assertEqual(int('1z141z5', 36), 4294967297) + def test_invalid_signs(self): + with self.assertRaises(ValueError): + int('+') + with self.assertRaises(ValueError): + int('-') + with self.assertRaises(ValueError): + int('- 1') + with self.assertRaises(ValueError): + int('+ 1') + with self.assertRaises(ValueError): + int(' + 1 ') + + def test_unicode(self): + self.assertEqual(int("१२३४५६७८९०1234567890"), 12345678901234567890) + self.assertEqual(int('١٢٣٤٥٦٧٨٩٠'), 1234567890) + self.assertEqual(int("१२३४५६७८९०1234567890", 0), 12345678901234567890) + self.assertEqual(int('١٢٣٤٥٦٧٨٩٠', 0), 1234567890) + def test_underscores(self): for lit in VALID_UNDERSCORE_LITERALS: if any(ch in lit for ch in '.eEjJ'): From 4b27972f5fe816d3616f97f8643d8ad922473ab5 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy <tjreedy@udel.edu> Date: Sun, 30 Apr 2023 21:36:27 -0400 Subject: [PATCH 21/26] gh-88496: Fix IDLE test hang on macOS (#104025) Replace widget.update() with widget.update_idletasks in two places. --- Lib/idlelib/colorizer.py | 2 +- Lib/idlelib/outwin.py | 2 +- .../next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index e9f19c145c8673..b4df353012b788 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -310,7 +310,7 @@ def recolorize_main(self): # crumb telling the next invocation to resume here # in case update tells us to leave. self.tag_add("TODO", next) - self.update() + self.update_idletasks() if self.stop_colorizing: if DEBUG: print("colorizing stopped") return diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index ac67c904ab9797..610031e26f1dff 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -112,7 +112,7 @@ def write(self, s, tags=(), mark="insert"): assert isinstance(s, str) self.text.insert(mark, s, tags) self.text.see(mark) - self.text.update() + self.text.update_idletasks() return len(s) def writelines(self, lines): diff --git a/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst b/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst new file mode 100644 index 00000000000000..4f390d189d23b5 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst @@ -0,0 +1 @@ +Fix IDLE test hang on macOS. From 93107aa2a49a9354ffb10b3cd263dc3e99ebdeff Mon Sep 17 00:00:00 2001 From: Ben Faulhaber <111227622+faulhaberben@users.noreply.github.com> Date: Mon, 1 May 2023 07:47:34 +0200 Subject: [PATCH 22/26] Adjust expression from `==` to `!=` in alignment with the meaning of the paragraph. (GH-104021) --- Doc/library/venv.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 52bf99e5bb0f67..9e5672545dea35 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -55,7 +55,7 @@ point to the directories of the virtual environment, whereas :data:`sys.base_prefix` and :data:`sys.base_exec_prefix` point to those of the base Python used to create the environment. It is sufficient to check -``sys.prefix == sys.base_prefix`` to determine if the current interpreter is +``sys.prefix != sys.base_prefix`` to determine if the current interpreter is running from a virtual environment. A virtual environment may be "activated" using a script in its binary directory From 487f55d5801a9ae7d79d37e259e8c377c9acd39b Mon Sep 17 00:00:00 2001 From: Carey Metcalfe <carey@cmetcalfe.ca> Date: Mon, 1 May 2023 01:32:04 -0600 Subject: [PATCH 23/26] gh-103895: Improve how invalid `Exception.__notes__` are displayed (#103897) --- Lib/test/test_traceback.py | 12 ++++++++++-- Lib/traceback.py | 8 ++++++-- .../2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst | 3 +++ Python/pythonrun.c | 5 ++++- 4 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 5e2b353782994e..19a2be88d2c1bc 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1539,11 +1539,11 @@ def __repr__(self): e.__notes__ = BadThing() notes_repr = 'bad repr' - self.assertEqual(self.get_report(e), vanilla + notes_repr) + self.assertEqual(self.get_report(e), vanilla + notes_repr + '\n') e.__notes__ = Unprintable() err_msg = '<__notes__ repr() failed>' - self.assertEqual(self.get_report(e), vanilla + err_msg) + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') # non-string item in the __notes__ sequence e.__notes__ = [BadThing(), 'Final Note'] @@ -1555,6 +1555,14 @@ def __repr__(self): err_msg = '<note str() failed>' self.assertEqual(self.get_report(e), vanilla + err_msg + '\nFinal Note\n') + e.__notes__ = "please do not explode me" + err_msg = "'please do not explode me'" + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + + e.__notes__ = b"please do not show me as numbers" + err_msg = "b'please do not show me as numbers'" + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + def test_exception_with_note_with_multiple_notes(self): e = ValueError(42) vanilla = self.get_report(e) diff --git a/Lib/traceback.py b/Lib/traceback.py index 9e720ac9948fce..ba4a9ffd001b53 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -852,12 +852,16 @@ def format_exception_only(self): yield _format_final_exc_line(stype, self._str) else: yield from self._format_syntax_error(stype) - if isinstance(self.__notes__, collections.abc.Sequence): + + if ( + isinstance(self.__notes__, collections.abc.Sequence) + and not isinstance(self.__notes__, (str, bytes)) + ): for note in self.__notes__: note = _safe_string(note, 'note') yield from [l + '\n' for l in note.split('\n')] elif self.__notes__ is not None: - yield _safe_string(self.__notes__, '__notes__', func=repr) + yield "{}\n".format(_safe_string(self.__notes__, '__notes__', func=repr)) def _format_syntax_error(self, stype): """Format SyntaxError exceptions (internal helper).""" diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst new file mode 100644 index 00000000000000..6fed304c9132b3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst @@ -0,0 +1,3 @@ +Improve handling of edge cases in showing ``Exception.__notes__``. Ensures +that the messages always end with a newline and that string/bytes are not +exploded over multiple lines. Patch by Carey Metcalfe. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b16d3f53f89fb9..05e7b4370869af 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1107,7 +1107,7 @@ print_exception_notes(struct exception_print_context *ctx, PyObject *value) if (notes == NULL) { return -1; } - if (!PySequence_Check(notes)) { + if (!PySequence_Check(notes) || PyUnicode_Check(notes) || PyBytes_Check(notes)) { int res = 0; if (write_indented_margin(ctx, f) < 0) { res = -1; @@ -1122,6 +1122,9 @@ print_exception_notes(struct exception_print_context *ctx, PyObject *value) Py_DECREF(s); } Py_DECREF(notes); + if (PyFile_WriteString("\n", f) < 0) { + res = -1; + } return res; } Py_ssize_t num_notes = PySequence_Length(notes); From 59c27fa5cb95e2d608747a50fc675bbe2fc96beb Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Mon, 1 May 2023 18:10:35 +0800 Subject: [PATCH 24/26] gh-102213: Optimize the performance of `__getattr__` (GH-103761) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kirill <80244920+Eclips4@users.noreply.github.com> Co-authored-by: Łukasz Langa <lukasz@langa.pl> Co-authored-by: Xiang Wang <34048878+wangxiang-hz@users.noreply.github.com> --- Lib/test/test_descr.py | 15 +++++++++++++- ...-05-01-08-08-05.gh-issue-102213.nfH-4C.rst | 1 + Objects/typeobject.c | 20 ++++++++++++------- 3 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index f17bb1813b9d87..ad3eefba365856 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5004,7 +5004,7 @@ class Child(Parent): self.assertEqual(Parent.__subclasses__(), []) def test_attr_raise_through_property(self): - # add test case for gh-103272 + # test case for gh-103272 class A: def __getattr__(self, name): raise ValueError("FOO") @@ -5016,6 +5016,19 @@ def foo(self): with self.assertRaisesRegex(ValueError, "FOO"): A().foo + # test case for gh-103551 + class B: + @property + def __getattr__(self, name): + raise ValueError("FOO") + + @property + def foo(self): + raise NotImplementedError("BAR") + + with self.assertRaisesRegex(NotImplementedError, "BAR"): + B().foo + class DictProxyTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst new file mode 100644 index 00000000000000..997bef226e713f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst @@ -0,0 +1 @@ +Fix performance loss when accessing an object's attributes with ``__getattr__`` defined. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 38b99315457a58..e807cc90faa16a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8306,17 +8306,23 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) if (getattribute == NULL || (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == - (void *)PyObject_GenericGetAttr)) - res = PyObject_GenericGetAttr(self, name); - else { + (void *)PyObject_GenericGetAttr)) { + res = _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); + /* if res == NULL with no exception set, then it must be an + AttributeError suppressed by us. */ + if (res == NULL && !PyErr_Occurred()) { + res = call_attribute(self, getattr, name); + } + } else { Py_INCREF(getattribute); res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); + if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + res = call_attribute(self, getattr, name); + } } - if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - res = call_attribute(self, getattr, name); - } + Py_DECREF(getattr); return res; } From 4181d078fc945313568eb39965cb9190881606b5 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora <80244920+Eclips4@users.noreply.github.com> Date: Mon, 1 May 2023 16:42:59 +0300 Subject: [PATCH 25/26] gh-104036: Fix direct invocation of test_typing (#104037) Previously, `python -m test test_typing` worked, but `python Lib/test/test_typing.py` did not. --- Lib/test/test_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f36bb958c88ef9..7c6a521c3c48f8 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -117,7 +117,7 @@ def test_repr(self): class Sub(Any): pass self.assertEqual( repr(Sub), - "<class 'test.test_typing.AnyTests.test_repr.<locals>.Sub'>", + f"<class '{__name__}.AnyTests.test_repr.<locals>.Sub'>", ) def test_errors(self): From e1476942525ae847875dab55541bef4a8a99dd3d Mon Sep 17 00:00:00 2001 From: Dong-hee Na <donghee.na@python.org> Date: Mon, 1 May 2023 23:03:24 +0900 Subject: [PATCH 26/26] gh-104028: Reduce object creation while calling callback function from gc (gh-104030) --- .../2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst | 2 ++ Modules/gcmodule.c | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst new file mode 100644 index 00000000000000..9c35ea88499dce --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst @@ -0,0 +1,2 @@ +Reduce object creation while calling callback function from gc. +Patch by Dong-hee Na. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 966c1e615502ef..3fd5f4cd70e832 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1388,10 +1388,19 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase, return; } } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_WriteUnraisable(NULL); + return; + } + + PyObject *stack[] = {phase_obj, info}; for (Py_ssize_t i=0; i<PyList_GET_SIZE(gcstate->callbacks); i++) { PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_CallFunction(cb, "sO", phase, info); + r = PyObject_Vectorcall(cb, stack, 2, NULL); if (r == NULL) { PyErr_WriteUnraisable(cb); } @@ -1400,6 +1409,7 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase, } Py_DECREF(cb); } + Py_DECREF(phase_obj); Py_XDECREF(info); assert(!_PyErr_Occurred(tstate)); }