diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml new file mode 100644 index 00000000000000..43a7afec73884e --- /dev/null +++ b/.github/workflows/documentation-links.yml @@ -0,0 +1,27 @@ +name: Read the Docs PR preview +# Automatically edits a pull request's descriptions with a link +# to the documentation's preview on Read the Docs. + +on: + pull_request_target: + types: + - opened + paths: + - 'Doc/**' + - '.github/workflows/doc.yml' + +permissions: + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/actions/preview@v1 + with: + project-slug: "cpython-previews" + single-version: "true" diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000000000..013c839ed6b7a4 --- /dev/null +++ b/.mailmap @@ -0,0 +1,3 @@ +# This file sets the canonical name for contributors to the repository. +# Documentation: https://git-scm.com/docs/gitmailmap +Amethyst Reese diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000000000..898a9ae89dbb92 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,18 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +# Project page: https://readthedocs.org/projects/cpython-previews/ + +version: 2 + +sphinx: + configuration: Doc/conf.py + +build: + os: ubuntu-22.04 + tools: + python: "3" + + commands: + - make -C Doc venv html + - mkdir _readthedocs + - mv Doc/build/html _readthedocs/html diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index cb5d64a50487fe..c3260a21bc7f8b 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -59,12 +59,31 @@ rules: Analogous to :c:func:`PyObject_New` but for container objects with the :const:`Py_TPFLAGS_HAVE_GC` flag set. - .. c:function:: TYPE* PyObject_GC_NewVar(TYPE, PyTypeObject *type, Py_ssize_t size) Analogous to :c:func:`PyObject_NewVar` but for container objects with the :const:`Py_TPFLAGS_HAVE_GC` flag set. +.. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size) + + Analogous to :c:func:`PyObject_GC_New` but allocates *extra_size* + bytes at the end of the object (at offset + :c:member:`~PyTypeObject.tp_basicsize`). + The allocated memory is initialized to zeros, + except for the :c:type:`Python object header `. + + The extra data will be deallocated with the object, but otherwise it is + not managed by Python. + + .. warning:: + The function is marked as unstable because the final mechanism + for reserving extra data after an instance is not yet decided. + For allocating a variable number of fields, prefer using + :c:type:`PyVarObject` and :c:member:`~PyTypeObject.tp_itemsize` + instead. + + .. versionadded:: 3.12 + .. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) diff --git a/Doc/conf.py b/Doc/conf.py index 42c23bf77c7034..6a3c01cd91a77b 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -114,12 +114,13 @@ # Short title used e.g. for HTML tags. html_short_title = '%s Documentation' % release -# Deployment preview information, from Netlify -# (See netlify.toml and https://docs.netlify.com/configure-builds/environment-variables/#git-metadata) +# Deployment preview information +# (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) +repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL") html_context = { - "is_deployment_preview": os.getenv("IS_DEPLOYMENT_PREVIEW"), - "repository_url": os.getenv("REPOSITORY_URL"), - "pr_id": os.getenv("REVIEW_ID") + "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external", + "repository_url": repository_url.removesuffix(".git") if repository_url else None, + "pr_id": os.getenv("READTHEDOCS_VERSION") } # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, @@ -263,11 +264,29 @@ linkcheck_allowed_redirects = { # bpo-NNNN -> BPO -> GH Issues - r'https://bugs.python.org/issue\?@action=redirect&bpo=\d+': 'https://github.com/python/cpython/issues/\d+', + r'https://bugs.python.org/issue\?@action=redirect&bpo=\d+': r'https://github.com/python/cpython/issues/\d+', # GH-NNNN used to refer to pull requests - r'https://github.com/python/cpython/issues/\d+': 'https://github.com/python/cpython/pull/\d+', + r'https://github.com/python/cpython/issues/\d+': r'https://github.com/python/cpython/pull/\d+', # :source:`something` linking files in the repository - r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*' + r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*', + # Intentional HTTP use at Misc/NEWS.d/3.5.0a1.rst + r'http://www.python.org/$': 'https://www.python.org/$', + # Used in license page, keep as is + r'https://www.zope.org/': r'https://www.zope.dev/', + # Microsoft's redirects to learn.microsoft.com + r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*', + r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*', + r'https://go.microsoft.com/fwlink/\?LinkID=\d+': 'https://learn.microsoft.com/.*', + # Language redirects + r'https://toml.io': 'https://toml.io/en/', + r'https://www.redhat.com': 'https://www.redhat.com/en', + # Other redirects + r'https://www.boost.org/libs/.+': r'https://www.boost.org/doc/libs/\d_\d+_\d/.+', + r'https://support.microsoft.com/en-us/help/\d+': 'https://support.microsoft.com/en-us/topic/.+', + r'https://perf.wiki.kernel.org$': 'https://perf.wiki.kernel.org/index.php/Main_Page', + r'https://www.sqlite.org': 'https://www.sqlite.org/index.html', + r'https://mitpress.mit.edu/sicp$': 'https://mitpress.mit.edu/9780262510875/structure-and-interpretation-of-computer-programs/', + r'https://www.python.org/psf/': 'https://www.python.org/psf-landing/', } linkcheck_anchors_ignore = [ diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 07282639e4f9b4..bc3080f60ee237 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -42,7 +42,7 @@ on what you're trying to do. .. XXX make sure these all work `Cython <https://cython.org>`_ and its relative `Pyrex -<https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/>`_ are compilers +<https://www.csse.canterbury.ac.nz/greg.ewing/python/Pyrex/>`_ are compilers that accept a slightly modified form of Python and generate the corresponding C code. Cython and Pyrex make it possible to write an extension without having to learn Python's C API. diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 6256deb5797c89..a9b2622e02ef3b 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -54,8 +54,8 @@ commercial use, to sell copies of Python in source or binary form (modified or unmodified), or to sell products that incorporate Python in some form. We would still like to know about all commercial use of Python, of course. -See `the PSF license page <https://www.python.org/psf/license/>`_ to find further -explanations and a link to the full text of the license. +See `the license page <https://docs.python.org/3/license.html>`_ to find further +explanations and the full text of the PSF License. The Python logo is trademarked, and in certain cases permission is required to use it. Consult `the Trademark Usage Policy @@ -215,7 +215,7 @@ every day, and Usenet readers are often more able to cope with this volume. Announcements of new software releases and events can be found in comp.lang.python.announce, a low-traffic moderated list that receives about five postings per day. It's available as `the python-announce mailing list -<https://mail.python.org/mailman/listinfo/python-announce-list>`_. +<https://mail.python.org/mailman3/lists/python-announce-list.python.org/>`_. More info about other mailing lists and newsgroups can be found at https://www.python.org/community/lists/. @@ -352,7 +352,7 @@ titled "Python X.Y Release Schedule", where X.Y is a version that hasn't been publicly released yet. New development is discussed on `the python-dev mailing list -<https://mail.python.org/mailman/listinfo/python-dev/>`_. +<https://mail.python.org/mailman3/lists/python-dev.python.org/>`_. Is it reasonable to propose incompatible changes to Python? diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 38f9b171618b26..ab5618db84f77e 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -61,7 +61,7 @@ Yes. `Pyflakes <https://github.com/PyCQA/pyflakes>`_ do basic checking that will help you catch bugs sooner. -Static type checkers such as `Mypy <http://mypy-lang.org/>`_, +Static type checkers such as `Mypy <https://mypy-lang.org/>`_, `Pyre <https://pyre-check.org/>`_, and `Pytype <https://github.com/google/pytype>`_ can check type hints in Python source code. diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index add1c11be534e3..baea3e85c3b84b 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -438,7 +438,7 @@ to make sure everything functions as expected in both versions of Python. .. _Futurize: https://python-future.org/automatic_conversion.html .. _importlib2: https://pypi.org/project/importlib2 .. _Modernize: https://python-modernize.readthedocs.io/ -.. _mypy: http://mypy-lang.org/ +.. _mypy: https://mypy-lang.org/ .. _Porting to Python 3: http://python3porting.com/ .. _Pylint: https://pypi.org/project/pylint diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index e982cc166a3f2d..8d0022cc66daac 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -529,7 +529,7 @@ Opening network connections specifies requirements for algorithms that reduce this user-visible delay and provides an algorithm. - For more information: https://tools.ietf.org/html/rfc6555 + For more information: https://datatracker.ietf.org/doc/html/rfc6555 .. versionchanged:: 3.11 diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index ba0f909c405a34..f8727b98066990 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -527,6 +527,42 @@ Running Tasks Concurrently and there is no running event loop. +Eager Task Factory +================== + +.. function:: eager_task_factory(loop, coro, *, name=None, context=None) + + A task factory for eager task execution. + + When using this factory (via :meth:`loop.set_task_factory(asyncio.eager_task_factory) <loop.set_task_factory>`), + coroutines begin execution synchronously during :class:`Task` construction. + Tasks are only scheduled on the event loop if they block. + This can be a performance improvement as the overhead of loop scheduling + is avoided for coroutines that complete synchronously. + + A common example where this is beneficial is coroutines which employ + caching or memoization to avoid actual I/O when possible. + + .. note:: + + Immediate execution of the coroutine is a semantic change. + If the coroutine returns or raises, the task is never scheduled + to the event loop. If the coroutine execution blocks, the task is + scheduled to the event loop. This change may introduce behavior + changes to existing applications. For example, + the application's task execution order is likely to change. + + .. versionadded:: 3.12 + +.. function:: create_eager_task_factory(custom_task_constructor) + + Create an eager task factory, similar to :func:`eager_task_factory`, + using the provided *custom_task_constructor* when creating a new task instead + of the default :class:`Task`. + + .. versionadded:: 3.12 + + Shielding From Cancellation =========================== diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 2107215c0c1967..afc3e66f0bf7ac 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -29,7 +29,7 @@ Such constructors may be factory functions or class instances. Declares that *function* should be used as a "reduction" function for objects of type *type*. *function* must return either a string or a tuple - containing two or five elements. See the :attr:`~pickle.Pickler.dispatch_table` + containing between two and six elements. See the :attr:`~pickle.Pickler.dispatch_table` for more details on the interface of *function*. The *constructor_ob* parameter is a legacy feature and is now ignored, but if diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index f8d10c0c295c7a..6275f96f7d4d19 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -432,7 +432,7 @@ Constructor functions also accept the following tree hashing parameters: :alt: Explanation of tree mode parameters. See section 2.10 in `BLAKE2 specification -<https://blake2.net/blake2_20130129.pdf>`_ for comprehensive review of tree +<https://www.blake2.net/blake2_20130129.pdf>`_ for comprehensive review of tree hashing. @@ -619,7 +619,7 @@ on the hash function used in digital signatures. by the signer. (`NIST SP-800-106 "Randomized Hashing for Digital Signatures" - <https://csrc.nist.gov/publications/detail/sp/800-106/final>`_) + <https://csrc.nist.gov/publications/detail/sp/800-106/archive/2009-02-25>`_) In BLAKE2 the salt is processed as a one-time input to the hash function during initialization, rather than as an input to each compression function. @@ -628,7 +628,7 @@ initialization, rather than as an input to each compression function. *Salted hashing* (or just hashing) with BLAKE2 or any other general-purpose cryptographic hash function, such as SHA-256, is not suitable for hashing - passwords. See `BLAKE2 FAQ <https://blake2.net/#qa>`_ for more + passwords. See `BLAKE2 FAQ <https://www.blake2.net/#qa>`_ for more information. .. @@ -764,9 +764,9 @@ Domain Dedication 1.0 Universal: * *Alexandr Sokolovskiy* -.. _BLAKE2: https://blake2.net +.. _BLAKE2: https://www.blake2.net .. _HMAC: https://en.wikipedia.org/wiki/Hash-based_message_authentication_code -.. _BLAKE: https://131002.net/blake/ +.. _BLAKE: https://web.archive.org/web/20200918190133/https://131002.net/blake/ .. _SHA-3: https://en.wikipedia.org/wiki/NIST_hash_function_competition .. _ChaCha: https://cr.yp.to/chacha.html .. _pyblake2: https://pythonhosted.org/pyblake2/ @@ -782,7 +782,7 @@ Domain Dedication 1.0 Universal: Module :mod:`base64` Another way to encode binary hashes for non-binary environments. - https://blake2.net + https://www.blake2.net Official BLAKE2 website. https://csrc.nist.gov/csrc/media/publications/fips/180/2/archive/2002-08-01/documents/fips180-2.pdf diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 38821b32c91cf1..abdc6b447a8b61 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -354,7 +354,7 @@ HTTPConnection Objects the CONNECT request. As HTTP/1.1 is used for HTTP CONNECT tunnelling request, `as per the RFC - <https://tools.ietf.org/html/rfc7231#section-4.3.6>`_, a HTTP ``Host:`` + <https://datatracker.ietf.org/doc/html/rfc7231#section-4.3.6>`_, a HTTP ``Host:`` header must be provided, matching the authority-form of the request target provided as the destination for the CONNECT request. If a HTTP ``Host:`` header is not provided via the headers argument, one is generated and diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index b306d5f55a714f..3097bcf47b627f 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -178,7 +178,7 @@ The "selectable" entry points were introduced in ``importlib_metadata`` no parameters and always returned a dictionary of entry points, keyed by group. With ``importlib_metadata`` 5.0 and Python 3.12, ``entry_points`` always returns an ``EntryPoints`` object. See -`backports.entry_points_selectable <https://pypi.org/project/backports.entry_points_selectable>`_ +`backports.entry_points_selectable <https://pypi.org/project/backports.entry-points-selectable>`_ for compatibility options. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 50e951c631fa88..76623c6305432a 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4593,7 +4593,7 @@ written in Python, such as a mail server's external command delivery program. :attr:`!children_system`, and :attr:`!elapsed` in that order. See the Unix manual page - :manpage:`times(2)` and `times(3) <https://www.freebsd.org/cgi/man.cgi?time(3)>`_ manual page on Unix or `the GetProcessTimes MSDN + :manpage:`times(2)` and `times(3) <https://man.freebsd.org/cgi/man.cgi?time(3)>`_ manual page on Unix or `the GetProcessTimes MSDN <https://docs.microsoft.com/windows/win32/api/processthreadsapi/nf-processthreadsapi-getprocesstimes>`_ on Windows. On Windows, only :attr:`!user` and :attr:`!system` are known; the other attributes are zero. diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 7aad15ec91a0ac..732ef3536863cc 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -46,7 +46,7 @@ or :class:`datetime.datetime` objects. .. seealso:: - `PList manual page <https://developer.apple.com/library/content/documentation/Cocoa/Conceptual/PropertyLists/>`_ + `PList manual page <https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/PropertyLists/>`_ Apple's documentation of the file format. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index e7bf45d7d569fa..a5324c82c63484 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -244,7 +244,7 @@ platform. used by all of this user id's processes. This limit is enforced only if bit 1 of the vm.overcommit sysctl is set. Please see - `tuning(7) <https://www.freebsd.org/cgi/man.cgi?query=tuning&sektion=7>`__ + `tuning(7) <https://man.freebsd.org/cgi/man.cgi?query=tuning&sektion=7>`__ for a complete description of this sysctl. .. availability:: FreeBSD. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 2890706bab729c..b0891b0c8f584a 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -505,7 +505,7 @@ Kqueue Objects Kevent Objects -------------- -https://www.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 .. attribute:: kevent.ident diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 4b60b7c643b62c..18a6c5ab4858a4 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1719,7 +1719,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.3 .. seealso:: - `SSL/TLS & Perfect Forward Secrecy <https://vincent.bernat.im/en/blog/2011-ssl-perfect-forward-secrecy>`_ + `SSL/TLS & Perfect Forward Secrecy <https://vincent.bernat.ch/en/blog/2011-ssl-perfect-forward-secrecy>`_ Vincent Bernat. .. method:: SSLContext.wrap_socket(sock, server_side=False, \ diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index f934b0e0319dca..395b324c860389 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -22,7 +22,7 @@ This module provides functions for calculating mathematical statistics of numeric (:class:`~numbers.Real`-valued) data. The module is not intended to be a competitor to third-party libraries such -as `NumPy <https://numpy.org>`_, `SciPy <https://www.scipy.org/>`_, or +as `NumPy <https://numpy.org>`_, `SciPy <https://scipy.org/>`_, or proprietary full-featured statistics packages aimed at professional statisticians such as Minitab, SAS and Matlab. It is aimed at the level of graphing and scientific calculators. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 9c0e32ba16bf68..78fd6e397ae635 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -602,4 +602,4 @@ The :mod:`struct` module also defines the following type: .. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_754-2008_revision -.. _IETF RFC 1700: https://tools.ietf.org/html/rfc1700 +.. _IETF RFC 1700: https://datatracker.ietf.org/doc/html/rfc1700 diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 7c0e85142e7716..95ad243bdde398 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -792,7 +792,7 @@ always available. additional garbage collector overhead if the object is managed by the garbage collector. - See `recursive sizeof recipe <https://code.activestate.com/recipes/577504>`_ + See `recursive sizeof recipe <https://code.activestate.com/recipes/577504/>`_ for an example of using :func:`getsizeof` recursively to find the size of containers and all their contents. diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 05392d04e52263..10138f4f406f85 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -107,6 +107,7 @@ Turtle motion | :func:`right` | :func:`rt` | :func:`left` | :func:`lt` | :func:`goto` | :func:`setpos` | :func:`setposition` + | :func:`teleport` | :func:`setx` | :func:`sety` | :func:`setheading` | :func:`seth` @@ -372,6 +373,44 @@ Turtle motion (0.00,0.00) +.. function:: teleport(x, y=None, *, fill_gap=False) + + :param x: a number or ``None`` + :param y: a number or ``None`` + :param fill_gap: a boolean + + Move turtle to an absolute position. Unlike goto(x, y), a line will not + be drawn. The turtle's orientation does not change. If currently + filling, the polygon(s) teleported from will be filled after leaving, + and filling will begin again after teleporting. This can be disabled + with fill_gap=True, which makes the imaginary line traveled during + teleporting act as a fill barrier like in goto(x, y). + + .. doctest:: + :skipif: _tkinter is None + :hide: + + >>> turtle.goto(0, 0) + + .. doctest:: + :skipif: _tkinter is None + + >>> tp = turtle.pos() + >>> tp + (0.00,0.00) + >>> turtle.teleport(60) + >>> turtle.pos() + (60.00,0.00) + >>> turtle.teleport(y=10) + >>> turtle.pos() + (60.00,10.00) + >>> turtle.teleport(20, 30) + >>> turtle.pos() + (20.00,30.00) + + .. versionadded: 3.12 + + .. function:: setx(x) :param x: a number (integer or float) @@ -537,8 +576,7 @@ Turtle motion :skipif: _tkinter is None >>> turtle.color("blue") - >>> turtle.stamp() - 11 + >>> stamp_id = turtle.stamp() >>> turtle.fd(50) @@ -575,15 +613,8 @@ Turtle motion .. doctest:: >>> for i in range(8): - ... turtle.stamp(); turtle.fd(30) - 13 - 14 - 15 - 16 - 17 - 18 - 19 - 20 + ... unused_stamp_id = turtle.stamp() + ... turtle.fd(30) >>> turtle.clearstamps(2) >>> turtle.clearstamps(-2) >>> turtle.clearstamps() diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 409a95d528b5d3..c22fc0b28a50d0 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2484,15 +2484,16 @@ Functions and decorators Ask a static type checker to confirm that *val* has an inferred type of *typ*. - When the type checker encounters a call to ``assert_type()``, it + At runtime this does nothing: it returns the first argument unchanged with no + checks or side effects, no matter the actual type of the argument. + + When a static type checker encounters a call to ``assert_type()``, it emits an error if the value is not of the specified type:: def greet(name: str) -> None: assert_type(name, str) # OK, inferred type of `name` is `str` assert_type(name, int) # type checker error - At runtime this returns the first argument unchanged with no side effects. - This function is useful for ensuring the type checker's understanding of a script is in line with the developer's intentions:: diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index f9a207bad6903f..895b9f9f07671b 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -1074,7 +1074,7 @@ subclass. Sometimes this is inconvenient. For example, `one user <https://code.google.com/archive/p/mock/issues/105>`_ is subclassing mock to created a `Twisted adaptor -<https://twistedmatrix.com/documents/11.0.0/api/twisted.python.components.html>`_. +<https://twisted.org/documents/11.0.0/api/twisted.python.components.html>`_. Having this applied to attributes too actually causes errors. ``Mock`` (in all its flavours) uses a method called ``_get_child_mock`` to create diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index c70153dfcd69e1..b26e6c0e6bc024 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -72,7 +72,7 @@ test runner a GUI tool for test discovery and execution. This is intended largely for ease of use for those new to unit testing. For production environments it is recommended that tests be driven by a continuous integration system such as - `Buildbot <https://buildbot.net/>`_, `Jenkins <https://jenkins.io/>`_, + `Buildbot <https://buildbot.net/>`_, `Jenkins <https://www.jenkins.io/>`_, `GitHub Actions <https://github.com/features/actions>`_, or `AppVeyor <https://www.appveyor.com/>`_. diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 52bf99e5bb0f67..9e5672545dea35 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -55,7 +55,7 @@ point to the directories of the virtual environment, whereas :data:`sys.base_prefix` and :data:`sys.base_exec_prefix` point to those of the base Python used to create the environment. It is sufficient to check -``sys.prefix == sys.base_prefix`` to determine if the current interpreter is +``sys.prefix != sys.base_prefix`` to determine if the current interpreter is running from a virtual environment. A virtual environment may be "activated" using a script in its binary directory diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index bd2c49a6edab7f..146c4fd768233b 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -161,7 +161,7 @@ between conformable Python objects and XML on the wire. .. seealso:: - `XML-RPC HOWTO <https://www.tldp.org/HOWTO/XML-RPC-HOWTO/index.html>`_ + `XML-RPC HOWTO <https://tldp.org/HOWTO/XML-RPC-HOWTO/index.html>`_ A good description of XML-RPC operation and client software in several languages. Contains pretty much everything an XML-RPC client developer needs to know. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 6f4826cb065c64..45f3d340bd82d3 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -128,7 +128,7 @@ The module defines the following items: Documentation on the ZIP file format by Phil Katz, the creator of the format and algorithms used. - `Info-ZIP Home Page <http://www.info-zip.org/>`_ + `Info-ZIP Home Page <https://infozip.sourceforge.net/>`_ Information about the Info-ZIP project's ZIP archive programs and development libraries. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 55431f1951e50d..c35bf4016a28d2 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1564,7 +1564,7 @@ Basic customization This is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict insertion, O(n\ :sup:`2`) complexity. See - http://www.ocert.org/advisories/ocert-2011-003.html for details. + http://ocert.org/advisories/ocert-2011-003.html for details. Changing hash values affects the iteration order of sets. Python has never made guarantees about this ordering diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index 914a11556c94e6..81f0a5c5d43883 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -74,7 +74,7 @@ PyPy and a Just in Time compiler. One of the goals of the project is to encourage experimentation with the language itself by making it easier to modify the interpreter (since it is written in Python). Additional information is - available on `the PyPy project's home page <https://pypy.org/>`_. + available on `the PyPy project's home page <https://www.pypy.org/>`_. Each of these implementations varies in some way from the language as documented in this manual, or introduces specific information beyond what's covered in the diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 460161cd320223..b91f8138553e62 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -11,11 +11,6 @@ {%- if is_deployment_preview %} <div id="deployment-preview-warning" style="padding: .5em; text-align: center; background-color: #fff2ba; color: #6a580e;"> - <div style="float: right; margin-top: -10px; margin-left: 10px;"> - <a href="https://www.netlify.com"> - <img src="https://www.netlify.com/img/global/badges/netlify-color-accent.svg" alt="Deploys by Netlify" /> - </a> - </div> {% trans %}This is a deploy preview created from a <a href="{{ repository_url }}/pull/{{ pr_id }}">pull request</a>. For authoritative documentation, see the {% endtrans %} <a href="https://docs.python.org/3/{{ pagename }}{{ file_suffix }}">{% trans %} the current stable release{% endtrans %}</a>. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b35e8454fa2a1a..9d4042ce5a7e8a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -370,7 +370,7 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict construction, O(n\ :sup:`2`) complexity. See - http://www.ocert.org/advisories/ocert-2011-003.html for details. + http://ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 9ae0270eaee7ab..69cd5c92d884d0 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -66,7 +66,7 @@ number of standard Unix command line editors, :program:`vim` and :program:`BBEdit` or :program:`TextWrangler` from Bare Bones Software (see http://www.barebones.com/products/bbedit/index.html) are good choices, as is :program:`TextMate` (see https://macromates.com/). Other editors include -:program:`Gvim` (https://macvim-dev.github.io/macvim/) and :program:`Aquamacs` +:program:`Gvim` (https://macvim.org/macvim/) and :program:`Aquamacs` (http://aquamacs.org/). To run your script from the Terminal window you must make sure that diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 380950eb507ffb..43e3c72f3e1cde 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -541,7 +541,7 @@ Besides the standard CPython distribution, there are modified packages including additional functionality. The following is a list of popular versions and their key features: -`ActivePython <https://www.activestate.com/activepython/>`_ +`ActivePython <https://www.activestate.com/products/python/>`_ Installer with multi-platform compatibility, documentation, PyWin32 `Anaconda <https://www.anaconda.com/download/>`_ diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index 4bcb2acae1e640..0eefefd863a68f 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -933,7 +933,7 @@ using it:: parser.parse( 'hamlet.xml' ) For more information, consult the Python documentation, or the XML HOWTO at -http://pyxml.sourceforge.net/topics/howto/xml-howto.html. +https://pyxml.sourceforge.net/topics/howto/xml-howto.html. DOM Support diff --git a/Doc/whatsnew/2.1.rst b/Doc/whatsnew/2.1.rst index 0136de58774038..676da702b39693 100644 --- a/Doc/whatsnew/2.1.rst +++ b/Doc/whatsnew/2.1.rst @@ -613,7 +613,7 @@ New and Improved Modules framework based on running embedded examples in docstrings and comparing the results against the expected output. PyUnit, contributed by Steve Purcell, is a unit testing framework inspired by JUnit, which was in turn an adaptation of - Kent Beck's Smalltalk testing framework. See http://pyunit.sourceforge.net/ for + Kent Beck's Smalltalk testing framework. See https://pyunit.sourceforge.net/ for more information about PyUnit. * The :mod:`difflib` module contains a class, :class:`SequenceMatcher`, which diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 0c3bfda1933957..82aff0be1ed3b3 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -632,10 +632,10 @@ queen threatens another) and the Knight's Tour (a route that takes a knight to every square of an $NxN$ chessboard without visiting any square twice). The idea of generators comes from other programming languages, especially Icon -(https://www.cs.arizona.edu/icon/), where the idea of generators is central. In +(https://www2.cs.arizona.edu/icon/), where the idea of generators is central. In Icon, every expression and function call behaves like a generator. One example from "An Overview of the Icon Programming Language" at -https://www.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks +https://www2.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks like:: sentence := "Store it in the neighboring harbor" diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index c6e2003e92f1b3..af489d7cb45c2a 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -218,10 +218,10 @@ queen threatens another) and the Knight's Tour (a route that takes a knight to every square of an $NxN$ chessboard without visiting any square twice). The idea of generators comes from other programming languages, especially Icon -(https://www.cs.arizona.edu/icon/), where the idea of generators is central. In +(https://www2.cs.arizona.edu/icon/), where the idea of generators is central. In Icon, every expression and function call behaves like a generator. One example from "An Overview of the Icon Programming Language" at -https://www.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks +https://www2.cs.arizona.edu/icon/docs/ipd266.htm gives an idea of what this looks like:: sentence := "Store it in the neighboring harbor" @@ -1332,7 +1332,7 @@ complete list of changes, or look through the CVS logs for all the details. (Contributed by Kevin O'Connor.) * The IDLE integrated development environment has been updated using the code - from the IDLEfork project (http://idlefork.sourceforge.net). The most notable feature is + from the IDLEfork project (https://idlefork.sourceforge.net). The most notable feature is that the code being developed is now executed in a subprocess, meaning that there's no longer any need for manual ``reload()`` operations. IDLE's core code has been incorporated into the standard library as the :mod:`idlelib` package. diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 63e819876ce310..98dc83fe935d5e 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -756,7 +756,7 @@ API that perform ASCII-only conversions, ignoring the locale setting: :c:expr:`double` to an ASCII string. The code for these functions came from the GLib library -(https://developer.gnome.org/glib/stable/), whose developers kindly +(https://developer-old.gnome.org/glib/2.26/), whose developers kindly relicensed the relevant functions and donated them to the Python Software Foundation. The :mod:`locale` module can now change the numeric locale, letting extensions such as GTK+ produce the correct results. diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 4ee2aacb108a36..84bb651e68eed5 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -1433,7 +1433,7 @@ one, :func:`math.trunc`, that's been backported to Python 2.6. `Scheme's numerical tower <https://www.gnu.org/software/guile/manual/html_node/Numerical-Tower.html#Numerical-Tower>`__, from the Guile manual. - `Scheme's number datatypes <https://schemers.org/Documents/Standards/R5RS/HTML/r5rs-Z-H-9.html#%_sec_6.2>`__ from the R5RS Scheme specification. + `Scheme's number datatypes <https://conservatory.scheme.org/schemers/Documents/Standards/R5RS/HTML/r5rs-Z-H-9.html#%_sec_6.2>`__ from the R5RS Scheme specification. The :mod:`fractions` Module @@ -2363,7 +2363,7 @@ changes, or look through the Subversion logs for all the details. negotiation itself. (Patch contributed by Bill Fenner; :issue:`829951`.) -* The :mod:`socket` module now supports TIPC (http://tipc.sourceforge.net/), +* The :mod:`socket` module now supports TIPC (https://tipc.sourceforge.net/), a high-performance non-IP-based protocol designed for use in clustered environments. TIPC addresses are 4- or 5-tuples. (Contributed by Alberto Bertogli; :issue:`1646`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 10fcfb6a0b5639..6b591d5e184ea7 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -220,7 +220,7 @@ The copy of the :ref:`launcher` included with Python 3.11 has been significantly updated. It now supports company/tag syntax as defined in :pep:`514` using the ``-V:<company>/<tag>`` argument instead of the limited ``-<major>.<minor>``. This allows launching distributions other than ``PythonCore``, -the one hosted on `python.org <https://python.org>`_. +the one hosted on `python.org <https://www.python.org>`_. When using ``-V:`` selectors, either company or tag can be omitted, but all installs will be searched. For example, ``-V:OtherPython/`` will select the @@ -666,19 +666,11 @@ enum for :meth:`~object.__str__` and :meth:`~object.__format__` (used by :func:`str`, :func:`format` and :term:`f-string`\s). -* Changed :class:`~enum.IntEnum`, :class:`~enum.IntFlag` and :class:`~enum.StrEnum` - to now inherit from :class:`~enum.ReprEnum`, - so their :func:`str` output now matches :func:`format` - (both ``str(AnIntEnum.ONE)`` and ``format(AnIntEnum.ONE)`` return ``'1'``, - whereas before ``str(AnIntEnum.ONE)`` returned ``'AnIntEnum.ONE'``. - -* Changed :meth:`Enum.__format__() <enum.Enum.__format__>` - (the default for :func:`format`, :meth:`str.format` and :term:`f-string`\s) - of enums with mixed-in types (e.g. :class:`int`, :class:`str`) - to also include the class name in the output, not just the member's key. - This matches the existing behavior of :meth:`enum.Enum.__str__`, - returning e.g. ``'AnEnum.MEMBER'`` for an enum ``AnEnum(str, Enum)`` - instead of just ``'MEMBER'``. +* Changed :meth:`Enum.__format__() <enum.Enum.__format__>` (the default for + :func:`format`, :meth:`str.format` and :term:`f-string`\s) to always produce + the same result as :meth:`Enum.__str__()`: for enums inheriting from + :class:`~enum.ReprEnum` it will be the member's value; for all other enums + it will be the enum and member name (e.g. ``Color.RED``). * Added a new *boundary* class parameter to :class:`~enum.Flag` enums and the :class:`~enum.FlagBoundary` enum with its options, @@ -2489,7 +2481,7 @@ Porting to Python 3.11 #endif Or use the `pythoncapi_compat project - <https://github.com/python/pythoncapi_compat>`__ to get these two + <https://github.com/python/pythoncapi-compat>`__ to get these two functions on older Python versions. * Changes of the :c:type:`PyThreadState` structure members: @@ -2541,8 +2533,8 @@ Porting to Python 3.11 } #endif - Or use `the pythoncapi_compat project - <https://github.com/python/pythoncapi_compat>`__ to get these functions + Or use `the pythoncapi-compat project + <https://github.com/python/pythoncapi-compat>`__ to get these functions on old Python functions. * Distributors are encouraged to build Python with the optimized Blake2 diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index f4ee30b0d4d9eb..a3fce7ccacf7c1 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -613,6 +613,11 @@ Optimizations * Speed up :class:`asyncio.Task` creation by deferring expensive string formatting. (Contributed by Itamar O in :gh:`103793`.) +* Added :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory` + functions to allow opting an event loop in to eager task execution, + speeding up some use-cases by up to 50%. + (Contributed by Jacob Bower & Itamar O in :gh:`102853`) + CPython bytecode changes ======================== diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 1b1455b72b9291..8dbe2a1d828b4e 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -785,8 +785,8 @@ functools (Contributed by Raymond Hettinger and incorporating design ideas from Jim Baker, Miki Tebeka, and Nick Coghlan; see `recipe 498245 - <https://code.activestate.com/recipes/498245>`_\, `recipe 577479 - <https://code.activestate.com/recipes/577479>`_\, :issue:`10586`, and + <https://code.activestate.com/recipes/498245/>`_\, `recipe 577479 + <https://code.activestate.com/recipes/577479/>`_\, :issue:`10586`, and :issue:`10593`.) * The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute @@ -2603,7 +2603,7 @@ Also, there were a number of updates to the Mac OS X build, see for details. For users running a 32/64-bit build, there is a known problem with the default Tcl/Tk on Mac OS X 10.6. Accordingly, we recommend installing an updated alternative such as -`ActiveState Tcl/Tk 8.5.9 <https://www.activestate.com/activetcl/downloads>`_\. +`ActiveState Tcl/Tk 8.5.9 <https://web.archive.org/web/20101208191259/https://www.activestate.com/activetcl/downloads>`_\. See https://www.python.org/download/mac/tcltk/ for additional details. Porting to Python 3.2 diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 9e8d42469b019c..5b6c3dcd45c6f5 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -1893,7 +1893,7 @@ socket * The :class:`~socket.socket` class now supports the PF_RDS protocol family (https://en.wikipedia.org/wiki/Reliable_Datagram_Sockets and - https://oss.oracle.com/projects/rds/). + `https://oss.oracle.com/projects/rds <https://web.archive.org/web/20130115155505/https://oss.oracle.com/projects/rds/>`__). * The :class:`~socket.socket` class now supports the ``PF_SYSTEM`` protocol family on OS X. (Contributed by Michael Goderbauer in :issue:`13777`.) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index f872579ef546f5..14b6425cea699e 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -425,7 +425,7 @@ are declared in the annotations:: While these annotations are available at runtime through the usual :attr:`__annotations__` attribute, *no automatic type checking happens at runtime*. Instead, it is assumed that a separate off-line type checker -(e.g. `mypy <http://mypy-lang.org>`_) will be used for on-demand +(e.g. `mypy <https://mypy-lang.org>`_) will be used for on-demand source code analysis. The type system supports unions, generic types, and a special type @@ -2212,7 +2212,7 @@ for details.) The :c:member:`PyTypeObject.tp_finalize` slot is now part of the stable ABI. Windows builds now require Microsoft Visual C++ 14.0, which -is available as part of `Visual Studio 2015 <https://www.visualstudio.com/>`_. +is available as part of `Visual Studio 2015 <https://visualstudio.microsoft.com/en/vs/older-downloads/#visual-studio-2015-and-other-products>`_. Extension modules now include a platform information tag in their filename on some platforms (the tag is optional, and CPython will import extensions without diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index e4294c88b58572..c7faaebfed62b3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -238,7 +238,7 @@ and the ``__annotations__`` attribute. and Guido van Rossum. Implemented by Ivan Levkivskyi. Tools that use or will use the new syntax: - `mypy <http://www.mypy-lang.org/>`_, + `mypy <https://www.mypy-lang.org/>`_, `pytype <https://github.com/google/pytype>`_, PyCharm, etc. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 37a6cf24e54562..85e088b64acb2d 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -2229,7 +2229,7 @@ The benchmarks were measured on an `Intel® Core™ i7-4960HQ processor <https://ark.intel.com/content/www/us/en/ark/products/76088/intel-core-i7-4960hq-processor-6m-cache-up-to-3-80-ghz.html>`_ running the macOS 64-bit builds found at -`python.org <https://www.python.org/downloads/mac-osx/>`_. +`python.org <https://www.python.org/downloads/macos/>`_. The benchmark script displays timings in nanoseconds. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index e974ee3a3f73ed..fd86db96302356 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -849,7 +849,7 @@ in nanoseconds. The benchmarks were measured on an `Intel® Core™ i7-4960HQ processor <https://ark.intel.com/content/www/us/en/ark/products/76088/intel-core-i7-4960hq-processor-6m-cache-up-to-3-80-ghz.html>`_ running the macOS 64-bit builds found at -`python.org <https://www.python.org/downloads/mac-osx/>`_. +`python.org <https://www.python.org/downloads/macos/>`_. Deprecated diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 0b038d31080be9..5a8cdd57c7845b 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -90,3 +90,6 @@ PyAPI_FUNC(int) PyObject_IS_GC(PyObject *obj); PyAPI_FUNC(int) PyType_SUPPORTS_WEAKREFS(PyTypeObject *type); PyAPI_FUNC(PyObject **) PyObject_GET_WEAKREFS_LISTPTR(PyObject *op); + +PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *, + size_t); diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 55378e3dfebf24..5d9342b562b002 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -116,6 +116,16 @@ _PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); } +PyObject *const * +_PyStack_UnpackDict(PyThreadState *tstate, + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject **p_kwnames); + +void +_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, + PyObject *kwnames); + +void _PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames); #ifdef __cplusplus } diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 1a032f652dddaf..4bd4ef57238f98 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -103,6 +103,10 @@ PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, PyObject *consts); +PyAPI_FUNC(PyCodeObject*) +_PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, + PyObject *instructions); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 4fa15d74b3ad64..f0740b68dd1114 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -882,6 +882,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst_dir_fd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(duration)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(e)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(encode)); @@ -972,6 +973,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(instructions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intern)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(intersection)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(is_running)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isatty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isinstance)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(isoformat)); @@ -1027,6 +1029,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(memlimit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(message)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metaclass)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index e19d8ff1b50468..234d5e2a09892c 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -370,6 +370,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(dst_dir_fd) STRUCT_FOR_ID(duration) STRUCT_FOR_ID(e) + STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) STRUCT_FOR_ID(element_factory) STRUCT_FOR_ID(encode) @@ -460,6 +461,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(instructions) STRUCT_FOR_ID(intern) STRUCT_FOR_ID(intersection) + STRUCT_FOR_ID(is_running) STRUCT_FOR_ID(isatty) STRUCT_FOR_ID(isinstance) STRUCT_FOR_ID(isoformat) @@ -515,6 +517,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(memlimit) STRUCT_FOR_ID(message) STRUCT_FOR_ID(metaclass) + STRUCT_FOR_ID(metadata) STRUCT_FOR_ID(method) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 2ca047846e0935..91853ad0525b55 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -272,8 +272,9 @@ _PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) { if (PyType_Check(op) && ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); static_builtin_state *state = _PyStaticType_GetState( - (PyTypeObject *)op); + interp, (PyTypeObject *)op); return _PyStaticType_GET_WEAKREFS_LISTPTR(state); } // Essentially _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(): @@ -330,10 +331,6 @@ extern int _Py_CheckSlotResult( const char *slot_name, int success); -// PyType_Ready() must be called if _PyType_IsReady() is false. -// See also the Py_TPFLAGS_READY flag. -#define _PyType_IsReady(type) ((type)->tp_dict != NULL) - // Test if a type supports weak references static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { return (type->tp_weaklistoffset != 0); @@ -391,8 +388,6 @@ _PyDictOrValues_SetValues(PyDictOrValues *ptr, PyDictValues *values) extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); -extern int _PyType_HasSubclasses(PyTypeObject *); -extern PyObject* _PyType_GetSubclasses(PyTypeObject *); // Access macro to the members which are floating "behind" the object static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index f96261a650dac7..7f8cc643ec0c96 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -39,7 +39,7 @@ extern PyStatus _PySys_Create( extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options); extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config); extern int _PySys_UpdateConfig(PyThreadState *tstate); -extern void _PySys_Fini(PyInterpreterState *interp); +extern void _PySys_FiniTypes(PyInterpreterState *interp); extern int _PyBuiltins_AddExceptions(PyObject * bltinmod); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 42c4874d9466bf..16f2147aa8e93b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -876,6 +876,7 @@ extern "C" { INIT_ID(dst_dir_fd), \ INIT_ID(duration), \ INIT_ID(e), \ + INIT_ID(eager_start), \ INIT_ID(effective_ids), \ INIT_ID(element_factory), \ INIT_ID(encode), \ @@ -966,6 +967,7 @@ extern "C" { INIT_ID(instructions), \ INIT_ID(intern), \ INIT_ID(intersection), \ + INIT_ID(is_running), \ INIT_ID(isatty), \ INIT_ID(isinstance), \ INIT_ID(isoformat), \ @@ -1021,6 +1023,7 @@ extern "C" { INIT_ID(memlimit), \ INIT_ID(message), \ INIT_ID(metaclass), \ + INIT_ID(metadata), \ INIT_ID(method), \ INIT_ID(mod), \ INIT_ID(mode), \ diff --git a/Include/internal/pycore_structseq.h b/Include/internal/pycore_structseq.h index d10a921c55ff8b..6f5dfc12707cf8 100644 --- a/Include/internal/pycore_structseq.h +++ b/Include/internal/pycore_structseq.h @@ -15,19 +15,23 @@ PyAPI_FUNC(PyTypeObject *) _PyStructSequence_NewType( PyStructSequence_Desc *desc, unsigned long tp_flags); -PyAPI_FUNC(int) _PyStructSequence_InitBuiltinWithFlags( +extern int _PyStructSequence_InitBuiltinWithFlags( + PyInterpreterState *interp, PyTypeObject *type, PyStructSequence_Desc *desc, unsigned long tp_flags); static inline int -_PyStructSequence_InitBuiltin(PyTypeObject *type, +_PyStructSequence_InitBuiltin(PyInterpreterState *interp, + PyTypeObject *type, PyStructSequence_Desc *desc) { - return _PyStructSequence_InitBuiltinWithFlags(type, desc, 0); + return _PyStructSequence_InitBuiltinWithFlags(interp, type, desc, 0); } -extern void _PyStructSequence_FiniType(PyTypeObject *type); +extern void _PyStructSequence_FiniBuiltin( + PyInterpreterState *interp, + PyTypeObject *type); #ifdef __cplusplus } diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 76253fd5fd864c..f865e51aeba503 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -104,10 +104,24 @@ _PyType_GetModuleState(PyTypeObject *type) } -extern int _PyStaticType_InitBuiltin(PyTypeObject *type); -extern static_builtin_state * _PyStaticType_GetState(PyTypeObject *); -extern void _PyStaticType_ClearWeakRefs(PyTypeObject *type); -extern void _PyStaticType_Dealloc(PyTypeObject *type); +extern int _PyStaticType_InitBuiltin(PyInterpreterState *, PyTypeObject *type); +extern static_builtin_state * _PyStaticType_GetState(PyInterpreterState *, PyTypeObject *); +extern void _PyStaticType_ClearWeakRefs(PyInterpreterState *, PyTypeObject *type); +extern void _PyStaticType_Dealloc(PyInterpreterState *, PyTypeObject *); + +PyAPI_FUNC(PyObject *) _PyType_GetDict(PyTypeObject *); +extern PyObject * _PyType_GetBases(PyTypeObject *type); +extern PyObject * _PyType_GetMRO(PyTypeObject *type); +extern PyObject* _PyType_GetSubclasses(PyTypeObject *); +extern int _PyType_HasSubclasses(PyTypeObject *); + +// PyType_Ready() must be called if _PyType_IsReady() is false. +// See also the Py_TPFLAGS_READY flag. +static inline int +_PyType_IsReady(PyTypeObject *type) +{ + return _PyType_GetDict(type) != NULL; +} PyObject * _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int *suppress_missing_attribute); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 6d9cd24d9f3a13..cd41b731537f0e 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -963,6 +963,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(e); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(eager_start); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(effective_ids); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1233,6 +1236,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(intersection); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(is_running); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(isatty); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1398,6 +1404,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(metaclass); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(metadata); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(method); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 74474f5bb8f976..5839c747a29275 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -626,7 +626,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String( /* --- ASCII Codecs ------------------------------------------------------- - Only 7-bit ASCII data is excepted. All other codes generate errors. + Only 7-bit ASCII data is expected. All other codes generate errors. */ diff --git a/Lib/asyncio/base_tasks.py b/Lib/asyncio/base_tasks.py index 26298e638cbf0d..c907b683413732 100644 --- a/Lib/asyncio/base_tasks.py +++ b/Lib/asyncio/base_tasks.py @@ -15,11 +15,13 @@ def _task_repr_info(task): info.insert(1, 'name=%r' % task.get_name()) - coro = coroutines._format_coroutine(task._coro) - info.insert(2, f'coro=<{coro}>') - if task._fut_waiter is not None: - info.insert(3, f'wait_for={task._fut_waiter!r}') + info.insert(2, f'wait_for={task._fut_waiter!r}') + + if task._coro: + coro = coroutines._format_coroutine(task._coro) + info.insert(2, f'coro=<{coro}>') + return info diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index c90d32c97add78..aa5269ade19a7f 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -6,6 +6,7 @@ 'wait', 'wait_for', 'as_completed', 'sleep', 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', 'current_task', 'all_tasks', + 'create_eager_task_factory', 'eager_task_factory', '_register_task', '_unregister_task', '_enter_task', '_leave_task', ) @@ -43,22 +44,26 @@ def all_tasks(loop=None): """Return a set of all tasks for the loop.""" if loop is None: loop = events.get_running_loop() - # Looping over a WeakSet (_all_tasks) isn't safe as it can be updated from another - # thread while we do so. Therefore we cast it to list prior to filtering. The list - # cast itself requires iteration, so we repeat it several times ignoring - # RuntimeErrors (which are not very likely to occur). See issues 34970 and 36607 for - # details. + # capturing the set of eager tasks first, so if an eager task "graduates" + # to a regular task in another thread, we don't risk missing it. + eager_tasks = list(_eager_tasks) + # Looping over the WeakSet isn't safe as it can be updated from another + # thread, therefore we cast it to list prior to filtering. The list cast + # itself requires iteration, so we repeat it several times ignoring + # RuntimeErrors (which are not very likely to occur). + # See issues 34970 and 36607 for details. + scheduled_tasks = None i = 0 while True: try: - tasks = list(_all_tasks) + scheduled_tasks = list(_scheduled_tasks) except RuntimeError: i += 1 if i >= 1000: raise else: break - return {t for t in tasks + return {t for t in itertools.chain(scheduled_tasks, eager_tasks) if futures._get_loop(t) is loop and not t.done()} @@ -93,7 +98,8 @@ class Task(futures._PyFuture): # Inherit Python Task implementation # status is still pending _log_destroy_pending = True - def __init__(self, coro, *, loop=None, name=None, context=None): + def __init__(self, coro, *, loop=None, name=None, context=None, + eager_start=False): super().__init__(loop=loop) if self._source_traceback: del self._source_traceback[-1] @@ -117,8 +123,11 @@ def __init__(self, coro, *, loop=None, name=None, context=None): else: self._context = context - self._loop.call_soon(self.__step, context=self._context) - _register_task(self) + if eager_start and self._loop.is_running(): + self.__eager_start() + else: + self._loop.call_soon(self.__step, context=self._context) + _register_task(self) def __del__(self): if self._state == futures._PENDING and self._log_destroy_pending: @@ -250,6 +259,25 @@ def uncancel(self): self._num_cancels_requested -= 1 return self._num_cancels_requested + def __eager_start(self): + prev_task = _swap_current_task(self._loop, self) + try: + _register_eager_task(self) + try: + self._context.run(self.__step_run_and_handle_result, None) + finally: + _unregister_eager_task(self) + finally: + try: + curtask = _swap_current_task(self._loop, prev_task) + assert curtask is self + finally: + if self.done(): + self._coro = None + self = None # Needed to break cycles when an exception occurs. + else: + _register_task(self) + def __step(self, exc=None): if self.done(): raise exceptions.InvalidStateError( @@ -258,11 +286,17 @@ def __step(self, exc=None): if not isinstance(exc, exceptions.CancelledError): exc = self._make_cancelled_error() self._must_cancel = False - coro = self._coro self._fut_waiter = None _enter_task(self._loop, self) - # Call either coro.throw(exc) or coro.send(None). + try: + self.__step_run_and_handle_result(exc) + finally: + _leave_task(self._loop, self) + self = None # Needed to break cycles when an exception occurs. + + def __step_run_and_handle_result(self, exc): + coro = self._coro try: if exc is None: # We use the `send` method directly, because coroutines @@ -334,7 +368,6 @@ def __step(self, exc=None): self._loop.call_soon( self.__step, new_exc, context=self._context) finally: - _leave_task(self._loop, self) self = None # Needed to break cycles when an exception occurs. def __wakeup(self, future): @@ -897,8 +930,27 @@ def callback(): return future -# WeakSet containing all alive tasks. -_all_tasks = weakref.WeakSet() +def create_eager_task_factory(custom_task_constructor): + + if "eager_start" not in inspect.signature(custom_task_constructor).parameters: + raise TypeError( + "Provided constructor does not support eager task execution") + + def factory(loop, coro, *, name=None, context=None): + return custom_task_constructor( + coro, loop=loop, name=name, context=context, eager_start=True) + + + return factory + +eager_task_factory = create_eager_task_factory(Task) + + +# Collectively these two sets hold references to the complete set of active +# tasks. Eagerly executed tasks use a faster regular set as an optimization +# but may graduate to a WeakSet if the task blocks on IO. +_scheduled_tasks = weakref.WeakSet() +_eager_tasks = set() # Dictionary containing tasks that are currently active in # all running event loops. {EventLoop: Task} @@ -906,8 +958,13 @@ def callback(): def _register_task(task): - """Register a new task in asyncio as executed by loop.""" - _all_tasks.add(task) + """Register an asyncio Task scheduled to run on an event loop.""" + _scheduled_tasks.add(task) + + +def _register_eager_task(task): + """Register an asyncio Task about to be eagerly executed.""" + _eager_tasks.add(task) def _enter_task(loop, task): @@ -926,28 +983,49 @@ def _leave_task(loop, task): del _current_tasks[loop] +def _swap_current_task(loop, task): + prev_task = _current_tasks.get(loop) + if task is None: + del _current_tasks[loop] + else: + _current_tasks[loop] = task + return prev_task + + def _unregister_task(task): - """Unregister a task.""" - _all_tasks.discard(task) + """Unregister a completed, scheduled Task.""" + _scheduled_tasks.discard(task) + + +def _unregister_eager_task(task): + """Unregister a task which finished its first eager step.""" + _eager_tasks.discard(task) _py_current_task = current_task _py_register_task = _register_task +_py_register_eager_task = _register_eager_task _py_unregister_task = _unregister_task +_py_unregister_eager_task = _unregister_eager_task _py_enter_task = _enter_task _py_leave_task = _leave_task +_py_swap_current_task = _swap_current_task try: - from _asyncio import (_register_task, _unregister_task, - _enter_task, _leave_task, - _all_tasks, _current_tasks, + from _asyncio import (_register_task, _register_eager_task, + _unregister_task, _unregister_eager_task, + _enter_task, _leave_task, _swap_current_task, + _scheduled_tasks, _eager_tasks, _current_tasks, current_task) except ImportError: pass else: _c_current_task = current_task _c_register_task = _register_task + _c_register_eager_task = _register_eager_task _c_unregister_task = _unregister_task + _c_unregister_eager_task = _unregister_eager_task _c_enter_task = _enter_task _c_leave_task = _leave_task + _c_swap_current_task = _swap_current_task diff --git a/Lib/calendar.py b/Lib/calendar.py index bbd4fea3b88ca4..ea56f12ccc41d0 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -83,7 +83,6 @@ class Day(IntEnum): SUNDAY = 6 - # Number of days per month (except for February in leap years) mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] @@ -156,7 +155,7 @@ def weekday(year, month, day): """Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31).""" if not datetime.MINYEAR <= year <= datetime.MAXYEAR: year = 2000 + year % 400 - return datetime.date(year, month, day).weekday() + return Day(datetime.date(year, month, day).weekday()) def monthrange(year, month): diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index a73cdc22a5f4b3..b0b8a773b7594f 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1227,8 +1227,10 @@ def _add_slots(cls, is_frozen, weakref_slot): if is_frozen: # Need this for pickling frozen classes with slots. - cls.__getstate__ = _dataclass_getstate - cls.__setstate__ = _dataclass_setstate + if '__getstate__' not in cls_dict: + cls.__getstate__ = _dataclass_getstate + if '__setstate__' not in cls_dict: + cls.__setstate__ = _dataclass_setstate return cls diff --git a/Lib/http/client.py b/Lib/http/client.py index 0f5cd35247ae82..74f7bcb68fb6bc 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -941,23 +941,26 @@ def _tunnel(self): del headers response = self.response_class(self.sock, method=self._method) - (version, code, message) = response._read_status() + try: + (version, code, message) = response._read_status() - if code != http.HTTPStatus.OK: - self.close() - raise OSError(f"Tunnel connection failed: {code} {message.strip()}") - while True: - line = response.fp.readline(_MAXLINE + 1) - if len(line) > _MAXLINE: - raise LineTooLong("header line") - if not line: - # for sites which EOF without sending a trailer - break - if line in (b'\r\n', b'\n', b''): - break + if code != http.HTTPStatus.OK: + self.close() + raise OSError(f"Tunnel connection failed: {code} {message.strip()}") + while True: + line = response.fp.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise LineTooLong("header line") + if not line: + # for sites which EOF without sending a trailer + break + if line in (b'\r\n', b'\n', b''): + break - if self.debuglevel > 0: - print('header:', line.decode()) + if self.debuglevel > 0: + print('header:', line.decode()) + finally: + response.close() def connect(self): """Connect to the host and port specified in __init__.""" diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index e9f19c145c8673..b4df353012b788 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -310,7 +310,7 @@ def recolorize_main(self): # crumb telling the next invocation to resume here # in case update tells us to leave. self.tag_add("TODO", next) - self.update() + self.update_idletasks() if self.stop_colorizing: if DEBUG: print("colorizing stopped") return diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index 5506fd2b0e22a5..fb52b3a0179553 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -57,7 +57,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.editwin.per.close() - cls.root.update() + cls.root.update_idletasks() cls.root.destroy() del cls.text, cls.text_frame, cls.editwin, cls.root @@ -695,7 +695,8 @@ def test_mousewheel(self): delta = -1 if sys.platform == 'darwin' else 120 sidebar.canvas.event_generate('<MouseWheel>', x=0, y=0, delta=delta) yield - self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) + if sys.platform != 'darwin': # .update_idletasks() does not work. + self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) # Scroll back down using the <Button-5> event. sidebar.canvas.event_generate('<Button-5>', x=0, y=0) diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index ac67c904ab9797..610031e26f1dff 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -112,7 +112,7 @@ def write(self, s, tags=(), mark="insert"): assert isinstance(s, str) self.text.insert(mark, s, tags) self.text.see(mark) - self.text.update() + self.text.update_idletasks() return len(s) def writelines(self, lines): diff --git a/Lib/inspect.py b/Lib/inspect.py index 6d1d7b766cb3bb..92c2675cfd7d32 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1766,7 +1766,9 @@ def stack(context=1): def trace(context=1): """Return a list of records for the stack below the current exception.""" - return getinnerframes(sys.exc_info()[2], context) + exc = sys.exception() + tb = None if exc is None else exc.__traceback__ + return getinnerframes(tb, context) # ------------------------------------------------ static version of getattr diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 67bc0d19c90af0..f999f8263a0b85 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -53,13 +53,18 @@ def _ignore_error(exception): return (getattr(exception, 'errno', None) in _IGNORED_ERRNOS or getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) -# -# Globbing helpers -# + +def _is_wildcard_pattern(pat): + # Whether this pattern needs actual matching using fnmatch, or can + # be looked up directly as a file. + return "*" in pat or "?" in pat or "[" in pat def _is_case_sensitive(flavour): return flavour.normcase('Aa') == 'Aa' +# +# Globbing helpers +# @functools.lru_cache() def _make_selector(pattern_parts, case_sensitive): @@ -69,10 +74,10 @@ def _make_selector(pattern_parts, case_sensitive): return _TerminatingSelector() if pat == '**': cls = _RecursiveWildcardSelector - elif '**' in pat: - raise ValueError("Invalid pattern: '**' can only be an entire path component") elif pat == '..': cls = _ParentSelector + elif '**' in pat: + raise ValueError("Invalid pattern: '**' can only be an entire path component") else: cls = _WildcardSelector return cls(pat, child_parts, case_sensitive) @@ -112,9 +117,10 @@ class _ParentSelector(_Selector): def __init__(self, name, child_parts, case_sensitive): _Selector.__init__(self, child_parts, case_sensitive) - def _select_from(self, parent_path, scandir): + def _select_from(self, parent_path, scandir): path = parent_path._make_child_relpath('..') - return self.successor._select_from(path, scandir) + for p in self.successor._select_from(path, scandir): + yield p class _WildcardSelector(_Selector): @@ -290,18 +296,27 @@ def __reduce__(self): return (self.__class__, self.parts) def __init__(self, *args): - if not args: - path = '' - elif len(args) == 1: - path = os.fspath(args[0]) + paths = [] + for arg in args: + if isinstance(arg, PurePath): + path = arg._raw_path + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + if len(paths) == 0: + self._raw_path = '' + elif len(paths) == 1: + self._raw_path = paths[0] else: - path = self._flavour.join(*args) - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - self._raw_path = path + self._raw_path = self._flavour.join(*paths) @classmethod def _parse_path(cls, path): @@ -610,7 +625,7 @@ def joinpath(self, *args): paths) or a totally different path (if one of the arguments is anchored). """ - return self.__class__(self._raw_path, *args) + return self.__class__(self, *args) def __truediv__(self, key): try: @@ -620,7 +635,7 @@ def __truediv__(self, key): def __rtruediv__(self, key): try: - return type(self)(key, self._raw_path) + return type(self)(key, self) except TypeError: return NotImplemented @@ -858,7 +873,7 @@ def absolute(self): cwd = self._flavour.abspath(self.drive) else: cwd = os.getcwd() - return type(self)(cwd, self._raw_path) + return type(self)(cwd, self) def resolve(self, strict=False): """ diff --git a/Lib/platform.py b/Lib/platform.py index 790ef860bf106e..7bb222088d5061 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -136,11 +136,11 @@ 'pl': 200, 'p': 200, } -_component_re = re.compile(r'([0-9]+|[._+-])') def _comparable_version(version): + component_re = re.compile(r'([0-9]+|[._+-])') result = [] - for v in _component_re.split(version): + for v in component_re.split(version): if v not in '._+-': try: v = int(v, 10) @@ -152,11 +152,6 @@ def _comparable_version(version): ### Platform specific APIs -_libc_search = re.compile(b'(__libc_init)' - b'|' - b'(GLIBC_([0-9.]+))' - b'|' - br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) def libc_ver(executable=None, lib='', version='', chunksize=16384): @@ -190,6 +185,12 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): # sys.executable is not set. return lib, version + libc_search = re.compile(b'(__libc_init)' + b'|' + b'(GLIBC_([0-9.]+))' + b'|' + br'(libc(_\w+)?\.so(?:\.(\d[0-9.]*))?)', re.ASCII) + V = _comparable_version # We use os.path.realpath() # here to work around problems with Cygwin not being @@ -200,7 +201,7 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): pos = 0 while pos < len(binary): if b'libc' in binary or b'GLIBC' in binary: - m = _libc_search.search(binary, pos) + m = libc_search.search(binary, pos) else: m = None if not m or m.end() == len(binary): @@ -247,9 +248,6 @@ def _norm_version(version, build=''): version = '.'.join(strings[:3]) return version -_ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' - r'.*' - r'\[.* ([\d.]+)\])') # Examples of VER command output: # @@ -295,9 +293,13 @@ def _syscmd_ver(system='', release='', version='', else: return system, release, version + ver_output = re.compile(r'(?:([\w ]+) ([\w.]+) ' + r'.*' + r'\[.* ([\d.]+)\])') + # Parse the output info = info.strip() - m = _ver_output.match(info) + m = ver_output.match(info) if m is not None: system, release, version = m.groups() # Strip trailing dots from version and release @@ -1033,18 +1035,6 @@ def processor(): ### Various APIs for extracting information from sys.version -_sys_version_parser = re.compile( - r'([\w.+]+)\s*' # "version<space>" - r'\(#?([^,]+)' # "(#buildno" - r'(?:,\s*([\w ]*)' # ", builddate" - r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)<space>" - r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" - -_pypy_sys_version_parser = re.compile( - r'([\w.+]+)\s*' - r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' - r'\[PyPy [^\]]+\]?') - _sys_version_cache = {} def _sys_version(sys_version=None): @@ -1076,10 +1066,17 @@ def _sys_version(sys_version=None): if result is not None: return result + sys_version_parser = re.compile( + r'([\w.+]+)\s*' # "version<space>" + r'\(#?([^,]+)' # "(#buildno" + r'(?:,\s*([\w ]*)' # ", builddate" + r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)<space>" + r'\[([^\]]+)\]?', re.ASCII) # "[compiler]" + if sys.platform.startswith('java'): # Jython name = 'Jython' - match = _sys_version_parser.match(sys_version) + match = sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse Jython sys.version: %s' % @@ -1091,8 +1088,13 @@ def _sys_version(sys_version=None): elif "PyPy" in sys_version: # PyPy + pypy_sys_version_parser = re.compile( + r'([\w.+]+)\s*' + r'\(#?([^,]+),\s*([\w ]+),\s*([\w :]+)\)\s*' + r'\[PyPy [^\]]+\]?') + name = "PyPy" - match = _pypy_sys_version_parser.match(sys_version) + match = pypy_sys_version_parser.match(sys_version) if match is None: raise ValueError("failed to parse PyPy sys.version: %s" % repr(sys_version)) @@ -1101,7 +1103,7 @@ def _sys_version(sys_version=None): else: # CPython - match = _sys_version_parser.match(sys_version) + match = sys_version_parser.match(sys_version) if match is None: raise ValueError( 'failed to parse CPython sys.version: %s' % @@ -1290,13 +1292,6 @@ def platform(aliased=False, terse=False): ### freedesktop.org os-release standard # https://www.freedesktop.org/software/systemd/man/os-release.html -# NAME=value with optional quotes (' or "). The regular expression is less -# strict than shell lexer, but that's ok. -_os_release_line = re.compile( - "^(?P<name>[a-zA-Z0-9_]+)=(?P<quote>[\"\']?)(?P<value>.*)(?P=quote)$" -) -# unescape five special characters mentioned in the standard -_os_release_unescape = re.compile(r"\\([\\\$\"\'`])") # /etc takes precedence over /usr/lib _os_release_candidates = ("/etc/os-release", "/usr/lib/os-release") _os_release_cache = None @@ -1311,10 +1306,18 @@ def _parse_os_release(lines): "PRETTY_NAME": "Linux", } + # NAME=value with optional quotes (' or "). The regular expression is less + # strict than shell lexer, but that's ok. + os_release_line = re.compile( + "^(?P<name>[a-zA-Z0-9_]+)=(?P<quote>[\"\']?)(?P<value>.*)(?P=quote)$" + ) + # unescape five special characters mentioned in the standard + os_release_unescape = re.compile(r"\\([\\\$\"\'`])") + for line in lines: - mo = _os_release_line.match(line) + mo = os_release_line.match(line) if mo is not None: - info[mo.group('name')] = _os_release_unescape.sub( + info[mo.group('name')] = os_release_unescape.sub( r"\1", mo.group('value') ) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index e9bb72a7d77ee1..61595277ed6d5a 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -143,6 +143,14 @@ def __str__(self) -> str: # set of tests that we don't want to be executed when using regrtest NOTTESTS = set() +#If these test directories are encountered recurse into them and treat each +# test_ .py or dir as a separate test module. This can increase parallelism. +# Beware this can't generally be done for any directory with sub-tests as the +# __init__.py may do things which alter what tests are to be run. + +SPLITTESTDIRS = { + "test_asyncio", +} # Storage of uncollectable objects FOUND_GARBAGE = [] @@ -158,7 +166,7 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_dirs=SPLITTESTDIRS, base_mod=""): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) names = os.listdir(testdir) @@ -166,8 +174,13 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): others = set(stdtests) | nottests for name in names: mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) + if mod[:5] == "test_" and mod not in others: + if mod in split_test_dirs: + subdir = os.path.join(testdir, mod) + mod = f"{base_mod or 'test'}.{mod}" + tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod)) + elif ext in (".py", ""): + tests.append(f"{base_mod}.{mod}" if base_mod else mod) return stdtests + sorted(tests) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 1d9b889c920986..357ec44dbc218d 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,7 +3,7 @@ import unittest import dis import io -from _testinternalcapi import compiler_codegen, optimize_cfg +from _testinternalcapi import compiler_codegen, optimize_cfg, assemble_code_object _UNSPECIFIED = object() @@ -108,6 +108,18 @@ def normalize_insts(self, insts): res.append((opcode, arg, *loc)) return res + def complete_insts_info(self, insts): + # fill in omitted fields in location, and oparg 0 for ops with no arg. + res = [] + for item in insts: + assert isinstance(item, tuple) + inst = list(item) + opcode = dis.opmap[inst[0]] + oparg = inst[1] + loc = inst[2:] + [-1] * (6 - len(inst)) + res.append((opcode, oparg, *loc)) + return res + class CodegenTestCase(CompilationStepTestCase): @@ -118,20 +130,14 @@ def generate_code(self, ast): class CfgOptimizationTestCase(CompilationStepTestCase): - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] - for item in insts: - assert isinstance(item, tuple) - inst = list(reversed(item)) - opcode = dis.opmap[inst.pop()] - oparg = inst.pop() - loc = inst + [-1] * (4 - len(inst)) - res.append((opcode, oparg, *loc)) - return res - def get_optimized(self, insts, consts): insts = self.normalize_insts(insts) insts = self.complete_insts_info(insts) insts = optimize_cfg(insts, consts) return insts, consts + +class AssemblerTestCase(CompilationStepTestCase): + + def get_code_object(self, filename, insts, metadata): + co = assemble_code_object(filename, insts, metadata) + return co diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py new file mode 100644 index 00000000000000..fe690934292a86 --- /dev/null +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -0,0 +1,344 @@ +"""Tests for base_events.py""" + +import asyncio +import contextvars +import gc +import time +import unittest + +from types import GenericAlias +from unittest import mock +from asyncio import base_events +from asyncio import tasks +from test.test_asyncio import utils as test_utils +from test.test_asyncio.test_tasks import get_innermost_context +from test import support + +MOCK_ANY = mock.ANY + + +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + +class EagerTaskFactoryLoopTests: + + Task = None + + def run_coro(self, coro): + """ + Helper method to run the `coro` coroutine in the test event loop. + It helps with making sure the event loop is running before starting + to execute `coro`. This is important for testing the eager step + functionality, since an eager step is taken only if the event loop + is already running. + """ + + async def coro_runner(): + self.assertTrue(asyncio.get_event_loop().is_running()) + return await coro + + return self.loop.run_until_complete(coro) + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.eager_task_factory = asyncio.create_eager_task_factory(self.Task) + self.loop.set_task_factory(self.eager_task_factory) + self.set_event_loop(self.loop) + + def test_eager_task_factory_set(self): + self.assertIsNotNone(self.eager_task_factory) + self.assertIs(self.loop.get_task_factory(), self.eager_task_factory) + + async def noop(): pass + + async def run(): + t = self.loop.create_task(noop()) + self.assertIsInstance(t, self.Task) + await t + + self.run_coro(run()) + + def test_await_future_during_eager_step(self): + + async def set_result(fut, val): + fut.set_result(val) + + async def run(): + fut = self.loop.create_future() + t = self.loop.create_task(set_result(fut, 'my message')) + # assert the eager step completed the task + self.assertTrue(t.done()) + return await fut + + self.assertEqual(self.run_coro(run()), 'my message') + + def test_eager_completion(self): + + async def coro(): + return 'hello' + + async def run(): + t = self.loop.create_task(coro()) + # assert the eager step completed the task + self.assertTrue(t.done()) + return await t + + self.assertEqual(self.run_coro(run()), 'hello') + + def test_block_after_eager_step(self): + + async def coro(): + await asyncio.sleep(0.1) + return 'finished after blocking' + + async def run(): + t = self.loop.create_task(coro()) + self.assertFalse(t.done()) + result = await t + self.assertTrue(t.done()) + return result + + self.assertEqual(self.run_coro(run()), 'finished after blocking') + + def test_cancellation_after_eager_completion(self): + + async def coro(): + return 'finished without blocking' + + async def run(): + t = self.loop.create_task(coro()) + t.cancel() + result = await t + # finished task can't be cancelled + self.assertFalse(t.cancelled()) + return result + + self.assertEqual(self.run_coro(run()), 'finished without blocking') + + def test_cancellation_after_eager_step_blocks(self): + + async def coro(): + await asyncio.sleep(0.1) + return 'finished after blocking' + + async def run(): + t = self.loop.create_task(coro()) + t.cancel('cancellation message') + self.assertGreater(t.cancelling(), 0) + result = await t + + with self.assertRaises(asyncio.CancelledError) as cm: + self.run_coro(run()) + + self.assertEqual('cancellation message', cm.exception.args[0]) + + def test_current_task(self): + captured_current_task = None + + async def coro(): + nonlocal captured_current_task + captured_current_task = asyncio.current_task() + # verify the task before and after blocking is identical + await asyncio.sleep(0.1) + self.assertIs(asyncio.current_task(), captured_current_task) + + async def run(): + t = self.loop.create_task(coro()) + self.assertIs(captured_current_task, t) + await t + + self.run_coro(run()) + captured_current_task = None + + def test_all_tasks_with_eager_completion(self): + captured_all_tasks = None + + async def coro(): + nonlocal captured_all_tasks + captured_all_tasks = asyncio.all_tasks() + + async def run(): + t = self.loop.create_task(coro()) + self.assertIn(t, captured_all_tasks) + self.assertNotIn(t, asyncio.all_tasks()) + + self.run_coro(run()) + + def test_all_tasks_with_blocking(self): + captured_eager_all_tasks = None + + async def coro(fut1, fut2): + nonlocal captured_eager_all_tasks + captured_eager_all_tasks = asyncio.all_tasks() + await fut1 + fut2.set_result(None) + + async def run(): + fut1 = self.loop.create_future() + fut2 = self.loop.create_future() + t = self.loop.create_task(coro(fut1, fut2)) + self.assertIn(t, captured_eager_all_tasks) + self.assertIn(t, asyncio.all_tasks()) + fut1.set_result(None) + await fut2 + self.assertNotIn(t, asyncio.all_tasks()) + + self.run_coro(run()) + + def test_context_vars(self): + cv = contextvars.ContextVar('cv', default=0) + + coro_first_step_ran = False + coro_second_step_ran = False + + async def coro(): + nonlocal coro_first_step_ran + nonlocal coro_second_step_ran + self.assertEqual(cv.get(), 1) + cv.set(2) + self.assertEqual(cv.get(), 2) + coro_first_step_ran = True + await asyncio.sleep(0.1) + self.assertEqual(cv.get(), 2) + cv.set(3) + self.assertEqual(cv.get(), 3) + coro_second_step_ran = True + + async def run(): + cv.set(1) + t = self.loop.create_task(coro()) + self.assertTrue(coro_first_step_ran) + self.assertFalse(coro_second_step_ran) + self.assertEqual(cv.get(), 1) + await t + self.assertTrue(coro_second_step_ran) + self.assertEqual(cv.get(), 1) + + self.run_coro(run()) + + +class PyEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): + Task = tasks._PyTask + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CEagerTaskFactoryLoopTests(EagerTaskFactoryLoopTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + + +class AsyncTaskCounter: + def __init__(self, loop, *, task_class, eager): + self.suspense_count = 0 + self.task_count = 0 + + def CountingTask(*args, eager_start=False, **kwargs): + if not eager_start: + self.task_count += 1 + kwargs["eager_start"] = eager_start + return task_class(*args, **kwargs) + + if eager: + factory = asyncio.create_eager_task_factory(CountingTask) + else: + def factory(loop, coro, **kwargs): + return CountingTask(coro, loop=loop, **kwargs) + loop.set_task_factory(factory) + + def get(self): + return self.task_count + + +async def awaitable_chain(depth): + if depth == 0: + return 0 + return 1 + await awaitable_chain(depth - 1) + + +async def recursive_taskgroups(width, depth): + if depth == 0: + return + + async with asyncio.TaskGroup() as tg: + futures = [ + tg.create_task(recursive_taskgroups(width, depth - 1)) + for _ in range(width) + ] + + +async def recursive_gather(width, depth): + if depth == 0: + return + + await asyncio.gather( + *[recursive_gather(width, depth - 1) for _ in range(width)] + ) + + +class BaseTaskCountingTests: + + Task = None + eager = None + expected_task_count = None + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.counter = AsyncTaskCounter(self.loop, task_class=self.Task, eager=self.eager) + self.set_event_loop(self.loop) + + def test_awaitables_chain(self): + observed_depth = self.loop.run_until_complete(awaitable_chain(100)) + self.assertEqual(observed_depth, 100) + self.assertEqual(self.counter.get(), 0 if self.eager else 1) + + def test_recursive_taskgroups(self): + num_tasks = self.loop.run_until_complete(recursive_taskgroups(5, 4)) + self.assertEqual(self.counter.get(), self.expected_task_count) + + def test_recursive_gather(self): + self.loop.run_until_complete(recursive_gather(5, 4)) + self.assertEqual(self.counter.get(), self.expected_task_count) + + +class BaseNonEagerTaskFactoryTests(BaseTaskCountingTests): + eager = False + expected_task_count = 781 # 1 + 5 + 5^2 + 5^3 + 5^4 + + +class BaseEagerTaskFactoryTests(BaseTaskCountingTests): + eager = True + expected_task_count = 0 + + +class NonEagerTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = asyncio.Task + + +class EagerTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = asyncio.Task + + +class NonEagerPyTaskTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = tasks._PyTask + + +class EagerPyTaskTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = tasks._PyTask + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class NonEagerCTaskTests(BaseNonEagerTaskFactoryTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class EagerCTaskTests(BaseEagerTaskFactoryTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 03388e8c55d5a8..24e472b5fee828 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -493,11 +493,11 @@ def test_format(self): class CalendarTestCase(unittest.TestCase): def test_deprecation_warning(self): - with warnings.catch_warnings(record=True) as w: + with self.assertWarnsRegex( + DeprecationWarning, + "The 'January' attribute is deprecated, use 'JANUARY' instead" + ): calendar.January - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, DeprecationWarning) - self.assertIn("The 'January' attribute is deprecated, use 'JANUARY' instead", str(w[0].message)) def test_isleap(self): # Make sure that the return is right for a few years, and diff --git a/Lib/test/test_capi/test_immortal.py b/Lib/test/test_capi/test_immortal.py new file mode 100644 index 00000000000000..ef5d32b7f01935 --- /dev/null +++ b/Lib/test/test_capi/test_immortal.py @@ -0,0 +1,16 @@ +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +class TestCAPI(unittest.TestCase): + def test_immortal_builtins(self): + _testcapi.test_immortal_builtins() + + def test_immortal_small_ints(self): + _testcapi.test_immortal_small_ints() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 9470cf12a7d1c4..9d5d1ca6e7dce2 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1043,6 +1043,20 @@ class dictsub(dict): ... # dict subclasses must work self.assertEqual(_testcapi.function_get_kw_defaults(some), None) self.assertEqual(some.__kwdefaults__, None) + def test_unstable_gc_new_with_extra_data(self): + class Data(_testcapi.ObjExtraData): + __slots__ = ('x', 'y') + + d = Data() + d.x = 10 + d.y = 20 + d.extra = 30 + self.assertEqual(d.x, 10) + self.assertEqual(d.y, 20) + self.assertEqual(d.extra, 30) + del d.extra + self.assertIsNone(d.extra) + class TestPendingCalls(unittest.TestCase): diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py new file mode 100644 index 00000000000000..96c1691e24a9c8 --- /dev/null +++ b/Lib/test/test_compiler_assemble.py @@ -0,0 +1,71 @@ + +import ast +import types + +from test.support.bytecode_helper import AssemblerTestCase + + +# Tests for the code-object creation stage of the compiler. + +class IsolatedAssembleTests(AssemblerTestCase): + + def complete_metadata(self, metadata, filename="myfile.py"): + if metadata is None: + metadata = {} + for key in ['name', 'qualname']: + metadata.setdefault(key, key) + for key in ['consts']: + metadata.setdefault(key, []) + for key in ['names', 'varnames', 'cellvars', 'freevars']: + metadata.setdefault(key, {}) + for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']: + metadata.setdefault(key, 0) + metadata.setdefault('firstlineno', 1) + metadata.setdefault('filename', filename) + return metadata + + def assemble_test(self, insts, metadata, expected): + metadata = self.complete_metadata(metadata) + insts = self.complete_insts_info(insts) + + co = self.get_code_object(metadata['filename'], insts, metadata) + self.assertIsInstance(co, types.CodeType) + + expected_metadata = {} + for key, value in metadata.items(): + if isinstance(value, list): + expected_metadata[key] = tuple(value) + elif isinstance(value, dict): + expected_metadata[key] = tuple(value.keys()) + else: + expected_metadata[key] = value + + for key, value in expected_metadata.items(): + self.assertEqual(getattr(co, "co_" + key), value) + + f = types.FunctionType(co, {}) + for args, res in expected.items(): + self.assertEqual(f(*args), res) + + def test_simple_expr(self): + metadata = { + 'filename' : 'avg.py', + 'name' : 'avg', + 'qualname' : 'stats.avg', + 'consts' : [2], + 'argcount' : 2, + 'varnames' : {'x' : 0, 'y' : 1}, + } + + # code for "return (x+y)/2" + insts = [ + ('RESUME', 0), + ('LOAD_FAST', 0, 1), # 'x' + ('LOAD_FAST', 1, 1), # 'y' + ('BINARY_OP', 0, 1), # '+' + ('LOAD_CONST', 0, 1), # 2 + ('BINARY_OP', 11, 1), # '/' + ('RETURN_VALUE', 1), + ] + expected = {(3, 4) : 3.5, (-100, 200) : 50, (10, 18) : 14} + self.assemble_test(insts, metadata, expected) diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 7dd81a8855f1be..6669f1c57e2e78 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -3184,6 +3184,74 @@ def test_frozen_pickle(self): self.assertIsNot(obj, p) self.assertEqual(obj, p) + @dataclass(frozen=True, slots=True) + class FrozenSlotsGetStateClass: + foo: str + bar: int + + getstate_called: bool = field(default=False, compare=False) + + def __getstate__(self): + object.__setattr__(self, 'getstate_called', True) + return [self.foo, self.bar] + + @dataclass(frozen=True, slots=True) + class FrozenSlotsSetStateClass: + foo: str + bar: int + + setstate_called: bool = field(default=False, compare=False) + + def __setstate__(self, state): + object.__setattr__(self, 'setstate_called', True) + object.__setattr__(self, 'foo', state[0]) + object.__setattr__(self, 'bar', state[1]) + + @dataclass(frozen=True, slots=True) + class FrozenSlotsAllStateClass: + foo: str + bar: int + + getstate_called: bool = field(default=False, compare=False) + setstate_called: bool = field(default=False, compare=False) + + def __getstate__(self): + object.__setattr__(self, 'getstate_called', True) + return [self.foo, self.bar] + + def __setstate__(self, state): + object.__setattr__(self, 'setstate_called', True) + object.__setattr__(self, 'foo', state[0]) + object.__setattr__(self, 'bar', state[1]) + + def test_frozen_slots_pickle_custom_state(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + obj = self.FrozenSlotsGetStateClass('a', 1) + dumped = pickle.dumps(obj, protocol=proto) + + self.assertTrue(obj.getstate_called) + self.assertEqual(obj, pickle.loads(dumped)) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + obj = self.FrozenSlotsSetStateClass('a', 1) + obj2 = pickle.loads(pickle.dumps(obj, protocol=proto)) + + self.assertTrue(obj2.setstate_called) + self.assertEqual(obj, obj2) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + obj = self.FrozenSlotsAllStateClass('a', 1) + dumped = pickle.dumps(obj, protocol=proto) + + self.assertTrue(obj.getstate_called) + + obj2 = pickle.loads(dumped) + self.assertTrue(obj2.setstate_called) + self.assertEqual(obj, obj2) + def test_slots_with_default_no_init(self): # Originally reported in bpo-44649. @dataclass(slots=True) @@ -3671,7 +3739,7 @@ def test_text_annotations(self): ByMakeDataClass = make_dataclass('ByMakeDataClass', [('x', int)]) ManualModuleMakeDataClass = make_dataclass('ManualModuleMakeDataClass', [('x', int)], - module='test.test_dataclasses') + module=__name__) WrongNameMakeDataclass = make_dataclass('Wrong', [('x', int)]) WrongModuleMakeDataclass = make_dataclass('WrongModuleMakeDataclass', [('x', int)], diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index f17bb1813b9d87..ad3eefba365856 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5004,7 +5004,7 @@ class Child(Parent): self.assertEqual(Parent.__subclasses__(), []) def test_attr_raise_through_property(self): - # add test case for gh-103272 + # test case for gh-103272 class A: def __getattr__(self, name): raise ValueError("FOO") @@ -5016,6 +5016,19 @@ def foo(self): with self.assertRaisesRegex(ValueError, "FOO"): A().foo + # test case for gh-103551 + class B: + @property + def __getattr__(self, name): + raise ValueError("FOO") + + @property + def foo(self): + raise NotImplementedError("BAR") + + with self.assertRaisesRegex(NotImplementedError, "BAR"): + B().foo + class DictProxyTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 69b0d5f1c5a515..6fa49dbc0b730c 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -619,6 +619,8 @@ def test_specifier_z_error(self): error_msg = re.escape("unsupported format character 'z'") with self.assertRaisesRegex(ValueError, error_msg): "%z.1f" % 0 # not allowed in old style string interpolation + with self.assertRaisesRegex(ValueError, error_msg): + b"%z.1f" % 0 if __name__ == "__main__": diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 5e94c99ae65af1..be71fde5aaba54 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -561,11 +561,28 @@ def test_mismatched_parens(self): ]) self.assertRaises(SyntaxError, eval, "f'{" + "("*500 + "}'") + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_fstring_nested_too_deeply(self): self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", ['f"{1+2:{1+2:{1+1:{1}}}}"']) + def create_nested_fstring(n): + if n == 0: + return "1+1" + prev = create_nested_fstring(n-1) + return f'f"{{{prev}}}"' + + self.assertAllRaise(SyntaxError, + "too many nested f-strings", + [create_nested_fstring(160)]) + + def test_syntax_error_in_nested_fstring(self): + # See gh-104016 for more information on this crash + self.assertAllRaise(SyntaxError, + "invalid syntax", + ['f"{1 1:' + ('{f"1:' * 199)]) + def test_double_braces(self): self.assertEqual(f'{{', '{') self.assertEqual(f'a{{', 'a{') diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index b4f4e2b14351a6..37f77fe0a320c7 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2390,6 +2390,29 @@ def test_tunnel_debuglog(self): lines = output.getvalue().splitlines() self.assertIn('header: {}'.format(expected_header), lines) + def test_tunnel_leak(self): + sock = None + + def _create_connection(address, timeout=None, source_address=None): + nonlocal sock + sock = FakeSocket( + 'HTTP/1.1 404 NOT FOUND\r\n\r\n', + host=address[0], + port=address[1], + ) + return sock + + self.conn._create_connection = _create_connection + self.conn.set_tunnel('destination.com') + exc = None + try: + self.conn.request('HEAD', '/', '') + except OSError as e: + # keeping a reference to exc keeps response alive in the traceback + exc = e + self.assertIsNotNone(exc) + self.assertTrue(sock.file_closed) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 334fea0774be51..5545ee39d8e942 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -155,6 +155,8 @@ def test_basic(self): self.assertEqual(int(' 0O123 ', 0), 83) self.assertEqual(int(' 0X123 ', 0), 291) self.assertEqual(int(' 0B100 ', 0), 4) + with self.assertRaises(ValueError): + int('010', 0) # without base still base 10 self.assertEqual(int('0123'), 123) @@ -221,6 +223,24 @@ def test_basic(self): self.assertEqual(int('2br45qc', 35), 4294967297) self.assertEqual(int('1z141z5', 36), 4294967297) + def test_invalid_signs(self): + with self.assertRaises(ValueError): + int('+') + with self.assertRaises(ValueError): + int('-') + with self.assertRaises(ValueError): + int('- 1') + with self.assertRaises(ValueError): + int('+ 1') + with self.assertRaises(ValueError): + int(' + 1 ') + + def test_unicode(self): + self.assertEqual(int("१२३४५६७८९०1234567890"), 12345678901234567890) + self.assertEqual(int('١٢٣٤٥٦٧٨٩٠'), 1234567890) + self.assertEqual(int("१२३४५६७८९०1234567890", 0), 12345678901234567890) + self.assertEqual(int('١٢٣٤٥٦٧٨٩٠', 0), 1234567890) + def test_underscores(self): for lit in VALID_UNDERSCORE_LITERALS: if any(ch in lit for ch in '.eEjJ'): diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 70e4efea69359a..c7eb92290e1b6d 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -236,7 +236,7 @@ def test_module_repr_with_full_loader(self): # Yes, a class not an instance. m.__loader__ = FullLoader self.assertEqual( - repr(m), "<module 'foo' (<class 'test.test_module.FullLoader'>)>") + repr(m), f"<module 'foo' (<class '{__name__}.FullLoader'>)>") def test_module_repr_with_bare_loader_and_filename(self): m = ModuleType('foo') diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index b57272618f7f6d..c1b18a4980e1a4 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -81,9 +81,9 @@ def test_bytes(self): r"where __fspath__ returns a str, not 'bytes'") with self.assertRaisesRegex(TypeError, message): P(b'a') - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, message): P(b'a', 'b') - with self.assertRaises(TypeError): + with self.assertRaisesRegex(TypeError, message): P('a', b'b') with self.assertRaises(TypeError): P('a').joinpath(b'b') @@ -1904,8 +1904,13 @@ def test_glob_dotdot(self): P = self.cls p = P(BASE) self.assertEqual(set(p.glob("..")), { P(BASE, "..") }) + self.assertEqual(set(p.glob("../..")), { P(BASE, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(BASE, "dirA", "..") }) self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) + self.assertEqual(set(p.glob("dirA/../file*/..")), set()) self.assertEqual(set(p.glob("../xyzzy")), set()) + self.assertEqual(set(p.glob("xyzzy/..")), set()) + self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(BASE, *[".."] * 50)}) @os_helper.skip_unless_symlink def test_glob_permissions(self): diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index ed773a3cff2a6d..698ab48f48eaa1 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -359,7 +359,7 @@ class C: def method(self): return super().msg - with patch("test.test_super.super", MySuper) as m: + with patch(f"{__name__}.super", MySuper) as m: self.assertEqual(C().method(), "super super") def test_shadowed_dynamic_two_arg(self): @@ -373,7 +373,7 @@ class C: def method(self): return super(1, 2).msg - with patch("test.test_super.super", MySuper) as m: + with patch(f"{__name__}.super", MySuper) as m: self.assertEqual(C().method(), "super super") self.assertEqual(call_args, [(1, 2)]) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 283a7c23609e67..911b53e5816588 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -11,7 +11,7 @@ from test.test_grammar import (VALID_UNDERSCORE_LITERALS, INVALID_UNDERSCORE_LITERALS) from test.support import os_helper -from test.support.script_helper import run_test_script, make_script +from test.support.script_helper import run_test_script, make_script, run_python_until_end import os import token @@ -1470,6 +1470,19 @@ def test_comment_at_the_end_of_the_source_without_newline(self): self.assertEqual(tok_name[tokens[i + 1].exact_type], tok_name[expected_tokens[i]]) self.assertEqual(tok_name[tokens[-1].exact_type], tok_name[token.ENDMARKER]) + def test_invalid_character_in_fstring_middle(self): + # See gh-103824 + script = b'''F""" + \xe5"""''' + + with os_helper.temp_dir() as temp_dir: + filename = os.path.join(temp_dir, "script.py") + with open(filename, 'wb') as file: + file.write(script) + rs, _ = run_python_until_end(filename) + self.assertIn(b"SyntaxError", rs.err) + + class UntokenizeTest(TestCase): def test_bad_input_order(self): diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 5e2b353782994e..19a2be88d2c1bc 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1539,11 +1539,11 @@ def __repr__(self): e.__notes__ = BadThing() notes_repr = 'bad repr' - self.assertEqual(self.get_report(e), vanilla + notes_repr) + self.assertEqual(self.get_report(e), vanilla + notes_repr + '\n') e.__notes__ = Unprintable() err_msg = '<__notes__ repr() failed>' - self.assertEqual(self.get_report(e), vanilla + err_msg) + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') # non-string item in the __notes__ sequence e.__notes__ = [BadThing(), 'Final Note'] @@ -1555,6 +1555,14 @@ def __repr__(self): err_msg = '<note str() failed>' self.assertEqual(self.get_report(e), vanilla + err_msg + '\nFinal Note\n') + e.__notes__ = "please do not explode me" + err_msg = "'please do not explode me'" + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + + e.__notes__ = b"please do not show me as numbers" + err_msg = "b'please do not show me as numbers'" + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + def test_exception_with_note_with_multiple_notes(self): e = ValueError(42) vanilla = self.get_report(e) diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index 95af84e3779824..3f9f129a3dd200 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -267,6 +267,14 @@ def test_goto(self): self.assertAlmostEqual(self.nav.xcor(), 100) self.assertAlmostEqual(self.nav.ycor(), -100) + def test_teleport(self): + self.nav.teleport(20, -30, fill_gap=True) + self.assertAlmostEqual(self.nav.xcor(), 20) + self.assertAlmostEqual(self.nav.ycor(), -30) + self.nav.teleport(-20, 30, fill_gap=False) + self.assertAlmostEqual(self.nav.xcor(), -20) + self.assertAlmostEqual(self.nav.ycor(), 30) + def test_pos(self): self.assertEqual(self.nav.pos(), self.nav._position) self.nav.goto(100, -100) @@ -440,6 +448,18 @@ def test_showturtle_hideturtle_and_isvisible(self): tpen.showturtle() self.assertTrue(tpen.isvisible()) + def test_teleport(self): + + tpen = turtle.TPen() + + for fill_gap_value in [True, False]: + tpen.penup() + tpen.teleport(100, 100, fill_gap=fill_gap_value) + self.assertFalse(tpen.isdown()) + tpen.pendown() + tpen.teleport(-100, -100, fill_gap=fill_gap_value) + self.assertTrue(tpen.isdown()) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f36bb958c88ef9..f162e587810ac0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -117,7 +117,7 @@ def test_repr(self): class Sub(Any): pass self.assertEqual( repr(Sub), - "<class 'test.test_typing.AnyTests.test_repr.<locals>.Sub'>", + f"<class '{__name__}.AnyTests.test_repr.<locals>.Sub'>", ) def test_errors(self): @@ -880,6 +880,11 @@ def test_cannot_be_called(self): with self.assertRaises(TypeError): Unpack() + def test_usage_with_kwargs(self): + Movie = TypedDict('Movie', {'name': str, 'year': int}) + def foo(**kwargs: Unpack[Movie]): ... + self.assertEqual(repr(foo.__annotations__['kwargs']), + f"typing.Unpack[{__name__}.Movie]") class TypeVarTupleTests(BaseTestCase): @@ -1050,14 +1055,14 @@ class G2(Generic[Unpack[Ts]]): pass self.assertEqual(repr(Ts), 'Ts') - self.assertEqual(repr((*Ts,)[0]), '*Ts') - self.assertEqual(repr(Unpack[Ts]), '*Ts') + self.assertEqual(repr((*Ts,)[0]), 'typing.Unpack[Ts]') + self.assertEqual(repr(Unpack[Ts]), 'typing.Unpack[Ts]') - self.assertEqual(repr(tuple[*Ts]), 'tuple[*Ts]') - self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[*Ts]') + self.assertEqual(repr(tuple[*Ts]), 'tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(Tuple[Unpack[Ts]]), 'typing.Tuple[typing.Unpack[Ts]]') - self.assertEqual(repr(*tuple[*Ts]), '*tuple[*Ts]') - self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), '*typing.Tuple[*Ts]') + self.assertEqual(repr(*tuple[*Ts]), '*tuple[typing.Unpack[Ts]]') + self.assertEqual(repr(Unpack[Tuple[Unpack[Ts]]]), 'typing.Unpack[typing.Tuple[typing.Unpack[Ts]]]') def test_variadic_class_repr_is_correct(self): Ts = TypeVarTuple('Ts') @@ -1074,86 +1079,86 @@ class B(Generic[Unpack[Ts]]): pass self.assertEndsWith(repr(A[*tuple[int, ...]]), 'A[*tuple[int, ...]]') self.assertEndsWith(repr(B[Unpack[Tuple[int, ...]]]), - 'B[*typing.Tuple[int, ...]]') + 'B[typing.Unpack[typing.Tuple[int, ...]]]') self.assertEndsWith(repr(A[float, *tuple[int, ...]]), 'A[float, *tuple[int, ...]]') self.assertEndsWith(repr(A[float, Unpack[Tuple[int, ...]]]), - 'A[float, *typing.Tuple[int, ...]]') + 'A[float, typing.Unpack[typing.Tuple[int, ...]]]') self.assertEndsWith(repr(A[*tuple[int, ...], str]), 'A[*tuple[int, ...], str]') self.assertEndsWith(repr(B[Unpack[Tuple[int, ...]], str]), - 'B[*typing.Tuple[int, ...], str]') + 'B[typing.Unpack[typing.Tuple[int, ...]], str]') self.assertEndsWith(repr(A[float, *tuple[int, ...], str]), 'A[float, *tuple[int, ...], str]') self.assertEndsWith(repr(B[float, Unpack[Tuple[int, ...]], str]), - 'B[float, *typing.Tuple[int, ...], str]') + 'B[float, typing.Unpack[typing.Tuple[int, ...]], str]') def test_variadic_class_alias_repr_is_correct(self): Ts = TypeVarTuple('Ts') class A(Generic[Unpack[Ts]]): pass B = A[*Ts] - self.assertEndsWith(repr(B), 'A[*Ts]') + self.assertEndsWith(repr(B), 'A[typing.Unpack[Ts]]') self.assertEndsWith(repr(B[()]), 'A[()]') self.assertEndsWith(repr(B[float]), 'A[float]') self.assertEndsWith(repr(B[float, str]), 'A[float, str]') C = A[Unpack[Ts]] - self.assertEndsWith(repr(C), 'A[*Ts]') + self.assertEndsWith(repr(C), 'A[typing.Unpack[Ts]]') self.assertEndsWith(repr(C[()]), 'A[()]') self.assertEndsWith(repr(C[float]), 'A[float]') self.assertEndsWith(repr(C[float, str]), 'A[float, str]') D = A[*Ts, int] - self.assertEndsWith(repr(D), 'A[*Ts, int]') + self.assertEndsWith(repr(D), 'A[typing.Unpack[Ts], int]') self.assertEndsWith(repr(D[()]), 'A[int]') self.assertEndsWith(repr(D[float]), 'A[float, int]') self.assertEndsWith(repr(D[float, str]), 'A[float, str, int]') E = A[Unpack[Ts], int] - self.assertEndsWith(repr(E), 'A[*Ts, int]') + self.assertEndsWith(repr(E), 'A[typing.Unpack[Ts], int]') self.assertEndsWith(repr(E[()]), 'A[int]') self.assertEndsWith(repr(E[float]), 'A[float, int]') self.assertEndsWith(repr(E[float, str]), 'A[float, str, int]') F = A[int, *Ts] - self.assertEndsWith(repr(F), 'A[int, *Ts]') + self.assertEndsWith(repr(F), 'A[int, typing.Unpack[Ts]]') self.assertEndsWith(repr(F[()]), 'A[int]') self.assertEndsWith(repr(F[float]), 'A[int, float]') self.assertEndsWith(repr(F[float, str]), 'A[int, float, str]') G = A[int, Unpack[Ts]] - self.assertEndsWith(repr(G), 'A[int, *Ts]') + self.assertEndsWith(repr(G), 'A[int, typing.Unpack[Ts]]') self.assertEndsWith(repr(G[()]), 'A[int]') self.assertEndsWith(repr(G[float]), 'A[int, float]') self.assertEndsWith(repr(G[float, str]), 'A[int, float, str]') H = A[int, *Ts, str] - self.assertEndsWith(repr(H), 'A[int, *Ts, str]') + self.assertEndsWith(repr(H), 'A[int, typing.Unpack[Ts], str]') self.assertEndsWith(repr(H[()]), 'A[int, str]') self.assertEndsWith(repr(H[float]), 'A[int, float, str]') self.assertEndsWith(repr(H[float, str]), 'A[int, float, str, str]') I = A[int, Unpack[Ts], str] - self.assertEndsWith(repr(I), 'A[int, *Ts, str]') + self.assertEndsWith(repr(I), 'A[int, typing.Unpack[Ts], str]') self.assertEndsWith(repr(I[()]), 'A[int, str]') self.assertEndsWith(repr(I[float]), 'A[int, float, str]') self.assertEndsWith(repr(I[float, str]), 'A[int, float, str, str]') J = A[*Ts, *tuple[str, ...]] - self.assertEndsWith(repr(J), 'A[*Ts, *tuple[str, ...]]') + self.assertEndsWith(repr(J), 'A[typing.Unpack[Ts], *tuple[str, ...]]') self.assertEndsWith(repr(J[()]), 'A[*tuple[str, ...]]') self.assertEndsWith(repr(J[float]), 'A[float, *tuple[str, ...]]') self.assertEndsWith(repr(J[float, str]), 'A[float, str, *tuple[str, ...]]') K = A[Unpack[Ts], Unpack[Tuple[str, ...]]] - self.assertEndsWith(repr(K), 'A[*Ts, *typing.Tuple[str, ...]]') - self.assertEndsWith(repr(K[()]), 'A[*typing.Tuple[str, ...]]') - self.assertEndsWith(repr(K[float]), 'A[float, *typing.Tuple[str, ...]]') - self.assertEndsWith(repr(K[float, str]), 'A[float, str, *typing.Tuple[str, ...]]') + self.assertEndsWith(repr(K), 'A[typing.Unpack[Ts], typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[()]), 'A[typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[float]), 'A[float, typing.Unpack[typing.Tuple[str, ...]]]') + self.assertEndsWith(repr(K[float, str]), 'A[float, str, typing.Unpack[typing.Tuple[str, ...]]]') def test_cannot_subclass(self): with self.assertRaisesRegex(TypeError, CANNOT_SUBCLASS_TYPE): @@ -1171,9 +1176,9 @@ class C(type(Unpack[Ts])): pass with self.assertRaisesRegex(TypeError, r'Cannot subclass typing\.Unpack'): class C(Unpack): pass - with self.assertRaisesRegex(TypeError, r'Cannot subclass \*Ts'): + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): class C(*Ts): pass - with self.assertRaisesRegex(TypeError, r'Cannot subclass \*Ts'): + with self.assertRaisesRegex(TypeError, r'Cannot subclass typing.Unpack\[Ts\]'): class C(Unpack[Ts]): pass def test_variadic_class_args_are_correct(self): @@ -4108,13 +4113,13 @@ class TsP(Generic[*Ts, P]): MyCallable[[int], bool]: "MyCallable[[int], bool]", MyCallable[[int, str], bool]: "MyCallable[[int, str], bool]", MyCallable[[int, list[int]], bool]: "MyCallable[[int, list[int]], bool]", - MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[*Ts, ~P], ~T]", + MyCallable[Concatenate[*Ts, P], T]: "MyCallable[typing.Concatenate[typing.Unpack[Ts], ~P], ~T]", DoubleSpec[P2, P, T]: "DoubleSpec[~P2, ~P, ~T]", DoubleSpec[[int], [str], bool]: "DoubleSpec[[int], [str], bool]", DoubleSpec[[int, int], [str, str], bool]: "DoubleSpec[[int, int], [str, str], bool]", - TsP[*Ts, P]: "TsP[*Ts, ~P]", + TsP[*Ts, P]: "TsP[typing.Unpack[Ts], ~P]", TsP[int, str, list[int], []]: "TsP[int, str, list[int], []]", TsP[int, [str, list[int]]]: "TsP[int, [str, list[int]]]", diff --git a/Lib/traceback.py b/Lib/traceback.py index 9e720ac9948fce..419f6e81b5e1be 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -179,12 +179,12 @@ def _safe_string(value, what, func=str): # -- def print_exc(limit=None, file=None, chain=True): - """Shorthand for 'print_exception(*sys.exc_info(), limit, file, chain)'.""" - print_exception(*sys.exc_info(), limit=limit, file=file, chain=chain) + """Shorthand for 'print_exception(sys.exception(), limit, file, chain)'.""" + print_exception(sys.exception(), limit=limit, file=file, chain=chain) def format_exc(limit=None, chain=True): """Like print_exc() but return a string.""" - return "".join(format_exception(*sys.exc_info(), limit=limit, chain=chain)) + return "".join(format_exception(sys.exception(), limit=limit, chain=chain)) def print_last(limit=None, file=None, chain=True): """This is a shorthand for 'print_exception(sys.last_exc, limit, file, chain)'.""" @@ -852,12 +852,16 @@ def format_exception_only(self): yield _format_final_exc_line(stype, self._str) else: yield from self._format_syntax_error(stype) - if isinstance(self.__notes__, collections.abc.Sequence): + + if ( + isinstance(self.__notes__, collections.abc.Sequence) + and not isinstance(self.__notes__, (str, bytes)) + ): for note in self.__notes__: note = _safe_string(note, 'note') yield from [l + '\n' for l in note.split('\n')] elif self.__notes__ is not None: - yield _safe_string(self.__notes__, '__notes__', func=repr) + yield "{}\n".format(_safe_string(self.__notes__, '__notes__', func=repr)) def _format_syntax_error(self, stype): """Format SyntaxError exceptions (internal helper).""" diff --git a/Lib/turtle.py b/Lib/turtle.py index 1b369327bc8eff..2de406e0f517af 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -135,7 +135,7 @@ 'pu', 'radians', 'right', 'reset', 'resizemode', 'rt', 'seth', 'setheading', 'setpos', 'setposition', 'settiltangle', 'setundobuffer', 'setx', 'sety', 'shape', 'shapesize', 'shapetransform', 'shearfactor', 'showturtle', - 'speed', 'st', 'stamp', 'tilt', 'tiltangle', 'towards', + 'speed', 'st', 'stamp', 'teleport', 'tilt', 'tiltangle', 'towards', 'turtlesize', 'undo', 'undobufferentries', 'up', 'width', 'write', 'xcor', 'ycor'] _tg_utilities = ['write_docstringdict', 'done'] @@ -1614,6 +1614,13 @@ def _goto(self, end): """move turtle to position end.""" self._position = end + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: + """To be overwritten by child class RawTurtle. + Includes no TPen references.""" + new_x = x if x is not None else self._position[0] + new_y = y if y is not None else self._position[1] + self._position = Vec2D(new_x, new_y) + def forward(self, distance): """Move the turtle forward by the specified distance. @@ -2293,6 +2300,15 @@ def fillcolor(self, *args): else: return self._color(self._fillcolor) + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: + """To be overwritten by child class RawTurtle. + Includes no TNavigator references. + """ + pendown = self.isdown() + if pendown: + self.pen(pendown=False) + self.pen(pendown=pendown) + def showturtle(self): """Makes the turtle visible. @@ -2710,6 +2726,54 @@ def _cc(self, args): if not ((0 <= r <= 255) and (0 <= g <= 255) and (0 <= b <= 255)): raise TurtleGraphicsError("bad color sequence: %s" % str(args)) return "#%02x%02x%02x" % (r, g, b) + + def teleport(self, x=None, y=None, *, fill_gap: bool = False) -> None: + """Instantly move turtle to an absolute position. + + Arguments: + x -- a number or None + y -- a number None + fill_gap -- a boolean This argument must be specified by name. + + call: teleport(x, y) # two coordinates + --or: teleport(x) # teleport to x position, keeping y as is + --or: teleport(y=y) # teleport to y position, keeping x as is + --or: teleport(x, y, fill_gap=True) + # teleport but fill the gap in between + + Move turtle to an absolute position. Unlike goto(x, y), a line will not + be drawn. The turtle's orientation does not change. If currently + filling, the polygon(s) teleported from will be filled after leaving, + and filling will begin again after teleporting. This can be disabled + with fill_gap=True, which makes the imaginary line traveled during + teleporting act as a fill barrier like in goto(x, y). + + Example (for a Turtle instance named turtle): + >>> tp = turtle.pos() + >>> tp + (0.00,0.00) + >>> turtle.teleport(60) + >>> turtle.pos() + (60.00,0.00) + >>> turtle.teleport(y=10) + >>> turtle.pos() + (60.00,10.00) + >>> turtle.teleport(20, 30) + >>> turtle.pos() + (20.00,30.00) + """ + pendown = self.isdown() + was_filling = self.filling() + if pendown: + self.pen(pendown=False) + if was_filling and not fill_gap: + self.end_fill() + new_x = x if x is not None else self._position[0] + new_y = y if y is not None else self._position[1] + self._position = Vec2D(new_x, new_y) + self.pen(pendown=pendown) + if was_filling and not fill_gap: + self.begin_fill() def clone(self): """Create and return a clone of the turtle. diff --git a/Lib/typing.py b/Lib/typing.py index 354bc80eb3abfa..0dacdd9031a776 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1753,6 +1753,17 @@ class Bar(Generic[Unpack[Ts]]): ... Foo[*tuple[int, str]] class Bar(Generic[*Ts]): ... + The operator can also be used along with a `TypedDict` to annotate + `**kwargs` in a function signature. For instance: + + class Movie(TypedDict): + name: str + year: int + + # This function expects two keyword arguments - *name* of type `str` and + # *year* of type `int`. + def foo(**kwargs: Unpack[Movie]): ... + Note that there is only some runtime checking of this operator. Not everything the runtime allows may be accepted by static type checkers. @@ -1767,7 +1778,7 @@ class _UnpackGenericAlias(_GenericAlias, _root=True): def __repr__(self): # `Unpack` only takes one argument, so __args__ should contain only # a single item. - return '*' + repr(self.__args__[0]) + return f'typing.Unpack[{_type_repr(self.__args__[0])}]' def __getitem__(self, args): if self.__typing_is_unpacked_typevartuple__: @@ -2308,15 +2319,16 @@ def cast(typ, val): def assert_type(val, typ, /): """Ask a static type checker to confirm that the value is of the given type. - When the type checker encounters a call to assert_type(), it + At runtime this does nothing: it returns the first argument unchanged with no + checks or side effects, no matter the actual type of the argument. + + When a static type checker encounters a call to assert_type(), it emits an error if the value is not of the specified type:: def greet(name: str) -> None: assert_type(name, str) # ok assert_type(name, int) # type checker error - At runtime this returns the first argument unchanged and otherwise - does nothing. """ return val diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 63fa21b2b33d17..2f5937489ac03d 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.40.1", - url="https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz", - checksum="42175b1a1d23529cb133bbd2b5900afd", + name="SQLite 3.41.2", + url="https://sqlite.org/2023/sqlite-autoconf-3410200.tar.gz", + checksum="862075fd1c38324878ef809eda39edfe", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' diff --git a/Misc/ACKS b/Misc/ACKS index 65be5cfc3c7945..42ec059a7c4ec2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1488,7 +1488,7 @@ John Redford Kalyan Reddy Terry J. Reedy Gareth Rees -John Reese +Amethyst Reese Steve Reeves Lennart Regebro John Regehr diff --git a/Misc/NEWS.d/3.7.0a1.rst b/Misc/NEWS.d/3.7.0a1.rst index 9bada1b76be7a8..ef93454784b77f 100644 --- a/Misc/NEWS.d/3.7.0a1.rst +++ b/Misc/NEWS.d/3.7.0a1.rst @@ -6255,7 +6255,7 @@ Fix python-gdb.py didn't support new dict implementation. .. section: Tools/Demos The pybench and pystone microbenchmark have been removed from Tools. Please -use the new Python benchmark suite https://github.com/python/performance +use the new Python benchmark suite https://github.com/python/pyperformance which is more reliable and includes a portable version of pybench working on Python 2 and Python 3. diff --git a/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst b/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst new file mode 100644 index 00000000000000..d074350ed3ebbe --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-04-24-10-31-59.gh-issue-103743.2xYA1K.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnstable_Object_GC_NewWithExtraData` function that can be used to +allocate additional memory after an object for data not managed by Python. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst new file mode 100644 index 00000000000000..f9e71bc1344bb3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-10-06-23-32-11.gh-issue-98003.xWE0Yu.rst @@ -0,0 +1,3 @@ +Complex function calls are now faster and consume no C stack +space. + diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst new file mode 100644 index 00000000000000..6fed304c9132b3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-17-56-18.gh-issue-103895.ESB6tn.rst @@ -0,0 +1,3 @@ +Improve handling of edge cases in showing ``Exception.__notes__``. Ensures +that the messages always end with a newline and that string/bytes are not +exploded over multiple lines. Patch by Carey Metcalfe. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst new file mode 100644 index 00000000000000..997bef226e713f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-08-08-05.gh-issue-102213.nfH-4C.rst @@ -0,0 +1 @@ +Fix performance loss when accessing an object's attributes with ``__getattr__`` defined. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst new file mode 100644 index 00000000000000..f3cadaee0e32d9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-12-03-52.gh-issue-104018.PFxGS4.rst @@ -0,0 +1 @@ +Disallow the "z" format specifier in %-format of bytes objects. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst new file mode 100644 index 00000000000000..9c35ea88499dce --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-10-38.gh-issue-104028.dxfh13.rst @@ -0,0 +1,2 @@ +Reduce object creation while calling callback function from gc. +Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst b/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst new file mode 100644 index 00000000000000..4f390d189d23b5 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2023-04-30-20-01-18.gh-issue-88496.y65vUb.rst @@ -0,0 +1 @@ +Fix IDLE test hang on macOS. diff --git a/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst b/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst new file mode 100644 index 00000000000000..0b3854d74eb991 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-15-12-18-07.gh-issue-97696.DtnpIC.rst @@ -0,0 +1,6 @@ +Implemented an eager task factory in asyncio. +When used as a task factory on an event loop, it performs eager execution of +coroutines. Coroutines that are able to complete synchronously (e.g. return or +raise without blocking) are returned immediately as a finished task, and the +task is never scheduled to the event loop. If the coroutine blocks, the +(pending) task is scheduled and returned. diff --git a/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst b/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst new file mode 100644 index 00000000000000..01d84f024bd4a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-12-13-04-16.gh-issue-103472.C6bOHv.rst @@ -0,0 +1,2 @@ +Avoid a potential :exc:`ResourceWarning` in :class:`http.client.HTTPConnection` +by closing the proxy / tunnel's CONNECT response explicitly. diff --git a/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst b/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst new file mode 100644 index 00000000000000..1414cb07dd9155 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-13-19-43-15.gh-issue-103525.uY4VYg.rst @@ -0,0 +1,2 @@ +Fix misleading exception message when mixed ``str`` and ``bytes`` arguments +are supplied to :class:`pathlib.PurePath` and :class:`~pathlib.Path`. diff --git a/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst b/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst new file mode 100644 index 00000000000000..f14c9533f3af87 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-28-18-04-23.gh-issue-88773.xXCNJw.rst @@ -0,0 +1 @@ +Added :func:`turtle.teleport` to the :mod:`turtle` module to move a turtle to a new point without tracing a line, visible or invisible. Patch by Liam Gersten. diff --git a/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst b/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst new file mode 100644 index 00000000000000..ff4005774a95d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-28-19-08-50.gh-issue-103977.msF70A.rst @@ -0,0 +1 @@ +Improve import time of :mod:`platform` module. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst b/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst new file mode 100644 index 00000000000000..8c8e3d6ba5fbc1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-01-16-43-28.gh-issue-104035.MrJBw8.rst @@ -0,0 +1,2 @@ +Do not ignore user-defined ``__getstate__`` and ``__setstate__`` methods for +slotted frozen dataclasses. diff --git a/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst b/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst new file mode 100644 index 00000000000000..7971ab66359c3d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-01-19-10-05.gh-issue-103629.81bpZz.rst @@ -0,0 +1 @@ +Update the ``repr`` of :class:`typing.Unpack` according to :pep:`692`. diff --git a/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst b/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst new file mode 100644 index 00000000000000..3daf9cc093807b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst @@ -0,0 +1 @@ +Update the return type of ``weekday`` to the newly added Day attribute diff --git a/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst b/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst new file mode 100644 index 00000000000000..7101de908a5004 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst @@ -0,0 +1,2 @@ +Improve performance of :meth:`pathlib.Path.glob` when evaluating patterns +that contain ``'../'`` segments. diff --git a/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst b/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst new file mode 100644 index 00000000000000..935a0e2a2bff18 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst @@ -0,0 +1,2 @@ +Improve performance of :meth:`pathlib.Path.glob` by using +:data:`re.IGNORECASE` to implement case-insensitive matching. diff --git a/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst b/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst new file mode 100644 index 00000000000000..c8f7259aecba6f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-03-24-11-25-28.gh-issue-102997.dredy2.rst @@ -0,0 +1 @@ +Update Windows installer to use SQLite 3.41.2. diff --git a/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst b/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst new file mode 100644 index 00000000000000..d0b390a896b743 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-03-24-11-20-47.gh-issue-102997.ZgQkbq.rst @@ -0,0 +1 @@ +Update macOS installer to SQLite 3.41.2. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index fe1b9f8f5380c1..a90c1e96ef0231 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c _testcapi/immortal.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_abc.c b/Modules/_abc.c index 9d6654b4e58aad..997b618d557ab2 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -452,7 +452,8 @@ _abc__abc_init(PyObject *module, PyObject *self) * their special status w.r.t. pattern matching. */ if (PyType_Check(self)) { PyTypeObject *cls = (PyTypeObject *)self; - PyObject *flags = PyDict_GetItemWithError(cls->tp_dict, + PyObject *dict = _PyType_GetDict(cls); + PyObject *flags = PyDict_GetItemWithError(dict, &_Py_ID(__abc_tpflags__)); if (flags == NULL) { if (PyErr_Occurred()) { @@ -471,7 +472,7 @@ _abc__abc_init(PyObject *module, PyObject *self) } ((PyTypeObject *)self)->tp_flags |= (val & COLLECTION_FLAGS); } - if (PyDict_DelItem(cls->tp_dict, &_Py_ID(__abc_tpflags__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__abc_tpflags__)) < 0) { return NULL; } } @@ -742,7 +743,7 @@ _abc__abc_subclasscheck_impl(PyObject *module, PyObject *self, Py_DECREF(ok); /* 4. Check if it's a direct subclass. */ - PyObject *mro = ((PyTypeObject *)subclass)->tp_mro; + PyObject *mro = _PyType_GetMRO((PyTypeObject *)subclass); assert(PyTuple_Check(mro)); for (pos = 0; pos < PyTuple_GET_SIZE(mro); pos++) { PyObject *mro_item = PyTuple_GET_ITEM(mro, pos); diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 82dbc087322aa9..8b1a29b6d33e8b 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -8,6 +8,7 @@ #include "pycore_runtime_init.h" // _Py_ID() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "structmember.h" // PyMemberDef +#include "cpython/context.h" #include <stddef.h> // offsetof() @@ -31,8 +32,11 @@ typedef struct { all running event loops. {EventLoop: Task} */ PyObject *current_tasks; - /* WeakSet containing all alive tasks. */ - PyObject *all_tasks; + /* WeakSet containing all tasks scheduled to run on event loops. */ + PyObject *scheduled_tasks; + + /* Set containing all eagerly executing tasks. */ + PyObject *eager_tasks; /* An isinstance type cache for the 'is_coroutine()' function. */ PyObject *iscoroutine_typecache; @@ -156,6 +160,9 @@ class _asyncio.Future "FutureObj *" "&Future_Type" /* Get FutureIter from Future */ static PyObject * future_new_iter(PyObject *); +static PyObject * +task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *result); + static int _is_coroutine(asyncio_state *state, PyObject *coro) @@ -1830,6 +1837,7 @@ class _asyncio.Task "TaskObj *" "&Task_Type" static int task_call_step_soon(asyncio_state *state, TaskObj *, PyObject *); static PyObject * task_wakeup(TaskObj *, PyObject *); static PyObject * task_step(asyncio_state *, TaskObj *, PyObject *); +static int task_eager_start(asyncio_state *state, TaskObj *task); /* ----- Task._step wrapper */ @@ -1940,7 +1948,7 @@ static PyMethodDef TaskWakeupDef = { static int register_task(asyncio_state *state, PyObject *task) { - PyObject *res = PyObject_CallMethodOneArg(state->all_tasks, + PyObject *res = PyObject_CallMethodOneArg(state->scheduled_tasks, &_Py_ID(add), task); if (res == NULL) { return -1; @@ -1949,11 +1957,16 @@ register_task(asyncio_state *state, PyObject *task) return 0; } +static int +register_eager_task(asyncio_state *state, PyObject *task) +{ + return PySet_Add(state->eager_tasks, task); +} static int unregister_task(asyncio_state *state, PyObject *task) { - PyObject *res = PyObject_CallMethodOneArg(state->all_tasks, + PyObject *res = PyObject_CallMethodOneArg(state->scheduled_tasks, &_Py_ID(discard), task); if (res == NULL) { return -1; @@ -1962,6 +1975,11 @@ unregister_task(asyncio_state *state, PyObject *task) return 0; } +static int +unregister_eager_task(asyncio_state *state, PyObject *task) +{ + return PySet_Discard(state->eager_tasks, task); +} static int enter_task(asyncio_state *state, PyObject *loop, PyObject *task) @@ -2015,6 +2033,54 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) return _PyDict_DelItem_KnownHash(state->current_tasks, loop, hash); } +static PyObject * +swap_current_task(asyncio_state *state, PyObject *loop, PyObject *task) +{ + PyObject *prev_task; + Py_hash_t hash; + hash = PyObject_Hash(loop); + if (hash == -1) { + return NULL; + } + + prev_task = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); + if (prev_task == NULL) { + if (PyErr_Occurred()) { + return NULL; + } + prev_task = Py_None; + } + + if (task == Py_None) { + if (_PyDict_DelItem_KnownHash(state->current_tasks, loop, hash) == -1) { + return NULL; + } + } else { + if (_PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash) == -1) { + return NULL; + } + } + + Py_INCREF(prev_task); + + return prev_task; +} + +static int +is_loop_running(PyObject *loop) +{ + PyObject *func = PyObject_GetAttr(loop, &_Py_ID(is_running)); + if (func == NULL) { + PyErr_Format(PyExc_TypeError, "Loop missing is_running()"); + return -1; + } + PyObject *res = PyObject_CallNoArgs(func); + int retval = Py_IsTrue(res); + Py_DECREF(func); + Py_DECREF(res); + return !!retval; +} + /* ----- Task */ /*[clinic input] @@ -2025,15 +2091,16 @@ _asyncio.Task.__init__ loop: object = None name: object = None context: object = None + eager_start: bool = False A coroutine wrapped in a Future. [clinic start generated code]*/ static int _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, - PyObject *name, PyObject *context) -/*[clinic end generated code: output=49ac96fe33d0e5c7 input=924522490c8ce825]*/ - + PyObject *name, PyObject *context, + int eager_start) +/*[clinic end generated code: output=7aced2d27836f1a1 input=18e3f113a51b829d]*/ { if (future_init((FutureObj*)self, loop)) { return -1; @@ -2083,6 +2150,19 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, return -1; } + if (eager_start) { + int loop_running = is_loop_running(self->task_loop); + if (loop_running == -1) { + return -1; + } + if (loop_running) { + if (task_eager_start(state, self)) { + return -1; + } + return 0; + } + } + if (task_call_step_soon(state, self, NULL)) { return -1; } @@ -2831,6 +2911,20 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) Py_RETURN_NONE; } + PyObject *ret = task_step_handle_result_impl(state, task, result); + return ret; + +fail: + return NULL; +} + + +static PyObject * +task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *result) +{ + int res; + PyObject *o; + if (result == (PyObject*)task) { /* We have a task that wants to await on itself */ goto self_await; @@ -3062,6 +3156,65 @@ task_step(asyncio_state *state, TaskObj *task, PyObject *exc) } } +static int +task_eager_start(asyncio_state *state, TaskObj *task) +{ + assert(task != NULL); + PyObject *prevtask = swap_current_task(state, task->task_loop, (PyObject *)task); + if (prevtask == NULL) { + return -1; + } + + if (register_eager_task(state, (PyObject *)task) == -1) { + Py_DECREF(prevtask); + return -1; + } + + if (PyContext_Enter(task->task_context) == -1) { + Py_DECREF(prevtask); + return -1; + } + + int retval = 0; + + PyObject *stepres = task_step_impl(state, task, NULL); + if (stepres == NULL) { + PyObject *exc = PyErr_GetRaisedException(); + _PyErr_ChainExceptions1(exc); + retval = -1; + } else { + Py_DECREF(stepres); + } + + PyObject *curtask = swap_current_task(state, task->task_loop, prevtask); + Py_DECREF(prevtask); + if (curtask == NULL) { + retval = -1; + } else { + assert(curtask == (PyObject *)task); + Py_DECREF(curtask); + } + + if (unregister_eager_task(state, (PyObject *)task) == -1) { + retval = -1; + } + + if (PyContext_Exit(task->task_context) == -1) { + retval = -1; + } + + if (task->task_state == STATE_PENDING) { + if (register_task(state, (PyObject *)task) == -1) { + retval = -1; + } + } else { + // This seems to really help performance on pyperformance benchmarks + Py_CLEAR(task->task_coro); + } + + return retval; +} + static PyObject * task_wakeup(TaskObj *task, PyObject *o) { @@ -3225,6 +3378,27 @@ _asyncio__register_task_impl(PyObject *module, PyObject *task) Py_RETURN_NONE; } +/*[clinic input] +_asyncio._register_eager_task + + task: object + +Register a new task in asyncio as executed by loop. + +Returns None. +[clinic start generated code]*/ + +static PyObject * +_asyncio__register_eager_task_impl(PyObject *module, PyObject *task) +/*[clinic end generated code: output=dfe1d45367c73f1a input=237f684683398c51]*/ +{ + asyncio_state *state = get_asyncio_state(module); + if (register_eager_task(state, task) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] _asyncio._unregister_task @@ -3247,6 +3421,27 @@ _asyncio__unregister_task_impl(PyObject *module, PyObject *task) Py_RETURN_NONE; } +/*[clinic input] +_asyncio._unregister_eager_task + + task: object + +Unregister a task. + +Returns None. +[clinic start generated code]*/ + +static PyObject * +_asyncio__unregister_eager_task_impl(PyObject *module, PyObject *task) +/*[clinic end generated code: output=a426922bd07f23d1 input=9d07401ef14ee048]*/ +{ + asyncio_state *state = get_asyncio_state(module); + if (unregister_eager_task(state, task) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + /*[clinic input] _asyncio._enter_task @@ -3298,6 +3493,27 @@ _asyncio__leave_task_impl(PyObject *module, PyObject *loop, PyObject *task) } +/*[clinic input] +_asyncio._swap_current_task + + loop: object + task: object + +Temporarily swap in the supplied task and return the original one (or None). + +This is intended for use during eager coroutine execution. + +[clinic start generated code]*/ + +static PyObject * +_asyncio__swap_current_task_impl(PyObject *module, PyObject *loop, + PyObject *task) +/*[clinic end generated code: output=9f88de958df74c7e input=c9c72208d3d38b6c]*/ +{ + return swap_current_task(get_asyncio_state(module), loop, task); +} + + /*[clinic input] _asyncio.current_task @@ -3379,7 +3595,8 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->asyncio_InvalidStateError); Py_VISIT(state->asyncio_CancelledError); - Py_VISIT(state->all_tasks); + Py_VISIT(state->scheduled_tasks); + Py_VISIT(state->eager_tasks); Py_VISIT(state->current_tasks); Py_VISIT(state->iscoroutine_typecache); @@ -3416,7 +3633,8 @@ module_clear(PyObject *mod) Py_CLEAR(state->asyncio_InvalidStateError); Py_CLEAR(state->asyncio_CancelledError); - Py_CLEAR(state->all_tasks); + Py_CLEAR(state->scheduled_tasks); + Py_CLEAR(state->eager_tasks); Py_CLEAR(state->current_tasks); Py_CLEAR(state->iscoroutine_typecache); @@ -3496,9 +3714,14 @@ module_init(asyncio_state *state) PyObject *weak_set; WITH_MOD("weakref") GET_MOD_ATTR(weak_set, "WeakSet"); - state->all_tasks = PyObject_CallNoArgs(weak_set); + state->scheduled_tasks = PyObject_CallNoArgs(weak_set); Py_CLEAR(weak_set); - if (state->all_tasks == NULL) { + if (state->scheduled_tasks == NULL) { + goto fail; + } + + state->eager_tasks = PySet_New(NULL); + if (state->eager_tasks == NULL) { goto fail; } @@ -3522,9 +3745,12 @@ static PyMethodDef asyncio_methods[] = { _ASYNCIO__GET_RUNNING_LOOP_METHODDEF _ASYNCIO__SET_RUNNING_LOOP_METHODDEF _ASYNCIO__REGISTER_TASK_METHODDEF + _ASYNCIO__REGISTER_EAGER_TASK_METHODDEF _ASYNCIO__UNREGISTER_TASK_METHODDEF + _ASYNCIO__UNREGISTER_EAGER_TASK_METHODDEF _ASYNCIO__ENTER_TASK_METHODDEF _ASYNCIO__LEAVE_TASK_METHODDEF + _ASYNCIO__SWAP_CURRENT_TASK_METHODDEF {NULL, NULL} }; @@ -3561,7 +3787,11 @@ module_exec(PyObject *mod) return -1; } - if (PyModule_AddObjectRef(mod, "_all_tasks", state->all_tasks) < 0) { + if (PyModule_AddObjectRef(mod, "_scheduled_tasks", state->scheduled_tasks) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(mod, "_eager_tasks", state->eager_tasks) < 0) { return -1; } diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 93bc784df5386f..f10cf58216acf7 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1823,7 +1823,7 @@ resize(PyObject *self, PyObject *args) dict = PyObject_stgdict((PyObject *)obj); if (dict == NULL) { PyErr_SetString(PyExc_TypeError, - "excepted ctypes instance"); + "expected ctypes instance"); return NULL; } if (size < dict->size) { diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index a3bfbc9ac5a1b1..8ec3a6081c98d9 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -680,7 +680,7 @@ _PyIO_InitTypes(PyInterpreterState *interp) for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; - if (_PyStaticType_InitBuiltin(type) < 0) { + if (_PyStaticType_InitBuiltin(interp, type) < 0) { return _PyStatus_ERR("Can't initialize builtin type"); } } @@ -691,15 +691,11 @@ _PyIO_InitTypes(PyInterpreterState *interp) void _PyIO_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - // Deallocate types in the reverse order to deallocate subclasses before // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types) - 1; i >= 0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(type); + _PyStaticType_Dealloc(interp, type); } } diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c new file mode 100644 index 00000000000000..10e1733d08a9ea --- /dev/null +++ b/Modules/_testcapi/immortal.c @@ -0,0 +1,47 @@ +#include "parts.h" + +int verify_immortality(PyObject *object) +{ + assert(_Py_IsImmortal(object)); + Py_ssize_t old_count = Py_REFCNT(object); + for (int j = 0; j < 10000; j++) { + Py_DECREF(object); + } + Py_ssize_t current_count = Py_REFCNT(object); + return old_count == current_count; +} + +static PyObject * +test_immortal_builtins(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *objects[] = {Py_True, Py_False, Py_None, Py_Ellipsis}; + Py_ssize_t n = Py_ARRAY_LENGTH(objects); + for (Py_ssize_t i = 0; i < n; i++) { + assert(verify_immortality(objects[i])); + } + Py_RETURN_NONE; +} + +static PyObject * +test_immortal_small_ints(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + for (int i = -5; i <= 256; i++) { + assert(verify_immortality(PyLong_FromLong(i))); + } + Py_RETURN_NONE; +} + +static PyMethodDef test_methods[] = { + {"test_immortal_builtins", test_immortal_builtins, METH_NOARGS}, + {"test_immortal_small_ints", test_immortal_small_ints, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Immortal(PyObject *mod) +{ + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 60ec81dad2ba9e..4d2d6832a827ae 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -39,6 +39,7 @@ int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); +int _PyTestCapi_Init_Immortal(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c1892f6fa0a4b8..30b8b6c6b3a87b 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3363,7 +3363,7 @@ test_gc_visit_objects_basic(PyObject *Py_UNUSED(self), } state.target = obj; state.found = 0; - + PyUnstable_GC_VisitObjects(gc_visit_callback_basic, &state); Py_DECREF(obj); if (!state.found) { @@ -3400,6 +3400,98 @@ test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self), Py_RETURN_NONE; } +typedef struct { + PyObject_HEAD +} ObjExtraData; + +static PyObject * +obj_extra_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + size_t extra_size = sizeof(PyObject *); + PyObject *obj = PyUnstable_Object_GC_NewWithExtraData(type, extra_size); + if (obj == NULL) { + return PyErr_NoMemory(); + } + PyObject_GC_Track(obj); + return obj; +} + +static PyObject ** +obj_extra_data_get_extra_storage(PyObject *self) +{ + return (PyObject **)((char *)self + Py_TYPE(self)->tp_basicsize); +} + +static PyObject * +obj_extra_data_get(PyObject *self, void *Py_UNUSED(ignored)) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + PyObject *value = *extra_storage; + if (!value) { + Py_RETURN_NONE; + } + return Py_NewRef(value); +} + +static int +obj_extra_data_set(PyObject *self, PyObject *newval, void *Py_UNUSED(ignored)) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + Py_CLEAR(*extra_storage); + if (newval) { + *extra_storage = Py_NewRef(newval); + } + return 0; +} + +static PyGetSetDef obj_extra_data_getset[] = { + {"extra", (getter)obj_extra_data_get, (setter)obj_extra_data_set, NULL}, + {NULL} +}; + +static int +obj_extra_data_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + PyObject *value = *extra_storage; + Py_VISIT(value); + return 0; +} + +static int +obj_extra_data_clear(PyObject *self) +{ + PyObject **extra_storage = obj_extra_data_get_extra_storage(self); + Py_CLEAR(*extra_storage); + return 0; +} + +static void +obj_extra_data_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + obj_extra_data_clear(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot ObjExtraData_Slots[] = { + {Py_tp_getset, obj_extra_data_getset}, + {Py_tp_dealloc, obj_extra_data_dealloc}, + {Py_tp_traverse, obj_extra_data_traverse}, + {Py_tp_clear, obj_extra_data_clear}, + {Py_tp_new, obj_extra_data_new}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec ObjExtraData_TypeSpec = { + .name = "_testcapi.ObjExtraData", + .basicsize = sizeof(ObjExtraData), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .slots = ObjExtraData_Slots, +}; struct atexit_data { int called; @@ -4124,6 +4216,17 @@ PyInit__testcapi(void) Py_INCREF(&MethStatic_Type); PyModule_AddObject(m, "MethStatic", (PyObject *)&MethStatic_Type); + PyObject *ObjExtraData_Type = PyType_FromModuleAndSpec( + m, &ObjExtraData_TypeSpec, NULL); + if (ObjExtraData_Type == 0) { + return NULL; + } + int ret = PyModule_AddType(m, (PyTypeObject*)ObjExtraData_Type); + Py_DECREF(&ObjExtraData_Type); + if (ret < 0) { + return NULL; + } + PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX)); PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN)); PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX)); @@ -4210,6 +4313,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_PyOS(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Immortal(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 632fac2de0c419..1e38f1aa63499b 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -14,7 +14,7 @@ #include "Python.h" #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg +#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg, _PyCompile_Assemble #include "pycore_fileutils.h" // _Py_normpath #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head @@ -625,6 +625,68 @@ _testinternalcapi_optimize_cfg_impl(PyObject *module, PyObject *instructions, return _PyCompile_OptimizeCfg(instructions, consts); } +static int +get_nonnegative_int_from_dict(PyObject *dict, const char *key) { + PyObject *obj = PyDict_GetItemString(dict, key); + if (obj == NULL) { + return -1; + } + return PyLong_AsLong(obj); +} + +/*[clinic input] + +_testinternalcapi.assemble_code_object -> object + + filename: object + instructions: object + metadata: object + +Create a code object for the given instructions. +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_assemble_code_object_impl(PyObject *module, + PyObject *filename, + PyObject *instructions, + PyObject *metadata) +/*[clinic end generated code: output=38003dc16a930f48 input=e713ad77f08fb3a8]*/ + +{ + assert(PyDict_Check(metadata)); + _PyCompile_CodeUnitMetadata umd; + + umd.u_name = PyDict_GetItemString(metadata, "name"); + umd.u_qualname = PyDict_GetItemString(metadata, "qualname"); + + assert(PyUnicode_Check(umd.u_name)); + assert(PyUnicode_Check(umd.u_qualname)); + + umd.u_consts = PyDict_GetItemString(metadata, "consts"); + umd.u_names = PyDict_GetItemString(metadata, "names"); + umd.u_varnames = PyDict_GetItemString(metadata, "varnames"); + umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars"); + umd.u_freevars = PyDict_GetItemString(metadata, "freevars"); + + assert(PyList_Check(umd.u_consts)); + assert(PyDict_Check(umd.u_names)); + assert(PyDict_Check(umd.u_varnames)); + assert(PyDict_Check(umd.u_cellvars)); + assert(PyDict_Check(umd.u_freevars)); + + umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount"); + umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount"); + umd.u_kwonlyargcount = get_nonnegative_int_from_dict(metadata, "kwonlyargcount"); + umd.u_firstlineno = get_nonnegative_int_from_dict(metadata, "firstlineno"); + + assert(umd.u_argcount >= 0); + assert(umd.u_posonlyargcount >= 0); + assert(umd.u_kwonlyargcount >= 0); + assert(umd.u_firstlineno >= 0); + + return (PyObject*)_PyCompile_Assemble(&umd, filename, instructions); +} + static PyObject * get_interp_settings(PyObject *self, PyObject *args) @@ -705,6 +767,7 @@ static PyMethodDef module_functions[] = { {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF + _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF {"get_interp_settings", get_interp_settings, METH_VARARGS, NULL}, {"clear_extension", clear_extension, METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 43c5d771798634..6a780a80cd0bc4 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -482,14 +482,15 @@ _asyncio_Future__make_cancelled_error(FutureObj *self, PyObject *Py_UNUSED(ignor } PyDoc_STRVAR(_asyncio_Task___init____doc__, -"Task(coro, *, loop=None, name=None, context=None)\n" +"Task(coro, *, loop=None, name=None, context=None, eager_start=False)\n" "--\n" "\n" "A coroutine wrapped in a Future."); static int _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, - PyObject *name, PyObject *context); + PyObject *name, PyObject *context, + int eager_start); static int _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) @@ -497,14 +498,14 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) int return_value = -1; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 4 + #define NUM_KEYWORDS 5 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(coro), &_Py_ID(loop), &_Py_ID(name), &_Py_ID(context), }, + .ob_item = { &_Py_ID(coro), &_Py_ID(loop), &_Py_ID(name), &_Py_ID(context), &_Py_ID(eager_start), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -513,14 +514,14 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"coro", "loop", "name", "context", NULL}; + static const char * const _keywords[] = {"coro", "loop", "name", "context", "eager_start", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "Task", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[4]; + PyObject *argsbuf[5]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; @@ -528,6 +529,7 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *loop = Py_None; PyObject *name = Py_None; PyObject *context = Py_None; + int eager_start = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { @@ -549,9 +551,18 @@ _asyncio_Task___init__(PyObject *self, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - context = fastargs[3]; + if (fastargs[3]) { + context = fastargs[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + eager_start = PyObject_IsTrue(fastargs[4]); + if (eager_start < 0) { + goto exit; + } skip_optional_kwonly: - return_value = _asyncio_Task___init___impl((TaskObj *)self, coro, loop, name, context); + return_value = _asyncio_Task___init___impl((TaskObj *)self, coro, loop, name, context, eager_start); exit: return return_value; @@ -1064,6 +1075,63 @@ _asyncio__register_task(PyObject *module, PyObject *const *args, Py_ssize_t narg return return_value; } +PyDoc_STRVAR(_asyncio__register_eager_task__doc__, +"_register_eager_task($module, /, task)\n" +"--\n" +"\n" +"Register a new task in asyncio as executed by loop.\n" +"\n" +"Returns None."); + +#define _ASYNCIO__REGISTER_EAGER_TASK_METHODDEF \ + {"_register_eager_task", _PyCFunction_CAST(_asyncio__register_eager_task), METH_FASTCALL|METH_KEYWORDS, _asyncio__register_eager_task__doc__}, + +static PyObject * +_asyncio__register_eager_task_impl(PyObject *module, PyObject *task); + +static PyObject * +_asyncio__register_eager_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(task), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"task", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_register_eager_task", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *task; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + task = args[0]; + return_value = _asyncio__register_eager_task_impl(module, task); + +exit: + return return_value; +} + PyDoc_STRVAR(_asyncio__unregister_task__doc__, "_unregister_task($module, /, task)\n" "--\n" @@ -1121,6 +1189,63 @@ _asyncio__unregister_task(PyObject *module, PyObject *const *args, Py_ssize_t na return return_value; } +PyDoc_STRVAR(_asyncio__unregister_eager_task__doc__, +"_unregister_eager_task($module, /, task)\n" +"--\n" +"\n" +"Unregister a task.\n" +"\n" +"Returns None."); + +#define _ASYNCIO__UNREGISTER_EAGER_TASK_METHODDEF \ + {"_unregister_eager_task", _PyCFunction_CAST(_asyncio__unregister_eager_task), METH_FASTCALL|METH_KEYWORDS, _asyncio__unregister_eager_task__doc__}, + +static PyObject * +_asyncio__unregister_eager_task_impl(PyObject *module, PyObject *task); + +static PyObject * +_asyncio__unregister_eager_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(task), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"task", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_unregister_eager_task", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *task; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + task = args[0]; + return_value = _asyncio__unregister_eager_task_impl(module, task); + +exit: + return return_value; +} + PyDoc_STRVAR(_asyncio__enter_task__doc__, "_enter_task($module, /, loop, task)\n" "--\n" @@ -1243,6 +1368,66 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } +PyDoc_STRVAR(_asyncio__swap_current_task__doc__, +"_swap_current_task($module, /, loop, task)\n" +"--\n" +"\n" +"Temporarily swap in the supplied task and return the original one (or None).\n" +"\n" +"This is intended for use during eager coroutine execution."); + +#define _ASYNCIO__SWAP_CURRENT_TASK_METHODDEF \ + {"_swap_current_task", _PyCFunction_CAST(_asyncio__swap_current_task), METH_FASTCALL|METH_KEYWORDS, _asyncio__swap_current_task__doc__}, + +static PyObject * +_asyncio__swap_current_task_impl(PyObject *module, PyObject *loop, + PyObject *task); + +static PyObject * +_asyncio__swap_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(loop), &_Py_ID(task), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"loop", "task", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_swap_current_task", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *loop; + PyObject *task; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + loop = args[0]; + task = args[1]; + return_value = _asyncio__swap_current_task_impl(module, loop, task); + +exit: + return return_value; +} + PyDoc_STRVAR(_asyncio_current_task__doc__, "current_task($module, /, loop=None)\n" "--\n" @@ -1302,4 +1487,4 @@ _asyncio_current_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=00f494214f2fd008 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b0e283177b07639 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index e8d5681b194916..89573222572594 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -128,4 +128,66 @@ _testinternalcapi_optimize_cfg(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=efe95836482fd542 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_testinternalcapi_assemble_code_object__doc__, +"assemble_code_object($module, /, filename, instructions, metadata)\n" +"--\n" +"\n" +"Create a code object for the given instructions."); + +#define _TESTINTERNALCAPI_ASSEMBLE_CODE_OBJECT_METHODDEF \ + {"assemble_code_object", _PyCFunction_CAST(_testinternalcapi_assemble_code_object), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_assemble_code_object__doc__}, + +static PyObject * +_testinternalcapi_assemble_code_object_impl(PyObject *module, + PyObject *filename, + PyObject *instructions, + PyObject *metadata); + +static PyObject * +_testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(filename), &_Py_ID(instructions), &_Py_ID(metadata), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"filename", "instructions", "metadata", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "assemble_code_object", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *filename; + PyObject *instructions; + PyObject *metadata; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + filename = args[0]; + instructions = args[1]; + metadata = args[2]; + return_value = _testinternalcapi_assemble_code_object_impl(module, filename, instructions, metadata); + +exit: + return return_value; +} +/*[clinic end generated code: output=d5e08c9d67f9721f input=a9049054013a1b77]*/ diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 966c1e615502ef..8a4d1a4398281e 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1388,10 +1388,19 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase, return; } } + + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_WriteUnraisable(NULL); + return; + } + + PyObject *stack[] = {phase_obj, info}; for (Py_ssize_t i=0; i<PyList_GET_SIZE(gcstate->callbacks); i++) { PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_CallFunction(cb, "sO", phase, info); + r = PyObject_Vectorcall(cb, stack, 2, NULL); if (r == NULL) { PyErr_WriteUnraisable(cb); } @@ -1400,6 +1409,7 @@ invoke_gc_callback(PyThreadState *tstate, const char *phase, } Py_DECREF(cb); } + Py_DECREF(phase_obj); Py_XDECREF(info); assert(!_PyErr_Occurred(tstate)); } @@ -2357,6 +2367,19 @@ _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) return op; } +PyObject * +PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *tp, size_t extra_size) +{ + size_t presize = _PyType_PreHeaderSize(tp); + PyObject *op = gc_alloc(_PyObject_SIZE(tp) + extra_size, presize); + if (op == NULL) { + return NULL; + } + memset(op, 0, _PyObject_SIZE(tp) + extra_size); + _PyObject_Init(op, tp); + return op; +} + PyVarObject * _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) { diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 27b2ad4f2cb38f..e7e85cc19cda75 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -705,7 +705,6 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, case ' ': flags |= F_BLANK; continue; case '#': flags |= F_ALT; continue; case '0': flags |= F_ZERO; continue; - case 'z': flags |= F_NO_NEG_0; continue; } break; } diff --git a/Objects/call.c b/Objects/call.c index bd027e41f8a9a5..cf6e357a990441 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -8,16 +8,6 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() -static PyObject *const * -_PyStack_UnpackDict(PyThreadState *tstate, - PyObject *const *args, Py_ssize_t nargs, - PyObject *kwargs, PyObject **p_kwnames); - -static void -_PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, - PyObject *kwnames); - - static PyObject * null_error(PyThreadState *tstate) { @@ -965,7 +955,7 @@ _PyStack_AsDict(PyObject *const *values, PyObject *kwnames) The newly allocated argument vector supports PY_VECTORCALL_ARGUMENTS_OFFSET. When done, you must call _PyStack_UnpackDict_Free(stack, nargs, kwnames) */ -static PyObject *const * +PyObject *const * _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames) @@ -1034,7 +1024,7 @@ _PyStack_UnpackDict(PyThreadState *tstate, return stack; } -static void +void _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames) { @@ -1042,6 +1032,12 @@ _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, for (Py_ssize_t i = 0; i < n; i++) { Py_DECREF(stack[i]); } + _PyStack_UnpackDict_FreeNoDecRef(stack, kwnames); +} + +void +_PyStack_UnpackDict_FreeNoDecRef(PyObject *const *stack, PyObject *kwnames) +{ PyMem_Free((PyObject **)stack - 1); Py_DECREF(kwnames); } diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 6c9dfbd9b415cf..ba5ee291f08b0c 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3598,7 +3598,7 @@ _PyExc_InitTypes(PyInterpreterState *interp) { for (size_t i=0; i < Py_ARRAY_LENGTH(static_exceptions); i++) { PyTypeObject *exc = static_exceptions[i].exc; - if (_PyStaticType_InitBuiltin(exc) < 0) { + if (_PyStaticType_InitBuiltin(interp, exc) < 0) { return -1; } } @@ -3609,13 +3609,9 @@ _PyExc_InitTypes(PyInterpreterState *interp) static void _PyExc_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - for (Py_ssize_t i=Py_ARRAY_LENGTH(static_exceptions) - 1; i >= 0; i--) { PyTypeObject *exc = static_exceptions[i].exc; - _PyStaticType_Dealloc(exc); + _PyStaticType_Dealloc(interp, exc); } } diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 9c2315781bed36..d257857d9c619c 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -12,7 +12,7 @@ #include "pycore_object.h" // _PyObject_Init() #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_structseq.h" // _PyStructSequence_FiniType() +#include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include <ctype.h> #include <float.h> @@ -1991,8 +1991,9 @@ PyStatus _PyFloat_InitTypes(PyInterpreterState *interp) { /* Init float info */ - if (_PyStructSequence_InitBuiltin(&FloatInfoType, - &floatinfo_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &FloatInfoType, + &floatinfo_desc) < 0) + { return _PyStatus_ERR("can't init float info type"); } @@ -2028,9 +2029,7 @@ _PyFloat_Fini(PyInterpreterState *interp) void _PyFloat_FiniType(PyInterpreterState *interp) { - if (_Py_IsMainInterpreter(interp)) { - _PyStructSequence_FiniType(&FloatInfoType); - } + _PyStructSequence_FiniBuiltin(interp, &FloatInfoType); } /* Print summary info about the state of the optimized allocator */ diff --git a/Objects/longobject.c b/Objects/longobject.c index f84809b8a8986a..853e934e2107ea 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -7,9 +7,8 @@ #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _Py_SmallInts #include "pycore_object.h" // _PyObject_Init() -#include "pycore_pystate.h" // _Py_IsMainInterpreter() #include "pycore_runtime.h" // _PY_NSMALLPOSINTS -#include "pycore_structseq.h" // _PyStructSequence_FiniType() +#include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include <ctype.h> #include <float.h> @@ -6352,7 +6351,9 @@ PyStatus _PyLong_InitTypes(PyInterpreterState *interp) { /* initialize int_info */ - if (_PyStructSequence_InitBuiltin(&Int_InfoType, &int_info_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &Int_InfoType, + &int_info_desc) < 0) + { return _PyStatus_ERR("can't init int info type"); } @@ -6363,9 +6364,5 @@ _PyLong_InitTypes(PyInterpreterState *interp) void _PyLong_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStructSequence_FiniType(&Int_InfoType); + _PyStructSequence_FiniBuiltin(interp, &Int_InfoType); } diff --git a/Objects/object.c b/Objects/object.c index 4ce10cf1192d3f..ee8690101d3cc0 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2105,7 +2105,7 @@ _PyTypes_InitTypes(PyInterpreterState *interp) // All other static types (unless initialized elsewhere) for (size_t i=0; i < Py_ARRAY_LENGTH(static_types); i++) { PyTypeObject *type = static_types[i]; - if (_PyStaticType_InitBuiltin(type) < 0) { + if (_PyStaticType_InitBuiltin(interp, type) < 0) { return _PyStatus_ERR("Can't initialize builtin type"); } if (type == &PyType_Type) { @@ -2128,15 +2128,11 @@ _PyTypes_InitTypes(PyInterpreterState *interp) void _PyTypes_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - // Deallocate types in the reverse order to deallocate subclasses before // their base classes. for (Py_ssize_t i=Py_ARRAY_LENGTH(static_types)-1; i>=0; i--) { PyTypeObject *type = static_types[i]; - _PyStaticType_Dealloc(type); + _PyStaticType_Dealloc(interp, type); } } diff --git a/Objects/structseq.c b/Objects/structseq.c index 88a71bc52958f5..f63660acb639c3 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -26,7 +26,7 @@ const char * const PyStructSequence_UnnamedField = "unnamed field"; static Py_ssize_t get_type_attr_as_size(PyTypeObject *tp, PyObject *name) { - PyObject *v = PyDict_GetItemWithError(tp->tp_dict, name); + PyObject *v = PyDict_GetItemWithError(_PyType_GetDict(tp), name); if (v == NULL && !PyErr_Occurred()) { PyErr_Format(PyExc_TypeError, "Missed attribute '%U' of type %s", @@ -433,12 +433,10 @@ initialize_structseq_dict(PyStructSequence_Desc *desc, PyObject* dict, static PyMemberDef * initialize_members(PyStructSequence_Desc *desc, - Py_ssize_t *pn_members, Py_ssize_t *pn_unnamed_members) + Py_ssize_t n_members, Py_ssize_t n_unnamed_members) { PyMemberDef *members; - Py_ssize_t n_members, n_unnamed_members; - n_members = count_members(desc, &n_unnamed_members); members = PyMem_NEW(PyMemberDef, n_members - n_unnamed_members + 1); if (members == NULL) { PyErr_NoMemory(); @@ -463,8 +461,6 @@ initialize_members(PyStructSequence_Desc *desc, } members[k].name = NULL; - *pn_members = n_members; - *pn_unnamed_members = n_unnamed_members; return members; } @@ -497,7 +493,7 @@ initialize_static_type(PyTypeObject *type, PyStructSequence_Desc *desc, Py_INCREF(type); if (initialize_structseq_dict( - desc, type->tp_dict, n_members, n_unnamed_members) < 0) { + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { Py_DECREF(type); return -1; } @@ -506,43 +502,63 @@ initialize_static_type(PyTypeObject *type, PyStructSequence_Desc *desc, } int -_PyStructSequence_InitBuiltinWithFlags(PyTypeObject *type, +_PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, + PyTypeObject *type, PyStructSequence_Desc *desc, unsigned long tp_flags) { - if (type->tp_flags & Py_TPFLAGS_READY) { - if (_PyStaticType_InitBuiltin(type) < 0) { - goto failed_init_builtin; + Py_ssize_t n_unnamed_members; + Py_ssize_t n_members = count_members(desc, &n_unnamed_members); + PyMemberDef *members = NULL; + + int initialized = 1; + if ((type->tp_flags & Py_TPFLAGS_READY) == 0) { + assert(type->tp_name == NULL); + assert(type->tp_members == NULL); + assert(type->tp_base == NULL); + + members = initialize_members(desc, n_members, n_unnamed_members); + if (members == NULL) { + goto error; } - return 0; - } - - PyMemberDef *members; - Py_ssize_t n_members, n_unnamed_members; + initialize_static_fields(type, desc, members, tp_flags); - members = initialize_members(desc, &n_members, &n_unnamed_members); - if (members == NULL) { - return -1; + _Py_SetImmortal(type); + initialized = 0; + } +#ifndef NDEBUG + else { + // Ensure that the type was initialized. + assert(type->tp_name != NULL); + assert(type->tp_members != NULL); + assert(type->tp_base == &PyTuple_Type); + assert((type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + assert(_Py_IsImmortal(type)); } - initialize_static_fields(type, desc, members, tp_flags); +#endif - Py_INCREF(type); // XXX It should be immortal. - if (_PyStaticType_InitBuiltin(type) < 0) { - PyMem_Free(members); - goto failed_init_builtin; + if (_PyStaticType_InitBuiltin(interp, type) < 0) { + PyErr_Format(PyExc_RuntimeError, + "Can't initialize builtin type %s", + desc->name); + goto error; + } + // This should be dropped if tp_dict is made per-interpreter. + if (initialized) { + return 0; } if (initialize_structseq_dict( - desc, type->tp_dict, n_members, n_unnamed_members) < 0) { - PyMem_Free(members); - return -1; + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { + goto error; } + return 0; -failed_init_builtin: - PyErr_Format(PyExc_RuntimeError, - "Can't initialize builtin type %s", - desc->name); +error: + if (members != NULL) { + PyMem_Free(members); + } return -1; } @@ -566,7 +582,8 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) return -1; } - members = initialize_members(desc, &n_members, &n_unnamed_members); + n_members = count_members(desc, &n_unnamed_members); + members = initialize_members(desc, n_members, n_unnamed_members); if (members == NULL) { return -1; } @@ -585,35 +602,34 @@ PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) } +/* This is exposed in the internal API, not the public API. + It is only called on builtin static types, which are all + initialized via _PyStructSequence_InitBuiltinWithFlags(). */ + void -_PyStructSequence_FiniType(PyTypeObject *type) +_PyStructSequence_FiniBuiltin(PyInterpreterState *interp, PyTypeObject *type) { // Ensure that the type is initialized assert(type->tp_name != NULL); assert(type->tp_base == &PyTuple_Type); + assert((type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); + assert(_Py_IsImmortal(type)); // Cannot delete a type if it still has subclasses if (_PyType_HasSubclasses(type)) { + // XXX Shouldn't this be an error? return; } - // Undo PyStructSequence_NewType() - type->tp_name = NULL; - PyMem_Free(type->tp_members); + _PyStaticType_Dealloc(interp, type); - _PyStaticType_Dealloc(type); - assert(Py_REFCNT(type) == 1); - // Undo Py_INCREF(type) of _PyStructSequence_InitType(). - // Don't use Py_DECREF(): static type must not be deallocated - Py_SET_REFCNT(type, 0); -#ifdef Py_REF_DEBUG - _Py_DecRefTotal(_PyInterpreterState_GET()); -#endif - - // Make sure that _PyStructSequence_InitType() will initialize - // the type again - assert(Py_REFCNT(type) == 0); - assert(type->tp_name == NULL); + if (_Py_IsMainInterpreter(interp)) { + // Undo _PyStructSequence_InitBuiltinWithFlags(). + type->tp_name = NULL; + PyMem_Free(type->tp_members); + type->tp_members = NULL; + type->tp_base = NULL; + } } @@ -627,7 +643,8 @@ _PyStructSequence_NewType(PyStructSequence_Desc *desc, unsigned long tp_flags) Py_ssize_t n_members, n_unnamed_members; /* Initialize MemberDefs */ - members = initialize_members(desc, &n_members, &n_unnamed_members); + n_members = count_members(desc, &n_unnamed_members); + members = initialize_members(desc, n_members, n_unnamed_members); if (members == NULL) { return NULL; } @@ -658,7 +675,7 @@ _PyStructSequence_NewType(PyStructSequence_Desc *desc, unsigned long tp_flags) } if (initialize_structseq_dict( - desc, type->tp_dict, n_members, n_unnamed_members) < 0) { + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { Py_DECREF(type); return NULL; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 38b99315457a58..a9d3a69263fb40 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -64,18 +64,28 @@ lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound); static int slot_tp_setattro(PyObject *self, PyObject *name, PyObject *value); -static inline PyTypeObject * subclass_from_ref(PyObject *ref); + +static inline PyTypeObject * +type_from_ref(PyObject *ref) +{ + assert(PyWeakref_CheckRef(ref)); + PyObject *obj = PyWeakref_GET_OBJECT(ref); // borrowed ref + assert(obj != NULL); + if (obj == Py_None) { + return NULL; + } + assert(PyType_Check(obj)); + return _PyType_CAST(obj); +} /* helpers for for static builtin types */ -#ifndef NDEBUG static inline int static_builtin_index_is_set(PyTypeObject *self) { return self->tp_subclasses != NULL; } -#endif static inline size_t static_builtin_index_get(PyTypeObject *self) @@ -107,43 +117,46 @@ static_builtin_state_get(PyInterpreterState *interp, PyTypeObject *self) /* For static types we store some state in an array on each interpreter. */ static_builtin_state * -_PyStaticType_GetState(PyTypeObject *self) +_PyStaticType_GetState(PyInterpreterState *interp, PyTypeObject *self) { assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); - PyInterpreterState *interp = _PyInterpreterState_GET(); return static_builtin_state_get(interp, self); } +/* Set the type's per-interpreter state. */ static void -static_builtin_state_init(PyTypeObject *self) +static_builtin_state_init(PyInterpreterState *interp, PyTypeObject *self) { - /* Set the type's per-interpreter state. */ - PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!static_builtin_index_is_set(self)) { + static_builtin_index_set(self, interp->types.num_builtins_initialized); + } + static_builtin_state *state = static_builtin_state_get(interp, self); /* It should only be called once for each builtin type. */ - assert(!static_builtin_index_is_set(self)); - - static_builtin_index_set(self, interp->types.num_builtins_initialized); - interp->types.num_builtins_initialized++; - - static_builtin_state *state = static_builtin_state_get(interp, self); + assert(state->type == NULL); state->type = self; + /* state->tp_subclasses is left NULL until init_subclasses() sets it. */ /* state->tp_weaklist is left NULL until insert_head() or insert_after() (in weakrefobject.c) sets it. */ + + interp->types.num_builtins_initialized++; } +/* Reset the type's per-interpreter state. + This basically undoes what static_builtin_state_init() did. */ static void -static_builtin_state_clear(PyTypeObject *self) +static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) { - /* Reset the type's per-interpreter state. - This basically undoes what static_builtin_state_init() did. */ - PyInterpreterState *interp = _PyInterpreterState_GET(); - static_builtin_state *state = static_builtin_state_get(interp, self); + + assert(state->type != NULL); state->type = NULL; assert(state->tp_weaklist == NULL); // It was already cleared out. - static_builtin_index_clear(self); + + if (_Py_IsMainInterpreter(interp)) { + static_builtin_index_clear(self); + } assert(interp->types.num_builtins_initialized > 0); interp->types.num_builtins_initialized--; @@ -154,6 +167,178 @@ static_builtin_state_clear(PyTypeObject *self) /* end static builtin helpers */ +/* accessors for objects stored on PyTypeObject */ + +static inline PyObject * +lookup_tp_dict(PyTypeObject *self) +{ + return self->tp_dict; +} + +PyObject * +_PyType_GetDict(PyTypeObject *self) +{ + return lookup_tp_dict(self); +} + +static inline void +set_tp_dict(PyTypeObject *self, PyObject *dict) +{ + self->tp_dict = dict; +} + +static inline void +clear_tp_dict(PyTypeObject *self) +{ + Py_CLEAR(self->tp_dict); +} + + +static inline PyObject * +lookup_tp_bases(PyTypeObject *self) +{ + return self->tp_bases; +} + +PyObject * +_PyType_GetBases(PyTypeObject *self) +{ + return lookup_tp_bases(self); +} + +static inline void +set_tp_bases(PyTypeObject *self, PyObject *bases) +{ + self->tp_bases = bases; +} + +static inline void +clear_tp_bases(PyTypeObject *self) +{ + Py_CLEAR(self->tp_bases); +} + + +static inline PyObject * +lookup_tp_mro(PyTypeObject *self) +{ + return self->tp_mro; +} + +PyObject * +_PyType_GetMRO(PyTypeObject *self) +{ + return lookup_tp_mro(self); +} + +static inline void +set_tp_mro(PyTypeObject *self, PyObject *mro) +{ + self->tp_mro = mro; +} + +static inline void +clear_tp_mro(PyTypeObject *self) +{ + Py_CLEAR(self->tp_mro); +} + + +static PyObject * +init_tp_subclasses(PyTypeObject *self) +{ + PyObject *subclasses = PyDict_New(); + if (subclasses == NULL) { + return NULL; + } + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + state->tp_subclasses = subclasses; + return subclasses; + } + self->tp_subclasses = (void *)subclasses; + return subclasses; +} + +static void +clear_tp_subclasses(PyTypeObject *self) +{ + /* Delete the dictionary to save memory. _PyStaticType_Dealloc() + callers also test if tp_subclasses is NULL to check if a static type + has no subclass. */ + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + Py_CLEAR(state->tp_subclasses); + return; + } + Py_CLEAR(self->tp_subclasses); +} + +static inline PyObject * +lookup_tp_subclasses(PyTypeObject *self) +{ + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + return state->tp_subclasses; + } + return (PyObject *)self->tp_subclasses; +} + +int +_PyType_HasSubclasses(PyTypeObject *self) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN + // XXX _PyStaticType_GetState() should never return NULL. + && _PyStaticType_GetState(interp, self) == NULL) + { + return 0; + } + if (lookup_tp_subclasses(self) == NULL) { + return 0; + } + return 1; +} + +PyObject* +_PyType_GetSubclasses(PyTypeObject *self) +{ + PyObject *list = PyList_New(0); + if (list == NULL) { + return NULL; + } + + PyObject *subclasses = lookup_tp_subclasses(self); // borrowed ref + if (subclasses == NULL) { + return list; + } + assert(PyDict_CheckExact(subclasses)); + // The loop cannot modify tp_subclasses, there is no need + // to hold a strong reference (use a borrowed reference). + + Py_ssize_t i = 0; + PyObject *ref; // borrowed ref + while (PyDict_Next(subclasses, &i, NULL, &ref)) { + PyTypeObject *subclass = type_from_ref(ref); // borrowed + if (subclass == NULL) { + continue; + } + + if (PyList_Append(list, _PyObject_CAST(subclass)) < 0) { + Py_DECREF(list); + return NULL; + } + } + return list; +} + +/* end accessors for objects stored on PyTypeObject */ + + /* * finds the beginning of the docstring's introspection signature. * if present, returns a pointer pointing to the first '('. @@ -224,7 +409,7 @@ _PyType_CheckConsistency(PyTypeObject *type) CHECK(PyType_Check(type)); CHECK(!(type->tp_flags & Py_TPFLAGS_READYING)); - CHECK(type->tp_dict != NULL); + CHECK(lookup_tp_dict(type) != NULL); if (type->tp_flags & Py_TPFLAGS_HAVE_GC) { // bpo-44263: tp_traverse is required if Py_TPFLAGS_HAVE_GC is set. @@ -234,7 +419,7 @@ _PyType_CheckConsistency(PyTypeObject *type) if (type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION) { CHECK(type->tp_new == NULL); - CHECK(PyDict_Contains(type->tp_dict, &_Py_ID(__new__)) == 0); + CHECK(PyDict_Contains(lookup_tp_dict(type), &_Py_ID(__new__)) == 0); } return 1; @@ -360,8 +545,6 @@ _PyTypes_Fini(PyInterpreterState *interp) } -static PyObject * lookup_subclasses(PyTypeObject *); - int PyType_AddWatcher(PyType_WatchCallback callback) { @@ -461,14 +644,14 @@ PyType_Modified(PyTypeObject *type) return; } - PyObject *subclasses = lookup_subclasses(type); + PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses != NULL) { assert(PyDict_CheckExact(subclasses)); Py_ssize_t i = 0; PyObject *ref; while (PyDict_Next(subclasses, &i, NULL, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == NULL) { continue; } @@ -597,7 +780,7 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) assert (type->tp_version_tag != 0); } - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); Py_ssize_t n = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < n; i++) { PyObject *b = PyTuple_GET_ITEM(bases, i); @@ -748,7 +931,8 @@ type_module(PyTypeObject *type, void *context) PyObject *mod; if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { - mod = PyDict_GetItemWithError(type->tp_dict, &_Py_ID(__module__)); + PyObject *dict = lookup_tp_dict(type); + mod = PyDict_GetItemWithError(dict, &_Py_ID(__module__)); if (mod == NULL) { if (!PyErr_Occurred()) { PyErr_Format(PyExc_AttributeError, "__module__"); @@ -780,7 +964,8 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context) PyType_Modified(type); - return PyDict_SetItem(type->tp_dict, &_Py_ID(__module__), value); + PyObject *dict = lookup_tp_dict(type); + return PyDict_SetItem(dict, &_Py_ID(__module__), value); } static PyObject * @@ -789,9 +974,10 @@ type_abstractmethods(PyTypeObject *type, void *context) PyObject *mod = NULL; /* type itself has an __abstractmethods__ descriptor (this). Don't return that. */ - if (type != &PyType_Type) - mod = PyDict_GetItemWithError(type->tp_dict, - &_Py_ID(__abstractmethods__)); + if (type != &PyType_Type) { + PyObject *dict = lookup_tp_dict(type); + mod = PyDict_GetItemWithError(dict, &_Py_ID(__abstractmethods__)); + } if (!mod) { if (!PyErr_Occurred()) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); @@ -809,15 +995,16 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) special to update subclasses. */ int abstract, res; + PyObject *dict = lookup_tp_dict(type); if (value != NULL) { abstract = PyObject_IsTrue(value); if (abstract < 0) return -1; - res = PyDict_SetItem(type->tp_dict, &_Py_ID(__abstractmethods__), value); + res = PyDict_SetItem(dict, &_Py_ID(__abstractmethods__), value); } else { abstract = 0; - res = PyDict_DelItem(type->tp_dict, &_Py_ID(__abstractmethods__)); + res = PyDict_DelItem(dict, &_Py_ID(__abstractmethods__)); if (res && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_SetObject(PyExc_AttributeError, &_Py_ID(__abstractmethods__)); return -1; @@ -836,7 +1023,7 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) static PyObject * type_get_bases(PyTypeObject *type, void *context) { - return Py_NewRef(type->tp_bases); + return Py_NewRef(lookup_tp_bases(type)); } static PyTypeObject *best_base(PyObject *); @@ -864,7 +1051,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) /* error / reentrance */ return res; } - PyObject *new_mro = type->tp_mro; + PyObject *new_mro = lookup_tp_mro(type); PyObject *tuple; if (old_mro != NULL) { @@ -883,7 +1070,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) Py_XDECREF(tuple); if (res < 0) { - type->tp_mro = old_mro; + set_tp_mro(type, old_mro); Py_DECREF(new_mro); return -1; } @@ -962,7 +1149,8 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) below), which in turn may cause an inheritance cycle through tp_base chain. And this is definitely not what you want to ever happen. */ - (base->tp_mro != NULL && type_is_subtype_base_chain(base, type))) + (lookup_tp_mro(base) != NULL + && type_is_subtype_base_chain(base, type))) { PyErr_SetString(PyExc_TypeError, "a __bases__ item causes an inheritance cycle"); @@ -979,11 +1167,11 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) return -1; } - PyObject *old_bases = type->tp_bases; + PyObject *old_bases = lookup_tp_bases(type); assert(old_bases != NULL); PyTypeObject *old_base = type->tp_base; - type->tp_bases = Py_NewRef(new_bases); + set_tp_bases(type, Py_NewRef(new_bases)); type->tp_base = (PyTypeObject *)Py_NewRef(new_base); PyObject *temp = PyList_New(0); @@ -998,7 +1186,7 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) /* Take no action in case if type->tp_bases has been replaced through reentrance. */ int res; - if (type->tp_bases == new_bases) { + if (lookup_tp_bases(type) == new_bases) { /* any base that was in __bases__ but now isn't, we need to remove |type| from its tp_subclasses. conversely, any class now in __bases__ that wasn't @@ -1029,18 +1217,18 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) PyArg_UnpackTuple(PyList_GET_ITEM(temp, i), "", 2, 3, &cls, &new_mro, &old_mro); /* Do not rollback if cls has a newer version of MRO. */ - if (cls->tp_mro == new_mro) { - cls->tp_mro = Py_XNewRef(old_mro); + if (lookup_tp_mro(cls) == new_mro) { + set_tp_mro(cls, Py_XNewRef(old_mro)); Py_DECREF(new_mro); } } Py_DECREF(temp); bail: - if (type->tp_bases == new_bases) { + if (lookup_tp_bases(type) == new_bases) { assert(type->tp_base == new_base); - type->tp_bases = old_bases; + set_tp_bases(type, old_bases); type->tp_base = old_base; Py_DECREF(new_bases); @@ -1058,10 +1246,11 @@ type_set_bases(PyTypeObject *type, PyObject *new_bases, void *context) static PyObject * type_dict(PyTypeObject *type, void *context) { - if (type->tp_dict == NULL) { + PyObject *dict = lookup_tp_dict(type); + if (dict == NULL) { Py_RETURN_NONE; } - return PyDictProxy_New(type->tp_dict); + return PyDictProxy_New(dict); } static PyObject * @@ -1071,7 +1260,8 @@ type_get_doc(PyTypeObject *type, void *context) if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE) && type->tp_doc != NULL) { return _PyType_GetDocFromInternalDoc(type->tp_name, type->tp_doc); } - result = PyDict_GetItemWithError(type->tp_dict, &_Py_ID(__doc__)); + PyObject *dict = lookup_tp_dict(type); + result = PyDict_GetItemWithError(dict, &_Py_ID(__doc__)); if (result == NULL) { if (!PyErr_Occurred()) { result = Py_NewRef(Py_None); @@ -1099,7 +1289,8 @@ type_set_doc(PyTypeObject *type, PyObject *value, void *context) if (!check_set_special_type_attr(type, value, "__doc__")) return -1; PyType_Modified(type); - return PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), value); + PyObject *dict = lookup_tp_dict(type); + return PyDict_SetItem(dict, &_Py_ID(__doc__), value); } static PyObject * @@ -1112,9 +1303,9 @@ type_get_annotations(PyTypeObject *type, void *context) PyObject *annotations; /* there's no _PyDict_GetItemId without WithError, so let's LBYL. */ - if (PyDict_Contains(type->tp_dict, &_Py_ID(__annotations__))) { - annotations = PyDict_GetItemWithError( - type->tp_dict, &_Py_ID(__annotations__)); + PyObject *dict = lookup_tp_dict(type); + if (PyDict_Contains(dict, &_Py_ID(__annotations__))) { + annotations = PyDict_GetItemWithError(dict, &_Py_ID(__annotations__)); /* ** PyDict_GetItemWithError could still fail, ** for instance with a well-timed Ctrl-C or a MemoryError. @@ -1132,7 +1323,7 @@ type_get_annotations(PyTypeObject *type, void *context) annotations = PyDict_New(); if (annotations) { int result = PyDict_SetItem( - type->tp_dict, &_Py_ID(__annotations__), annotations); + dict, &_Py_ID(__annotations__), annotations); if (result) { Py_CLEAR(annotations); } else { @@ -1154,16 +1345,17 @@ type_set_annotations(PyTypeObject *type, PyObject *value, void *context) } int result; + PyObject *dict = lookup_tp_dict(type); if (value != NULL) { /* set */ - result = PyDict_SetItem(type->tp_dict, &_Py_ID(__annotations__), value); + result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); } else { /* delete */ - if (!PyDict_Contains(type->tp_dict, &_Py_ID(__annotations__))) { + if (!PyDict_Contains(dict, &_Py_ID(__annotations__))) { PyErr_Format(PyExc_AttributeError, "__annotations__"); return -1; } - result = PyDict_DelItem(type->tp_dict, &_Py_ID(__annotations__)); + result = PyDict_DelItem(dict, &_Py_ID(__annotations__)); } if (result == 0) { @@ -1750,7 +1942,7 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) { PyObject *mro; - mro = a->tp_mro; + mro = lookup_tp_mro(a); if (mro != NULL) { /* Deal with multiple inheritance without recursion by walking the MRO tuple */ @@ -2134,17 +2326,17 @@ mro_implementation(PyTypeObject *type) return NULL; } - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); Py_ssize_t n = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); - if (base->tp_mro == NULL) { + if (lookup_tp_mro(base) == NULL) { PyErr_Format(PyExc_TypeError, "Cannot extend an incomplete type '%.100s'", base->tp_name); return NULL; } - assert(PyTuple_Check(base->tp_mro)); + assert(PyTuple_Check(lookup_tp_mro(base))); } if (n == 1) { @@ -2152,7 +2344,8 @@ mro_implementation(PyTypeObject *type) * is trivial. */ PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, 0)); - Py_ssize_t k = PyTuple_GET_SIZE(base->tp_mro); + PyObject *base_mro = lookup_tp_mro(base); + Py_ssize_t k = PyTuple_GET_SIZE(base_mro); PyObject *result = PyTuple_New(k + 1); if (result == NULL) { return NULL; @@ -2161,7 +2354,7 @@ mro_implementation(PyTypeObject *type) ; PyTuple_SET_ITEM(result, 0, Py_NewRef(type)); for (Py_ssize_t i = 0; i < k; i++) { - PyObject *cls = PyTuple_GET_ITEM(base->tp_mro, i); + PyObject *cls = PyTuple_GET_ITEM(base_mro, i); PyTuple_SET_ITEM(result, i + 1, Py_NewRef(cls)); } return result; @@ -2188,7 +2381,7 @@ mro_implementation(PyTypeObject *type) for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); - to_merge[i] = base->tp_mro; + to_merge[i] = lookup_tp_mro(base); } to_merge[n] = bases; @@ -2343,9 +2536,9 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) /* Keep a reference to be able to do a reentrancy check below. Don't let old_mro be GC'ed and its address be reused for another object, like (suddenly!) a new tp_mro. */ - old_mro = Py_XNewRef(type->tp_mro); + old_mro = Py_XNewRef(lookup_tp_mro(type)); new_mro = mro_invoke(type); /* might cause reentrance */ - reent = (type->tp_mro != old_mro); + reent = (lookup_tp_mro(type) != old_mro); Py_XDECREF(old_mro); if (new_mro == NULL) { return -1; @@ -2356,12 +2549,12 @@ mro_internal(PyTypeObject *type, PyObject **p_old_mro) return 0; } - type->tp_mro = new_mro; + set_tp_mro(type, new_mro); - type_mro_modified(type, type->tp_mro); + type_mro_modified(type, new_mro); /* corner case: the super class might have been hidden from the custom MRO */ - type_mro_modified(type, type->tp_bases); + type_mro_modified(type, lookup_tp_bases(type)); // XXX Expand this to Py_TPFLAGS_IMMUTABLETYPE? if (!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)) { @@ -2963,7 +3156,7 @@ type_new_alloc(type_new_ctx *ctx) type->tp_as_mapping = &et->as_mapping; type->tp_as_buffer = &et->as_buffer; - type->tp_bases = Py_NewRef(ctx->bases); + set_tp_bases(type, Py_NewRef(ctx->bases)); type->tp_base = (PyTypeObject *)Py_NewRef(ctx->base); type->tp_dealloc = subtype_dealloc; @@ -3003,7 +3196,8 @@ type_new_set_name(const type_new_ctx *ctx, PyTypeObject *type) static int type_new_set_module(PyTypeObject *type) { - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__module__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__module__)); if (r < 0) { return -1; } @@ -3024,7 +3218,7 @@ type_new_set_module(PyTypeObject *type) return 0; } - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__module__), module) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__module__), module) < 0) { return -1; } return 0; @@ -3037,8 +3231,8 @@ static int type_new_set_ht_name(PyTypeObject *type) { PyHeapTypeObject *et = (PyHeapTypeObject *)type; - PyObject *qualname = PyDict_GetItemWithError( - type->tp_dict, &_Py_ID(__qualname__)); + PyObject *dict = lookup_tp_dict(type); + PyObject *qualname = PyDict_GetItemWithError(dict, &_Py_ID(__qualname__)); if (qualname != NULL) { if (!PyUnicode_Check(qualname)) { PyErr_Format(PyExc_TypeError, @@ -3047,7 +3241,7 @@ type_new_set_ht_name(PyTypeObject *type) return -1; } et->ht_qualname = Py_NewRef(qualname); - if (PyDict_DelItem(type->tp_dict, &_Py_ID(__qualname__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__qualname__)) < 0) { return -1; } } @@ -3067,7 +3261,8 @@ type_new_set_ht_name(PyTypeObject *type) static int type_new_set_doc(PyTypeObject *type) { - PyObject *doc = PyDict_GetItemWithError(type->tp_dict, &_Py_ID(__doc__)); + PyObject *dict = lookup_tp_dict(type); + PyObject *doc = PyDict_GetItemWithError(dict, &_Py_ID(__doc__)); if (doc == NULL) { if (PyErr_Occurred()) { return -1; @@ -3102,7 +3297,8 @@ type_new_set_doc(PyTypeObject *type) static int type_new_staticmethod(PyTypeObject *type, PyObject *attr) { - PyObject *func = PyDict_GetItemWithError(type->tp_dict, attr); + PyObject *dict = lookup_tp_dict(type); + PyObject *func = PyDict_GetItemWithError(dict, attr); if (func == NULL) { if (PyErr_Occurred()) { return -1; @@ -3117,7 +3313,7 @@ type_new_staticmethod(PyTypeObject *type, PyObject *attr) if (static_func == NULL) { return -1; } - if (PyDict_SetItem(type->tp_dict, attr, static_func) < 0) { + if (PyDict_SetItem(dict, attr, static_func) < 0) { Py_DECREF(static_func); return -1; } @@ -3129,7 +3325,8 @@ type_new_staticmethod(PyTypeObject *type, PyObject *attr) static int type_new_classmethod(PyTypeObject *type, PyObject *attr) { - PyObject *func = PyDict_GetItemWithError(type->tp_dict, attr); + PyObject *dict = lookup_tp_dict(type); + PyObject *func = PyDict_GetItemWithError(dict, attr); if (func == NULL) { if (PyErr_Occurred()) { return -1; @@ -3145,7 +3342,7 @@ type_new_classmethod(PyTypeObject *type, PyObject *attr) return -1; } - if (PyDict_SetItem(type->tp_dict, attr, method) < 0) { + if (PyDict_SetItem(dict, attr, method) < 0) { Py_DECREF(method); return -1; } @@ -3231,8 +3428,8 @@ type_new_set_slots(const type_new_ctx *ctx, PyTypeObject *type) static int type_new_set_classcell(PyTypeObject *type) { - PyObject *cell = PyDict_GetItemWithError( - type->tp_dict, &_Py_ID(__classcell__)); + PyObject *dict = lookup_tp_dict(type); + PyObject *cell = PyDict_GetItemWithError(dict, &_Py_ID(__classcell__)); if (cell == NULL) { if (PyErr_Occurred()) { return -1; @@ -3249,7 +3446,7 @@ type_new_set_classcell(PyTypeObject *type) } (void)PyCell_Set(cell, (PyObject *) type); - if (PyDict_DelItem(type->tp_dict, &_Py_ID(__classcell__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__classcell__)) < 0) { return -1; } return 0; @@ -3356,7 +3553,7 @@ type_new_init(type_new_ctx *ctx) goto error; } - type->tp_dict = dict; + set_tp_dict(type, dict); PyHeapTypeObject *et = (PyHeapTypeObject*)type; et->ht_slots = ctx->slots; @@ -3855,7 +4052,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, /* Set slots we have prepared */ type->tp_base = (PyTypeObject *)Py_NewRef(base); - type->tp_bases = bases; + set_tp_bases(type, bases); bases = NULL; // We give our reference to bases to the type type->tp_doc = tp_doc; @@ -3935,12 +4132,13 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, goto finally; } + PyObject *dict = lookup_tp_dict(type); if (type->tp_doc) { PyObject *__doc__ = PyUnicode_FromString(_PyType_DocWithoutSignature(type->tp_name, type->tp_doc)); if (!__doc__) { goto finally; } - r = PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), __doc__); + r = PyDict_SetItem(dict, &_Py_ID(__doc__), __doc__); Py_DECREF(__doc__); if (r < 0) { goto finally; @@ -3948,18 +4146,18 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, } if (weaklistoffset) { - if (PyDict_DelItem((PyObject *)type->tp_dict, &_Py_ID(__weaklistoffset__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__weaklistoffset__)) < 0) { goto finally; } } if (dictoffset) { - if (PyDict_DelItem((PyObject *)type->tp_dict, &_Py_ID(__dictoffset__)) < 0) { + if (PyDict_DelItem(dict, &_Py_ID(__dictoffset__)) < 0) { goto finally; } } /* Set type.__module__ */ - r = PyDict_Contains(type->tp_dict, &_Py_ID(__module__)); + r = PyDict_Contains(dict, &_Py_ID(__module__)); if (r < 0) { goto finally; } @@ -3971,7 +4169,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, if (modname == NULL) { goto finally; } - r = PyDict_SetItem(type->tp_dict, &_Py_ID(__module__), modname); + r = PyDict_SetItem(dict, &_Py_ID(__module__), modname); Py_DECREF(modname); if (r != 0) { goto finally; @@ -4093,7 +4291,7 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) { assert(PyType_Check(type)); - PyObject *mro = type->tp_mro; + PyObject *mro = lookup_tp_mro(type); // The type must be ready assert(mro != NULL); assert(PyTuple_Check(mro)); @@ -4142,14 +4340,14 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) } /* Look in tp_dict of types in MRO */ - PyObject *mro = type->tp_mro; + PyObject *mro = lookup_tp_mro(type); if (mro == NULL) { if ((type->tp_flags & Py_TPFLAGS_READYING) == 0) { if (PyType_Ready(type) < 0) { *error = -1; return NULL; } - mro = type->tp_mro; + mro = lookup_tp_mro(type); } if (mro == NULL) { *error = 1; @@ -4164,7 +4362,7 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 0; i < n; i++) { PyObject *base = PyTuple_GET_ITEM(mro, i); - PyObject *dict = _PyType_CAST(base)->tp_dict; + PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); assert(dict && PyDict_Check(dict)); res = _PyDict_GetItem_KnownHash(dict, name, hash); if (res != NULL) { @@ -4438,20 +4636,19 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys); static void type_dealloc_common(PyTypeObject *type) { - if (type->tp_bases != NULL) { + PyObject *bases = lookup_tp_bases(type); + if (bases != NULL) { PyObject *exc = PyErr_GetRaisedException(); - remove_all_subclasses(type, type->tp_bases); + remove_all_subclasses(type, bases); PyErr_SetRaisedException(exc); } } -static void clear_subclasses(PyTypeObject *self); - static void clear_static_tp_subclasses(PyTypeObject *type) { - PyObject *subclasses = lookup_subclasses(type); + PyObject *subclasses = lookup_tp_subclasses(type); if (subclasses == NULL) { return; } @@ -4480,7 +4677,7 @@ clear_static_tp_subclasses(PyTypeObject *type) Py_ssize_t i = 0; PyObject *key, *ref; // borrowed ref while (PyDict_Next(subclasses, &i, &key, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == NULL) { continue; } @@ -4488,36 +4685,40 @@ clear_static_tp_subclasses(PyTypeObject *type) assert(!(subclass->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); } - clear_subclasses(type); + clear_tp_subclasses(type); +} + +static void +clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) +{ + if (_Py_IsMainInterpreter(interp)) { + clear_tp_dict(type); + clear_tp_bases(type); + clear_tp_mro(type); + Py_CLEAR(type->tp_cache); + } + clear_static_tp_subclasses(type); } void -_PyStaticType_Dealloc(PyTypeObject *type) +_PyStaticType_Dealloc(PyInterpreterState *interp, PyTypeObject *type) { - assert(!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)); + assert(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); + assert(_Py_IsImmortal((PyObject *)type)); type_dealloc_common(type); - Py_CLEAR(type->tp_dict); - Py_CLEAR(type->tp_bases); - Py_CLEAR(type->tp_mro); - Py_CLEAR(type->tp_cache); - clear_static_tp_subclasses(type); + clear_static_type_objects(interp, type); - // PyObject_ClearWeakRefs() raises an exception if Py_REFCNT() != 0 - if (Py_REFCNT(type) == 0) { - PyObject_ClearWeakRefs((PyObject *)type); + if (_Py_IsMainInterpreter(interp)) { + type->tp_flags &= ~Py_TPFLAGS_READY; + type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; + type->tp_version_tag = 0; } - type->tp_flags &= ~Py_TPFLAGS_READY; - type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; - - if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - _PyStaticType_ClearWeakRefs(type); - static_builtin_state_clear(type); - /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ - } + _PyStaticType_ClearWeakRefs(interp, type); + static_builtin_state_clear(interp, type); + /* We leave _Py_TPFLAGS_STATIC_BUILTIN set on tp_flags. */ } @@ -4540,7 +4741,7 @@ type_dealloc(PyTypeObject *type) Py_XDECREF(type->tp_bases); Py_XDECREF(type->tp_mro); Py_XDECREF(type->tp_cache); - clear_subclasses(type); + clear_tp_subclasses(type); /* A type's tp_doc is heap allocated, unlike the tp_doc slots * of most other objects. It's okay to cast it to char *. @@ -4560,63 +4761,6 @@ type_dealloc(PyTypeObject *type) } -static PyObject * -lookup_subclasses(PyTypeObject *self) -{ - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - static_builtin_state *state = _PyStaticType_GetState(self); - assert(state != NULL); - return state->tp_subclasses; - } - return (PyObject *)self->tp_subclasses; -} - -int -_PyType_HasSubclasses(PyTypeObject *self) -{ - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN && - _PyStaticType_GetState(self) == NULL) { - return 0; - } - if (lookup_subclasses(self) == NULL) { - return 0; - } - return 1; -} - -PyObject* -_PyType_GetSubclasses(PyTypeObject *self) -{ - PyObject *list = PyList_New(0); - if (list == NULL) { - return NULL; - } - - PyObject *subclasses = lookup_subclasses(self); // borrowed ref - if (subclasses == NULL) { - return list; - } - assert(PyDict_CheckExact(subclasses)); - // The loop cannot modify tp_subclasses, there is no need - // to hold a strong reference (use a borrowed reference). - - Py_ssize_t i = 0; - PyObject *ref; // borrowed ref - while (PyDict_Next(subclasses, &i, NULL, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed - if (subclass == NULL) { - continue; - } - - if (PyList_Append(list, _PyObject_CAST(subclass)) < 0) { - Py_DECREF(list); - return NULL; - } - } - return list; -} - - /*[clinic input] type.__subclasses__ @@ -4830,8 +4974,9 @@ type_clear(PyTypeObject *type) */ PyType_Modified(type); - if (type->tp_dict) { - PyDict_Clear(type->tp_dict); + PyObject *dict = lookup_tp_dict(type); + if (dict) { + PyDict_Clear(dict); } Py_CLEAR(((PyHeapTypeObject *)type)->ht_module); @@ -5380,7 +5525,8 @@ _PyType_GetSlotNames(PyTypeObject *cls) assert(PyType_Check(cls)); /* Get the slot names from the cache in the class if possible. */ - slotnames = PyDict_GetItemWithError(cls->tp_dict, &_Py_ID(__slotnames__)); + PyObject *dict = lookup_tp_dict(cls); + slotnames = PyDict_GetItemWithError(dict, &_Py_ID(__slotnames__)); if (slotnames != NULL) { if (slotnames != Py_None && !PyList_Check(slotnames)) { PyErr_Format(PyExc_TypeError, @@ -5880,8 +6026,8 @@ object___reduce_ex___impl(PyObject *self, int protocol) PyObject *reduce, *res; if (objreduce == NULL) { - objreduce = PyDict_GetItemWithError( - PyBaseObject_Type.tp_dict, &_Py_ID(__reduce__)); + PyObject *dict = lookup_tp_dict(&PyBaseObject_Type); + objreduce = PyDict_GetItemWithError(dict, &_Py_ID(__reduce__)); if (objreduce == NULL && PyErr_Occurred()) { return NULL; } @@ -6147,11 +6293,12 @@ type_add_method(PyTypeObject *type, PyMethodDef *meth) } int err; + PyObject *dict = lookup_tp_dict(type); if (!(meth->ml_flags & METH_COEXIST)) { - err = PyDict_SetDefault(type->tp_dict, name, descr) == NULL; + err = PyDict_SetDefault(dict, name, descr) == NULL; } else { - err = PyDict_SetItem(type->tp_dict, name, descr) < 0; + err = PyDict_SetItem(dict, name, descr) < 0; } if (!isdescr) { Py_DECREF(name); @@ -6190,7 +6337,7 @@ type_add_members(PyTypeObject *type) return 0; } - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); for (; memb->name != NULL; memb++) { PyObject *descr = PyDescr_NewMember(type, memb); if (descr == NULL) @@ -6214,7 +6361,7 @@ type_add_getset(PyTypeObject *type) return 0; } - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); for (; gsp->name != NULL; gsp++) { PyObject *descr = PyDescr_NewGetSet(type, gsp); if (descr == NULL) { @@ -6293,7 +6440,7 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) static int overrides_hash(PyTypeObject *type) { - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); assert(dict != NULL); int r = PyDict_Contains(dict, &_Py_ID(__eq__)); @@ -6572,7 +6719,7 @@ type_ready_set_bases(PyTypeObject *type) } /* Initialize tp_bases */ - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); if (bases == NULL) { PyTypeObject *base = type->tp_base; if (base == NULL) { @@ -6584,7 +6731,7 @@ type_ready_set_bases(PyTypeObject *type) if (bases == NULL) { return -1; } - type->tp_bases = bases; + set_tp_bases(type, bases); } return 0; } @@ -6593,7 +6740,7 @@ type_ready_set_bases(PyTypeObject *type) static int type_ready_set_dict(PyTypeObject *type) { - if (type->tp_dict != NULL) { + if (lookup_tp_dict(type) != NULL) { return 0; } @@ -6601,7 +6748,7 @@ type_ready_set_dict(PyTypeObject *type) if (dict == NULL) { return -1; } - type->tp_dict = dict; + set_tp_dict(type, dict); return 0; } @@ -6611,7 +6758,8 @@ type_ready_set_dict(PyTypeObject *type) static int type_dict_set_doc(PyTypeObject *type) { - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__doc__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__doc__)); if (r < 0) { return -1; } @@ -6627,14 +6775,14 @@ type_dict_set_doc(PyTypeObject *type) return -1; } - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), doc) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__doc__), doc) < 0) { Py_DECREF(doc); return -1; } Py_DECREF(doc); } else { - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__doc__), Py_None) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__doc__), Py_None) < 0) { return -1; } } @@ -6699,14 +6847,14 @@ type_ready_mro(PyTypeObject *type) if (mro_internal(type, NULL) < 0) { return -1; } - assert(type->tp_mro != NULL); - assert(PyTuple_Check(type->tp_mro)); + PyObject *mro = lookup_tp_mro(type); + assert(mro != NULL); + assert(PyTuple_Check(mro)); /* All bases of statically allocated type should be statically allocated, and static builtin types must have static builtin bases. */ if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { assert(type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); - PyObject *mro = type->tp_mro; Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(mro, i)); @@ -6767,8 +6915,8 @@ type_ready_inherit(PyTypeObject *type) } // Inherit slots - PyObject *mro = type->tp_mro; - Py_ssize_t n = PyTuple_GET_SIZE(type->tp_mro); + PyObject *mro = lookup_tp_mro(type); + Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 1; i < n; i++) { PyObject *b = PyTuple_GET_ITEM(mro, i); if (PyType_Check(b)) { @@ -6813,7 +6961,8 @@ type_ready_set_hash(PyTypeObject *type) return 0; } - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__hash__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__hash__)); if (r < 0) { return -1; } @@ -6821,7 +6970,7 @@ type_ready_set_hash(PyTypeObject *type) return 0; } - if (PyDict_SetItem(type->tp_dict, &_Py_ID(__hash__), Py_None) < 0) { + if (PyDict_SetItem(dict, &_Py_ID(__hash__), Py_None) < 0) { return -1; } type->tp_hash = PyObject_HashNotImplemented; @@ -6833,7 +6982,7 @@ type_ready_set_hash(PyTypeObject *type) static int type_ready_add_subclasses(PyTypeObject *type) { - PyObject *bases = type->tp_bases; + PyObject *bases = lookup_tp_bases(type); Py_ssize_t nbase = PyTuple_GET_SIZE(bases); for (Py_ssize_t i = 0; i < nbase; i++) { PyObject *b = PyTuple_GET_ITEM(bases, i); @@ -6938,7 +7087,8 @@ type_ready_post_checks(PyTypeObject *type) else if (type->tp_dictoffset < (Py_ssize_t)sizeof(PyObject)) { if (type->tp_dictoffset + type->tp_basicsize <= 0) { PyErr_Format(PyExc_SystemError, - "type %s has a tp_dictoffset that is too small"); + "type %s has a tp_dictoffset that is too small", + type->tp_name); } } return 0; @@ -7029,16 +7179,32 @@ PyType_Ready(PyTypeObject *type) } int -_PyStaticType_InitBuiltin(PyTypeObject *self) +_PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) { + assert(_Py_IsImmortal((PyObject *)self)); assert(!(self->tp_flags & Py_TPFLAGS_HEAPTYPE)); + assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); + assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); +#ifndef NDEBUG + int ismain = _Py_IsMainInterpreter(interp); +#endif if (self->tp_flags & Py_TPFLAGS_READY) { + assert(!ismain); assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); + assert(self->tp_flags & Py_TPFLAGS_VALID_VERSION_TAG); + + static_builtin_state_init(interp, self); + + /* Per-interpreter tp_subclasses is done lazily. + Otherwise we would initialize it here. */ + assert(_PyType_CheckConsistency(self)); return 0; } + assert(ismain); + self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; @@ -7046,46 +7212,16 @@ _PyStaticType_InitBuiltin(PyTypeObject *self) self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; - static_builtin_state_init(self); + static_builtin_state_init(interp, self); int res = type_ready(self); if (res < 0) { - static_builtin_state_clear(self); + static_builtin_state_clear(interp, self); } return res; } -static PyObject * -init_subclasses(PyTypeObject *self) -{ - PyObject *subclasses = PyDict_New(); - if (subclasses == NULL) { - return NULL; - } - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - static_builtin_state *state = _PyStaticType_GetState(self); - state->tp_subclasses = subclasses; - return subclasses; - } - self->tp_subclasses = (void *)subclasses; - return subclasses; -} - -static void -clear_subclasses(PyTypeObject *self) -{ - /* Delete the dictionary to save memory. _PyStaticType_Dealloc() - callers also test if tp_subclasses is NULL to check if a static type - has no subclass. */ - if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { - static_builtin_state *state = _PyStaticType_GetState(self); - Py_CLEAR(state->tp_subclasses); - return; - } - Py_CLEAR(self->tp_subclasses); -} - static int add_subclass(PyTypeObject *base, PyTypeObject *type) { @@ -7102,9 +7238,9 @@ add_subclass(PyTypeObject *base, PyTypeObject *type) // Only get tp_subclasses after creating the key and value. // PyWeakref_NewRef() can trigger a garbage collection which can execute // arbitrary Python code and so modify base->tp_subclasses. - PyObject *subclasses = lookup_subclasses(base); + PyObject *subclasses = lookup_tp_subclasses(base); if (subclasses == NULL) { - subclasses = init_subclasses(base); + subclasses = init_tp_subclasses(base); if (subclasses == NULL) { Py_DECREF(key); Py_DECREF(ref); @@ -7135,19 +7271,6 @@ add_all_subclasses(PyTypeObject *type, PyObject *bases) return res; } -static inline PyTypeObject * -subclass_from_ref(PyObject *ref) -{ - assert(PyWeakref_CheckRef(ref)); - PyObject *obj = PyWeakref_GET_OBJECT(ref); // borrowed ref - assert(obj != NULL); - if (obj == Py_None) { - return NULL; - } - assert(PyType_Check(obj)); - return _PyType_CAST(obj); -} - static PyObject * get_subclasses_key(PyTypeObject *type, PyTypeObject *base) { @@ -7161,10 +7284,10 @@ get_subclasses_key(PyTypeObject *type, PyTypeObject *base) We fall back to manually traversing the values. */ Py_ssize_t i = 0; PyObject *ref; // borrowed ref - PyObject *subclasses = lookup_subclasses(base); + PyObject *subclasses = lookup_tp_subclasses(base); if (subclasses != NULL) { while (PyDict_Next(subclasses, &i, &key, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == type) { return Py_NewRef(key); } @@ -7177,7 +7300,7 @@ get_subclasses_key(PyTypeObject *type, PyTypeObject *base) static void remove_subclass(PyTypeObject *base, PyTypeObject *type) { - PyObject *subclasses = lookup_subclasses(base); // borrowed ref + PyObject *subclasses = lookup_tp_subclasses(base); // borrowed ref if (subclasses == NULL) { return; } @@ -7193,7 +7316,7 @@ remove_subclass(PyTypeObject *base, PyTypeObject *type) Py_XDECREF(key); if (PyDict_Size(subclasses) == 0) { - clear_subclasses(base); + clear_tp_subclasses(base); } } @@ -7491,7 +7614,7 @@ static int hackcheck(PyObject *self, setattrofunc func, const char *what) { PyTypeObject *type = Py_TYPE(self); - PyObject *mro = type->tp_mro; + PyObject *mro = lookup_tp_mro(type); if (!mro) { /* Probably ok not to check the call in this case. */ return 1; @@ -7779,7 +7902,8 @@ static struct PyMethodDef tp_new_methoddef[] = { static int add_tp_new_wrapper(PyTypeObject *type) { - int r = PyDict_Contains(type->tp_dict, &_Py_ID(__new__)); + PyObject *dict = lookup_tp_dict(type); + int r = PyDict_Contains(dict, &_Py_ID(__new__)); if (r > 0) { return 0; } @@ -7791,7 +7915,7 @@ add_tp_new_wrapper(PyTypeObject *type) if (func == NULL) { return -1; } - r = PyDict_SetItem(type->tp_dict, &_Py_ID(__new__), func); + r = PyDict_SetItem(dict, &_Py_ID(__new__), func); Py_DECREF(func); return r; } @@ -8306,17 +8430,23 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) if (getattribute == NULL || (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == - (void *)PyObject_GenericGetAttr)) - res = PyObject_GenericGetAttr(self, name); - else { + (void *)PyObject_GenericGetAttr)) { + res = _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); + /* if res == NULL with no exception set, then it must be an + AttributeError suppressed by us. */ + if (res == NULL && !PyErr_Occurred()) { + res = call_attribute(self, getattr, name); + } + } else { Py_INCREF(getattribute); res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); + if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + res = call_attribute(self, getattr, name); + } } - if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - res = call_attribute(self, getattr, name); - } + Py_DECREF(getattr); return res; } @@ -9149,7 +9279,8 @@ update_all_slots(PyTypeObject* type) static int type_new_set_names(PyTypeObject *type) { - PyObject *names_to_set = PyDict_Copy(type->tp_dict); + PyObject *dict = lookup_tp_dict(type); + PyObject *names_to_set = PyDict_Copy(dict); if (names_to_set == NULL) { return -1; } @@ -9238,7 +9369,7 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, // It is safe to use a borrowed reference because update_subclasses() is // only used with update_slots_callback() which doesn't modify // tp_subclasses. - PyObject *subclasses = lookup_subclasses(type); // borrowed ref + PyObject *subclasses = lookup_tp_subclasses(type); // borrowed ref if (subclasses == NULL) { return 0; } @@ -9247,13 +9378,13 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, Py_ssize_t i = 0; PyObject *ref; while (PyDict_Next(subclasses, &i, NULL, &ref)) { - PyTypeObject *subclass = subclass_from_ref(ref); // borrowed + PyTypeObject *subclass = type_from_ref(ref); // borrowed if (subclass == NULL) { continue; } /* Avoid recursing down into unaffected classes */ - PyObject *dict = subclass->tp_dict; + PyObject *dict = lookup_tp_dict(subclass); if (dict != NULL && PyDict_Check(dict)) { int r = PyDict_Contains(dict, attr_name); if (r < 0) { @@ -9304,7 +9435,7 @@ recurse_down_subclasses(PyTypeObject *type, PyObject *attr_name, static int add_operators(PyTypeObject *type) { - PyObject *dict = type->tp_dict; + PyObject *dict = lookup_tp_dict(type); pytype_slotdef *p; PyObject *descr; void **ptr; @@ -9400,7 +9531,7 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * PyObject *mro, *res; Py_ssize_t i, n; - mro = su_obj_type->tp_mro; + mro = lookup_tp_mro(su_obj_type); if (mro == NULL) return NULL; @@ -9421,7 +9552,7 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * Py_INCREF(mro); do { PyObject *obj = PyTuple_GET_ITEM(mro, i); - PyObject *dict = _PyType_CAST(obj)->tp_dict; + PyObject *dict = lookup_tp_dict(_PyType_CAST(obj)); assert(dict != NULL && PyDict_Check(dict)); res = PyDict_GetItemWithError(dict, name); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7537c12e92680c..6ae68cc20f7dee 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14573,13 +14573,13 @@ _PyUnicode_InitGlobalObjects(PyInterpreterState *interp) PyStatus _PyUnicode_InitTypes(PyInterpreterState *interp) { - if (_PyStaticType_InitBuiltin(&EncodingMapType) < 0) { + if (_PyStaticType_InitBuiltin(interp, &EncodingMapType) < 0) { goto error; } - if (_PyStaticType_InitBuiltin(&PyFieldNameIter_Type) < 0) { + if (_PyStaticType_InitBuiltin(interp, &PyFieldNameIter_Type) < 0) { goto error; } - if (_PyStaticType_InitBuiltin(&PyFormatterIter_Type) < 0) { + if (_PyStaticType_InitBuiltin(interp, &PyFormatterIter_Type) < 0) { goto error; } return _PyStatus_OK(); @@ -15158,13 +15158,9 @@ unicode_is_finalizing(void) void _PyUnicode_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStaticType_Dealloc(&EncodingMapType); - _PyStaticType_Dealloc(&PyFieldNameIter_Type); - _PyStaticType_Dealloc(&PyFormatterIter_Type); + _PyStaticType_Dealloc(interp, &EncodingMapType); + _PyStaticType_Dealloc(interp, &PyFieldNameIter_Type); + _PyStaticType_Dealloc(interp, &PyFormatterIter_Type); } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index c1afe63ecf66f6..aee79fc1410b29 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1017,9 +1017,9 @@ PyObject_ClearWeakRefs(PyObject *object) * or anything else. */ void -_PyStaticType_ClearWeakRefs(PyTypeObject *type) +_PyStaticType_ClearWeakRefs(PyInterpreterState *interp, PyTypeObject *type) { - static_builtin_state *state = _PyStaticType_GetState(type); + static_builtin_state *state = _PyStaticType_GetState(interp, type); PyObject **list = _PyStaticType_GET_WEAKREFS_LISTPTR(state); while (*list != NULL) { /* Note that clear_weakref() pops the first ref off the type's diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 439cd687fda61d..21941247eb9692 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -110,6 +110,7 @@ <ClCompile Include="..\Modules\_testcapi\exceptions.c" /> <ClCompile Include="..\Modules\_testcapi\code.c" /> <ClCompile Include="..\Modules\_testcapi\pyos.c" /> + <ClCompile Include="..\Modules\_testcapi\immortal.c" /> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc" /> diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 128241393f9f09..30ee873af9af24 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.1t -set libraries=%libraries% sqlite-3.40.1.0 +set libraries=%libraries% sqlite-3.41.2.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tix-8.4.3.6 diff --git a/PCbuild/python.props b/PCbuild/python.props index 7994fbe7cd5e0b..29add07795f900 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -68,7 +68,7 @@ <Import Project="$(ExternalProps)" Condition="$(ExternalProps) != '' and Exists('$(ExternalProps)')" /> <PropertyGroup> - <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.40.1.0\</sqlite3Dir> + <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.41.2.0\</sqlite3Dir> <bz2Dir Condition="$(bz2Dir) == ''">$(ExternalsDir)bzip2-1.0.8\</bz2Dir> <lzmaDir Condition="$(lzmaDir) == ''">$(ExternalsDir)xz-5.2.5\</lzmaDir> <libffiDir Condition="$(libffiDir) == ''">$(ExternalsDir)libffi-3.4.4\</libffiDir> diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 4c799b64c461c1..9df56685b76a87 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -188,7 +188,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.40.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.41.2, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 8de0572a1fc459..d2f9fee110ebf5 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -43,12 +43,12 @@ #ifdef Py_DEBUG static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) { assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index < MAXLEVEL); + assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL); return &(tok->tok_mode_stack[tok->tok_mode_stack_index]); } static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index < MAXLEVEL); + assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]); } #else @@ -2235,6 +2235,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t p_start = tok->start; p_end = tok->cur; + if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) { + return MAKE_TOKEN(syntaxerror(tok, "too many nested f-strings")); + } tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok); the_current_tok->kind = TOK_FSTRING_MODE; the_current_tok->f_string_quote = quote; @@ -2552,6 +2555,10 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct while (end_quote_size != current_tok->f_string_quote_size) { int c = tok_nextc(tok); if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) { + if (tok->decoding_erred) { + return MAKE_TOKEN(ERRORTOKEN); + } + assert(tok->multi_line_start != NULL); // shift the tok_state's location into // the start of string, and report the error diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 8b4213c4ce3b5a..5e2171885ac75b 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -10,8 +10,9 @@ extern "C" { #include "pycore_token.h" /* For token types */ -#define MAXINDENT 100 /* Max indentation level */ -#define MAXLEVEL 200 /* Max parentheses level */ +#define MAXINDENT 100 /* Max indentation level */ +#define MAXLEVEL 200 /* Max parentheses level */ +#define MAXFSTRINGLEVEL 150 /* Max f-string nesting level */ enum decoding_state { STATE_INIT, @@ -123,7 +124,7 @@ struct tok_state { enum interactive_underflow_t interactive_underflow; int report_warnings; // TODO: Factor this into its own thing - tokenizer_mode tok_mode_stack[MAXLEVEL]; + tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL]; int tok_mode_stack_index; int tok_report_warnings; #ifdef Py_DEBUG diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9de0d92e382d3d..e83894e8902872 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3103,6 +3103,25 @@ dummy_func( } } else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); + } result = PyObject_Call(func, callargs, kwargs); } DECREF_INPUTS(); diff --git a/Python/ceval.c b/Python/ceval.c index 5d5221b2e40990..958689debc87f8 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -212,6 +212,9 @@ static _PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, PyObject* const* args, size_t argcount, PyObject *kwnames); +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); static void _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); @@ -1501,6 +1504,49 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, return NULL; } +/* Same as _PyEvalFramePushAndInit but takes an args tuple and kwargs dict. + Steals references to func, callargs and kwargs. +*/ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs) +{ + bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); + PyObject *kwnames = NULL; + PyObject *const *newargs; + if (has_dict) { + newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); + if (newargs == NULL) { + Py_DECREF(func); + goto error; + } + } + else { + newargs = &PyTuple_GET_ITEM(callargs, 0); + /* We need to incref all our args since the new frame steals the references. */ + for (Py_ssize_t i = 0; i < nargs; ++i) { + Py_INCREF(PyTuple_GET_ITEM(callargs, i)); + } + } + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)func, locals, + newargs, nargs, kwnames + ); + if (has_dict) { + _PyStack_UnpackDict_FreeNoDecRef(newargs, kwnames); + } + /* No need to decref func here because the reference has been stolen by + _PyEvalFramePushAndInit. + */ + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return new_frame; +error: + Py_DECREF(callargs); + Py_XDECREF(kwargs); + return NULL; +} + PyObject * _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, diff --git a/Python/compile.c b/Python/compile.c index e8789def867308..cbe5403aafbc87 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -308,7 +308,6 @@ instr_sequence_fini(instr_sequence *seq) { seq->s_instrs = NULL; } - static int instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { memset(g, 0, sizeof(cfg_builder)); @@ -6754,11 +6753,11 @@ _PyCompile_ConstCacheMergeOne(PyObject *const_cache, PyObject **obj) static int * -build_cellfixedoffsets(struct compiler_unit *u) +build_cellfixedoffsets(_PyCompile_CodeUnitMetadata *umd) { - int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(u->u_metadata.u_freevars); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); int noffsets = ncellvars + nfreevars; int *fixed = PyMem_New(int, noffsets); @@ -6772,8 +6771,8 @@ build_cellfixedoffsets(struct compiler_unit *u) PyObject *varname, *cellindex; Py_ssize_t pos = 0; - while (PyDict_Next(u->u_metadata.u_cellvars, &pos, &varname, &cellindex)) { - PyObject *varindex = PyDict_GetItem(u->u_metadata.u_varnames, varname); + while (PyDict_Next(umd->u_cellvars, &pos, &varname, &cellindex)) { + PyObject *varindex = PyDict_GetItem(umd->u_varnames, varname); if (varindex != NULL) { assert(PyLong_AS_LONG(cellindex) < INT_MAX); assert(PyLong_AS_LONG(varindex) < INT_MAX); @@ -6787,17 +6786,17 @@ build_cellfixedoffsets(struct compiler_unit *u) } static int -insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, +insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixed, int nfreevars, int code_flags) { - assert(u->u_metadata.u_firstlineno > 0); + assert(umd->u_firstlineno > 0); /* Add the generator prefix instructions. */ if (code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { cfg_instr make_gen = { .i_opcode = RETURN_GENERATOR, .i_oparg = 0, - .i_loc = LOCATION(u->u_metadata.u_firstlineno, u->u_metadata.u_firstlineno, -1, -1), + .i_loc = LOCATION(umd->u_firstlineno, umd->u_firstlineno, -1, -1), .i_target = NULL, }; RETURN_IF_ERROR(_PyBasicblock_InsertInstruction(entryblock, 0, &make_gen)); @@ -6811,12 +6810,12 @@ insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, } /* Set up cells for any variable that escapes, to be put in a closure. */ - const int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); + const int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); if (ncellvars) { - // u->u_metadata.u_cellvars has the cells out of order so we sort them + // umd->u_cellvars has the cells out of order so we sort them // before adding the MAKE_CELL instructions. Note that we // adjust for arg cells, which come first. - const int nvars = ncellvars + (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); + const int nvars = ncellvars + (int)PyDict_GET_SIZE(umd->u_varnames); int *sorted = PyMem_RawCalloc(nvars, sizeof(int)); if (sorted == NULL) { PyErr_NoMemory(); @@ -6860,11 +6859,11 @@ insert_prefix_instructions(struct compiler_unit *u, basicblock *entryblock, } static int -fix_cell_offsets(struct compiler_unit *u, basicblock *entryblock, int *fixedmap) +fix_cell_offsets(_PyCompile_CodeUnitMetadata *umd, basicblock *entryblock, int *fixedmap) { - int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(u->u_metadata.u_freevars); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); int noffsets = ncellvars + nfreevars; // First deal with duplicates (arg cells). @@ -6906,30 +6905,30 @@ fix_cell_offsets(struct compiler_unit *u, basicblock *entryblock, int *fixedmap) static int -prepare_localsplus(struct compiler_unit* u, cfg_builder *g, int code_flags) +prepare_localsplus(_PyCompile_CodeUnitMetadata *umd, cfg_builder *g, int code_flags) { - assert(PyDict_GET_SIZE(u->u_metadata.u_varnames) < INT_MAX); - assert(PyDict_GET_SIZE(u->u_metadata.u_cellvars) < INT_MAX); - assert(PyDict_GET_SIZE(u->u_metadata.u_freevars) < INT_MAX); - int nlocals = (int)PyDict_GET_SIZE(u->u_metadata.u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(u->u_metadata.u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(u->u_metadata.u_freevars); + assert(PyDict_GET_SIZE(umd->u_varnames) < INT_MAX); + assert(PyDict_GET_SIZE(umd->u_cellvars) < INT_MAX); + assert(PyDict_GET_SIZE(umd->u_freevars) < INT_MAX); + int nlocals = (int)PyDict_GET_SIZE(umd->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(umd->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(umd->u_freevars); assert(INT_MAX - nlocals - ncellvars > 0); assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); int nlocalsplus = nlocals + ncellvars + nfreevars; - int* cellfixedoffsets = build_cellfixedoffsets(u); + int* cellfixedoffsets = build_cellfixedoffsets(umd); if (cellfixedoffsets == NULL) { return ERROR; } // This must be called before fix_cell_offsets(). - if (insert_prefix_instructions(u, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { + if (insert_prefix_instructions(umd, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { PyMem_Free(cellfixedoffsets); return ERROR; } - int numdropped = fix_cell_offsets(u, g->g_entryblock, cellfixedoffsets); + int numdropped = fix_cell_offsets(umd, g->g_entryblock, cellfixedoffsets); PyMem_Free(cellfixedoffsets); // At this point we're done with it. cellfixedoffsets = NULL; if (numdropped < 0) { @@ -6980,7 +6979,7 @@ optimize_and_assemble_code_unit(struct compiler_unit *u, PyObject *const_cache, } /** Assembly **/ - int nlocalsplus = prepare_localsplus(u, &g, code_flags); + int nlocalsplus = prepare_localsplus(&u->u_metadata, &g, code_flags); if (nlocalsplus < 0) { goto error; } @@ -7157,11 +7156,6 @@ instructions_to_instr_sequence(PyObject *instructions, instr_sequence *seq) goto error; } } - if (seq->s_used && !IS_TERMINATOR_OPCODE(seq->s_instrs[seq->s_used-1].i_opcode)) { - if (instr_sequence_addop(seq, RETURN_VALUE, 0, NO_LOCATION) < 0) { - goto error; - } - } PyMem_Free(is_target); return SUCCESS; error: @@ -7328,6 +7322,67 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) return res; } +int _PyCfg_JumpLabelsToTargets(basicblock *entryblock); + +PyCodeObject * +_PyCompile_Assemble(_PyCompile_CodeUnitMetadata *umd, PyObject *filename, + PyObject *instructions) +{ + PyCodeObject *co = NULL; + instr_sequence optimized_instrs; + memset(&optimized_instrs, 0, sizeof(instr_sequence)); + + PyObject *const_cache = PyDict_New(); + if (const_cache == NULL) { + return NULL; + } + + cfg_builder g; + if (instructions_to_cfg(instructions, &g) < 0) { + goto error; + } + + if (_PyCfg_JumpLabelsToTargets(g.g_entryblock) < 0) { + goto error; + } + + int code_flags = 0; + int nlocalsplus = prepare_localsplus(umd, &g, code_flags); + if (nlocalsplus < 0) { + goto error; + } + + int maxdepth = _PyCfg_Stackdepth(g.g_entryblock, code_flags); + if (maxdepth < 0) { + goto error; + } + + _PyCfg_ConvertExceptionHandlersToNops(g.g_entryblock); + + /* Order of basic blocks must have been determined by now */ + + if (_PyCfg_ResolveJumps(&g) < 0) { + goto error; + } + + /* Can't modify the bytecode after computing jump offsets. */ + + if (cfg_to_instr_sequence(&g, &optimized_instrs) < 0) { + goto error; + } + + PyObject *consts = umd->u_consts; + co = _PyAssemble_MakeCodeObject(umd, const_cache, + consts, maxdepth, &optimized_instrs, + nlocalsplus, code_flags, filename); + +error: + Py_DECREF(const_cache); + _PyCfgBuilder_Fini(&g); + instr_sequence_fini(&optimized_instrs); + return co; +} + /* Retained for API compatibility. * Optimization is now done in _PyCfg_OptimizeCodeUnit */ diff --git a/Python/context.c b/Python/context.c index 5d385508405ede..1ffae9871be7b3 100644 --- a/Python/context.c +++ b/Python/context.c @@ -1309,7 +1309,7 @@ _PyContext_Init(PyInterpreterState *interp) PyObject *missing = get_token_missing(); if (PyDict_SetItemString( - PyContextToken_Type.tp_dict, "MISSING", missing)) + _PyType_GetDict(&PyContextToken_Type), "MISSING", missing)) { Py_DECREF(missing); return _PyStatus_ERR("can't init context types"); diff --git a/Python/errors.c b/Python/errors.c index 7fc267385c569b..a8000ac94918db 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -6,7 +6,7 @@ #include "pycore_initconfig.h" // _PyStatus_ERR() #include "pycore_pyerrors.h" // _PyErr_Format() #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_structseq.h" // _PyStructSequence_FiniType() +#include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include "pycore_sysmodule.h" // _PySys_Audit() #include "pycore_traceback.h" // _PyTraceBack_FromFrame() @@ -1342,8 +1342,9 @@ static PyStructSequence_Desc UnraisableHookArgs_desc = { PyStatus _PyErr_InitTypes(PyInterpreterState *interp) { - if (_PyStructSequence_InitBuiltin(&UnraisableHookArgsType, - &UnraisableHookArgs_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &UnraisableHookArgsType, + &UnraisableHookArgs_desc) < 0) + { return _PyStatus_ERR("failed to initialize UnraisableHookArgs type"); } return _PyStatus_OK(); @@ -1353,11 +1354,7 @@ _PyErr_InitTypes(PyInterpreterState *interp) void _PyErr_FiniTypes(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStructSequence_FiniType(&UnraisableHookArgsType); + _PyStructSequence_FiniBuiltin(interp, &UnraisableHookArgsType); } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 6f83a910cab392..f79afb4c66cd8b 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -223,6 +223,15 @@ dump_basicblock(const basicblock *b) } } } + +void +_PyCfgBuilder_DumpGraph(const basicblock *entryblock) +{ + for (const basicblock *b = entryblock; b != NULL; b = b->b_next) { + dump_basicblock(b); + } +} + #endif @@ -592,6 +601,11 @@ translate_jump_labels_to_targets(basicblock *entryblock) return SUCCESS; } +int +_PyCfg_JumpLabelsToTargets(basicblock *entryblock) +{ + return translate_jump_labels_to_targets(entryblock); +} static int mark_except_handlers(basicblock *entryblock) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 864a4f7bcaff0f..069a7ced0a4c25 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4296,16 +4296,35 @@ } } else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + assert(PyTuple_CheckExact(callargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, + (PyFunctionObject *)func, locals, + nargs, callargs, kwargs); + // Need to manually shrink the stack since we exit with DISPATCH_INLINED. + STACK_SHRINK(oparg + 3); + if (new_frame == NULL) { + goto error; + } + frame->return_offset = 0; + DISPATCH_INLINED(new_frame); + } result = PyObject_Call(func, callargs, kwargs); } - #line 4302 "Python/generated_cases.c.h" + #line 4321 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 3109 "Python/bytecodes.c" + #line 3128 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4309 "Python/generated_cases.c.h" + #line 4328 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4320,7 +4339,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 3119 "Python/bytecodes.c" + #line 3138 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4349,14 +4368,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4353 "Python/generated_cases.c.h" + #line 4372 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 3150 "Python/bytecodes.c" + #line 3169 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4377,7 +4396,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4381 "Python/generated_cases.c.h" + #line 4400 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4385,15 +4404,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 3173 "Python/bytecodes.c" + #line 3192 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4391 "Python/generated_cases.c.h" + #line 4410 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 3175 "Python/bytecodes.c" + #line 3194 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4397 "Python/generated_cases.c.h" + #line 4416 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4404,7 +4423,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 3179 "Python/bytecodes.c" + #line 3198 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4439,7 +4458,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4443 "Python/generated_cases.c.h" + #line 4462 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4448,10 +4467,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3216 "Python/bytecodes.c" + #line 3235 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4455 "Python/generated_cases.c.h" + #line 4474 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4463,7 +4482,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3221 "Python/bytecodes.c" + #line 3240 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4478,12 +4497,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4482 "Python/generated_cases.c.h" + #line 4501 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3236 "Python/bytecodes.c" + #line 3255 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4487 "Python/generated_cases.c.h" + #line 4506 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4493,16 +4512,16 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3241 "Python/bytecodes.c" + #line 3260 "Python/bytecodes.c" assert(oparg >= 2); - #line 4499 "Python/generated_cases.c.h" + #line 4518 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(INSTRUMENTED_LINE) { - #line 3245 "Python/bytecodes.c" + #line 3264 "Python/bytecodes.c" _Py_CODEUNIT *here = next_instr-1; _PyFrame_SetStackPointer(frame, stack_pointer); int original_opcode = _Py_call_instrumentation_line( @@ -4522,11 +4541,11 @@ } opcode = original_opcode; DISPATCH_GOTO(); - #line 4526 "Python/generated_cases.c.h" + #line 4545 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_INSTRUCTION) { - #line 3267 "Python/bytecodes.c" + #line 3286 "Python/bytecodes.c" int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, next_instr-1); if (next_opcode < 0) goto error; @@ -4538,26 +4557,26 @@ assert(next_opcode > 0 && next_opcode < 256); opcode = next_opcode; DISPATCH_GOTO(); - #line 4542 "Python/generated_cases.c.h" + #line 4561 "Python/generated_cases.c.h" } TARGET(INSTRUMENTED_JUMP_FORWARD) { - #line 3281 "Python/bytecodes.c" + #line 3300 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP); - #line 4548 "Python/generated_cases.c.h" + #line 4567 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_JUMP_BACKWARD) { - #line 3285 "Python/bytecodes.c" + #line 3304 "Python/bytecodes.c" INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP); - #line 4555 "Python/generated_cases.c.h" + #line 4574 "Python/generated_cases.c.h" CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - #line 3290 "Python/bytecodes.c" + #line 3309 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4566,12 +4585,12 @@ assert(err == 0 || err == 1); int offset = err*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4570 "Python/generated_cases.c.h" + #line 4589 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - #line 3301 "Python/bytecodes.c" + #line 3320 "Python/bytecodes.c" PyObject *cond = POP(); int err = PyObject_IsTrue(cond); Py_DECREF(cond); @@ -4580,12 +4599,12 @@ assert(err == 0 || err == 1); int offset = (1-err)*oparg; INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4584 "Python/generated_cases.c.h" + #line 4603 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - #line 3312 "Python/bytecodes.c" + #line 3331 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4598,12 +4617,12 @@ offset = 0; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4602 "Python/generated_cases.c.h" + #line 4621 "Python/generated_cases.c.h" DISPATCH(); } TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - #line 3327 "Python/bytecodes.c" + #line 3346 "Python/bytecodes.c" PyObject *value = POP(); _Py_CODEUNIT *here = next_instr-1; int offset; @@ -4616,30 +4635,30 @@ offset = oparg; } INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH); - #line 4620 "Python/generated_cases.c.h" + #line 4639 "Python/generated_cases.c.h" DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3342 "Python/bytecodes.c" + #line 3361 "Python/bytecodes.c" assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4631 "Python/generated_cases.c.h" + #line 4650 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3350 "Python/bytecodes.c" + #line 3369 "Python/bytecodes.c" assert(0 && "Executing a cache."); Py_UNREACHABLE(); - #line 4638 "Python/generated_cases.c.h" + #line 4657 "Python/generated_cases.c.h" } TARGET(RESERVED) { - #line 3355 "Python/bytecodes.c" + #line 3374 "Python/bytecodes.c" assert(0 && "Executing RESERVED instruction."); Py_UNREACHABLE(); - #line 4645 "Python/generated_cases.c.h" + #line 4664 "Python/generated_cases.c.h" } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ba248d208e425a..b8a115236900b9 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1663,8 +1663,10 @@ flush_std_files(void) static void finalize_interp_types(PyInterpreterState *interp) { + _PyIO_FiniTypes(interp); + _PyUnicode_FiniTypes(interp); - _PySys_Fini(interp); + _PySys_FiniTypes(interp); _PyExc_Fini(interp); _PyAsyncGen_Fini(interp); _PyContext_Fini(interp); @@ -1706,8 +1708,6 @@ finalize_interp_clear(PyThreadState *tstate) /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); - _PyIO_FiniTypes(tstate->interp); - /* Clear all loghooks */ /* Both _PySys_Audit function and users still need PyObject, such as tuple. Call _PySys_ClearAuditHooks when PyObject available. */ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b16d3f53f89fb9..05e7b4370869af 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1107,7 +1107,7 @@ print_exception_notes(struct exception_print_context *ctx, PyObject *value) if (notes == NULL) { return -1; } - if (!PySequence_Check(notes)) { + if (!PySequence_Check(notes) || PyUnicode_Check(notes) || PyBytes_Check(notes)) { int res = 0; if (write_indented_margin(ctx, f) < 0) { res = -1; @@ -1122,6 +1122,9 @@ print_exception_notes(struct exception_print_context *ctx, PyObject *value) Py_DECREF(s); } Py_DECREF(notes); + if (PyFile_WriteString("\n", f) < 0) { + res = -1; + } return res; } Py_ssize_t num_notes = PySequence_Length(notes); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d673e40af5e1de..781588b0df4ead 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3141,6 +3141,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) { PyObject *version_info; int res; + PyInterpreterState *interp = tstate->interp; /* stdin/stdout/stderr are set in pylifecycle.c */ @@ -3166,7 +3167,9 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("float_info", PyFloat_GetInfo()); SET_SYS("int_info", PyLong_GetInfo()); /* initialize hash_info */ - if (_PyStructSequence_InitBuiltin(&Hash_InfoType, &hash_info_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &Hash_InfoType, + &hash_info_desc) < 0) + { goto type_init_failed; } SET_SYS("hash_info", get_hash_info(tstate)); @@ -3190,7 +3193,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) #define ENSURE_INFO_TYPE(TYPE, DESC) \ do { \ if (_PyStructSequence_InitBuiltinWithFlags( \ - &TYPE, &DESC, Py_TPFLAGS_DISALLOW_INSTANTIATION) < 0) { \ + interp, &TYPE, &DESC, Py_TPFLAGS_DISALLOW_INSTANTIATION) < 0) { \ goto type_init_failed; \ } \ } while (0) @@ -3226,8 +3229,9 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("thread_info", PyThread_GetInfo()); /* initialize asyncgen_hooks */ - if (_PyStructSequence_InitBuiltin( - &AsyncGenHooksType, &asyncgen_hooks_desc) < 0) { + if (_PyStructSequence_InitBuiltin(interp, &AsyncGenHooksType, + &asyncgen_hooks_desc) < 0) + { goto type_init_failed; } @@ -3489,20 +3493,20 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) void -_PySys_Fini(PyInterpreterState *interp) +_PySys_FiniTypes(PyInterpreterState *interp) { - if (_Py_IsMainInterpreter(interp)) { - _PyStructSequence_FiniType(&VersionInfoType); - _PyStructSequence_FiniType(&FlagsType); + _PyStructSequence_FiniBuiltin(interp, &VersionInfoType); + _PyStructSequence_FiniBuiltin(interp, &FlagsType); #if defined(MS_WINDOWS) - _PyStructSequence_FiniType(&WindowsVersionType); + _PyStructSequence_FiniBuiltin(interp, &WindowsVersionType); #endif - _PyStructSequence_FiniType(&Hash_InfoType); - _PyStructSequence_FiniType(&AsyncGenHooksType); + _PyStructSequence_FiniBuiltin(interp, &Hash_InfoType); + _PyStructSequence_FiniBuiltin(interp, &AsyncGenHooksType); #ifdef __EMSCRIPTEN__ + if (_Py_IsMainInterpreter(interp)) { Py_CLEAR(EmscriptenInfoType); -#endif } +#endif } diff --git a/Python/thread.c b/Python/thread.c index 7fdedb0b9b7e26..7fc53f9b61360b 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -7,7 +7,7 @@ #include "Python.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_structseq.h" // _PyStructSequence_FiniType() +#include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() #include "pycore_pythread.h" #ifndef DONT_HAVE_STDIO_H @@ -137,7 +137,8 @@ PyThread_GetInfo(void) int len; #endif - if (_PyStructSequence_InitBuiltin(&ThreadInfoType, &threadinfo_desc) < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyStructSequence_InitBuiltin(interp, &ThreadInfoType, &threadinfo_desc) < 0) { return NULL; } @@ -191,9 +192,5 @@ PyThread_GetInfo(void) void _PyThread_FiniType(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return; - } - - _PyStructSequence_FiniType(&ThreadInfoType); + _PyStructSequence_FiniBuiltin(interp, &ThreadInfoType); } diff --git a/netlify.toml b/netlify.toml deleted file mode 100644 index f5790fc5fec74f..00000000000000 --- a/netlify.toml +++ /dev/null @@ -1,11 +0,0 @@ -[build] - base = "Doc/" - command = "make html" - publish = "build/html" - # Do not trigger netlify builds if docs were not changed. - # Changed files should be in sync with `.github/workflows/doc.yml` - ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF . ../netlify.toml" - -[build.environment] - PYTHON_VERSION = "3.8" - IS_DEPLOYMENT_PREVIEW = "true"