diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eec11e25a7c7f6..2241b0b8aa409e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -162,13 +162,11 @@ jobs: - uses: actions/checkout@v3 - name: Install Homebrew dependencies run: brew install pkg-config openssl@1.1 xz gdbm tcl-tk - - name: Prepare Homebrew environment variables - run: | - echo "CFLAGS=-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" >> $GITHUB_ENV - echo "LDFLAGS=-L$(brew --prefix gdbm)/lib -I$(brew --prefix xz)/lib" >> $GITHUB_ENV - echo "PKG_CONFIG_PATH=$(brew --prefix openssl@1.1)/lib/pkgconfig:$(brew --prefix tcl-tk)/lib/pkgconfig" >> $GITHUB_ENV - name: Configure CPython run: | + CFLAGS="-I$(brew --prefix gdbm)/include -I$(brew --prefix xz)/include" \ + LDFLAGS="-L$(brew --prefix gdbm)/lib -I$(brew --prefix xz)/lib" \ + PKG_CONFIG_PATH="$(brew --prefix tcl-tk)/lib/pkgconfig" \ ./configure \ --with-pydebug \ --prefix=/opt/python-dev \ diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index b2b63472d83421..b2a76ef7d36153 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -19,13 +19,13 @@ jobs: - name: Send notification uses: actions/github-script@v6 env: - MAILGUN_API_KEY: ${{ secrets.PSF_MAILGUN_KEY }} + MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: script: | const Mailgun = require("mailgun.js"); const formData = require('form-data'); const mailgun = new Mailgun(formData); - const DOMAIN = "mg.python.org"; + const DOMAIN = "mailgun.python.org"; const mg = mailgun.client({username: 'api', key: process.env.MAILGUN_API_KEY}); github.rest.issues.get({ issue_number: context.issue.number, @@ -44,7 +44,7 @@ jobs: }; const data = { - from: "CPython Issues ", + from: "CPython Issues ", to: "new-bugs-announce@python.org", subject: `[Issue ${issue.data.number}] ${issue.data.title}`, template: "new-github-issue", diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 4f30ef19ee4d8a..d98192b369603e 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -70,7 +70,7 @@ Click on the "New issue" button in the top bar to report a new issue. The submission form has two fields, "Title" and "Comment". For the "Title" field, enter a *very* short description of the problem; -less than ten words is good. +fewer than ten words is good. In the "Comment" field, describe the problem in detail, including what you expected to happen and what did happen. Be sure to include whether any diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index ae75d68901d7ba..062ef3a1fea93c 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -33,28 +33,47 @@ bound into a function. Return the number of free variables in *co*. -.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable) +.. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable) Return a new code object. If you need a dummy code object to create a frame, - use :c:func:`PyCode_NewEmpty` instead. Calling :c:func:`PyCode_New` directly - will bind you to a precise Python version since the definition of the bytecode - changes often. The many arguments of this function are inter-dependent in complex + use :c:func:`PyCode_NewEmpty` instead. + + Since the definition of the bytecode changes often, calling + :c:func:`PyCode_New` directly can bind you to a precise Python version. + + The many arguments of this function are inter-dependent in complex ways, meaning that subtle changes to values are likely to result in incorrect execution or VM crashes. Use this function only with extreme care. .. versionchanged:: 3.11 Added ``exceptiontable`` parameter. -.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable) + .. index:: single: PyCode_New + + .. versionchanged:: 3.12 + + Renamed from ``PyCode_New`` as part of :ref:`unstable-c-api`. + The old name is deprecated, but will remain available until the + signature changes again. + +.. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *linetable, PyObject *exceptiontable) Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positional-only arguments. The same caveats that apply to ``PyCode_New`` also apply to this function. - .. versionadded:: 3.8 + .. index:: single: PyCode_NewWithPosOnlyArgs + + .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` .. versionchanged:: 3.11 Added ``exceptiontable`` parameter. + .. versionchanged:: 3.12 + + Renamed to ``PyUnstable_Code_NewWithPosOnlyArgs``. + The old name is deprecated, but will remain available until the + signature changes again. + .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) Return a new empty code object with the specified filename, @@ -165,3 +184,70 @@ bound into a function. :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. .. versionadded:: 3.12 + + +Extra information +----------------- + +To support low-level extensions to frame evaluation, such as external +just-in-time compilers, it is possible to attach arbitrary extra data to +code objects. + +These functions are part of the unstable C API tier: +this functionality is a CPython implementation detail, and the API +may change without deprecation warnings. + +.. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) + + Return a new an opaque index value used to adding data to code objects. + + You generally call this function once (per interpreter) and use the result + with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate + data on individual code objects. + + If *free* is not ``NULL``: when a code object is deallocated, + *free* will be called on non-``NULL`` data stored under the new index. + Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. + + .. index:: single: _PyEval_RequestCodeExtraIndex + + .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` + + .. versionchanged:: 3.12 + + Renamed to ``PyUnstable_Eval_RequestCodeExtraIndex``. + The old private name is deprecated, but will be available until the API + changes. + +.. c:function:: int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) + + Set *extra* to the extra data stored under the given index. + Return 0 on success. Set an exception and return -1 on failure. + + If no data was set under the index, set *extra* to ``NULL`` and return + 0 without setting an exception. + + .. index:: single: _PyCode_GetExtra + + .. versionadded:: 3.6 as ``_PyCode_GetExtra`` + + .. versionchanged:: 3.12 + + Renamed to ``PyUnstable_Code_GetExtra``. + The old private name is deprecated, but will be available until the API + changes. + +.. c:function:: int PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra) + + Set the extra data stored under the given index to *extra*. + Return 0 on success. Set an exception and return -1 on failure. + + .. index:: single: _PyCode_SetExtra + + .. versionadded:: 3.6 as ``_PyCode_SetExtra`` + + .. versionchanged:: 3.12 + + Renamed to ``PyUnstable_Code_SetExtra``. + The old private name is deprecated, but will be available until the API + changes. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index e735f8db402363..8836c973f1f579 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -438,7 +438,9 @@ Querying the error indicator .. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - As of 3.12, this function is deprecated. Use :c:func:`PyErr_GetRaisedException` instead. + .. deprecated:: 3.12 + + Use :c:func:`PyErr_GetRaisedException` instead. Retrieve the error indicator into three variables whose addresses are passed. If the error indicator is not set, set all three variables to ``NULL``. If it is @@ -447,8 +449,10 @@ Querying the error indicator .. note:: - This function is normally only used by code that needs to catch exceptions or - by code that needs to save and restore the error indicator temporarily, e.g.:: + This function is normally only used by legacy code that needs to catch + exceptions or save and restore the error indicator temporarily. + + For example:: { PyObject *type, *value, *traceback; @@ -459,15 +463,17 @@ Querying the error indicator PyErr_Restore(type, value, traceback); } - .. deprecated:: 3.12 - .. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) - As of 3.12, this function is deprecated. Use :c:func:`PyErr_SetRaisedException` instead. + .. deprecated:: 3.12 + + Use :c:func:`PyErr_SetRaisedException` instead. - Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are ``NULL``, the error + Set the error indicator from the three objects, + *type*, *value*, and *traceback*, + clearing the existing exception if one is set. + If the objects are ``NULL``, the error indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or traceback. The exception type should be a class. Do not pass an invalid exception type or value. (Violating these rules will cause subtle problems @@ -478,18 +484,17 @@ Querying the error indicator .. note:: - This function is normally only used by code that needs to save and restore the - error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current - error indicator. - - .. deprecated:: 3.12 + This function is normally only used by legacy code that needs to + save and restore the error indicator temporarily. + Use :c:func:`PyErr_Fetch` to save the current error indicator. .. c:function:: void PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb) - As of 3.12, this function is deprecated. - Use :c:func:`PyErr_GetRaisedException` instead of :c:func:`PyErr_Fetch` to avoid - any possible de-normalization. + .. deprecated:: 3.12 + + Use :c:func:`PyErr_GetRaisedException` instead, + to avoid any possible de-normalization. Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is @@ -507,8 +512,6 @@ Querying the error indicator PyException_SetTraceback(val, tb); } - .. deprecated:: 3.12 - .. c:function:: PyObject* PyErr_GetHandledException(void) @@ -756,14 +759,12 @@ Exception Objects .. c:function:: PyObject* PyException_GetArgs(PyObject *ex) - Return args of the given exception as a new reference, - as accessible from Python through :attr:`args`. + Return :attr:`~BaseException.args` of exception *ex*. .. c:function:: void PyException_SetArgs(PyObject *ex, PyObject *args) - Set the args of the given exception, - as accessible from Python through :attr:`args`. + Set :attr:`~BaseException.args` of exception *ex* to *args*. .. _unicodeexceptions: diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index ad06616eeb0e63..b50ee3b3803e29 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -818,7 +818,7 @@ Process-wide parameters .. deprecated:: 3.11 -.. c:function:: w_char* Py_GetPythonHome() +.. c:function:: wchar_t* Py_GetPythonHome() Return the default "home", that is, the value set by a previous call to :c:func:`Py_SetPythonHome`, or the value of the :envvar:`PYTHONHOME` diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 4ae20e93e36785..3721fc0697f5cd 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -6,9 +6,9 @@ C API Stability *************** -Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`. -While the C API will change with every minor release (e.g. from 3.9 to 3.10), -most changes will be source-compatible, typically by only adding new API. +Unless documented otherwise, Python's C API is covered by the Backwards +Compatibility Policy, :pep:`387`. +Most changes to it are source-compatible (typically by only adding new API). Changing existing API or removing API is only done after a deprecation period or to fix serious issues. @@ -18,8 +18,38 @@ way; see :ref:`stable-abi-platform` below). So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa, but will need to be compiled separately for 3.9.x and 3.10.x. +There are two tiers of C API with different stability exepectations: + +- *Unstable API*, may change in minor versions without a deprecation period. + It is marked by the ``PyUnstable`` prefix in names. +- *Limited API*, is compatible across several minor releases. + When :c:macro:`Py_LIMITED_API` is defined, only this subset is exposed + from ``Python.h``. + +These are discussed in more detail below. + Names prefixed by an underscore, such as ``_Py_InternalState``, are private API that can change without notice even in patch releases. +If you need to use this API, consider reaching out to +`CPython developers `_ +to discuss adding public API for your use case. + +.. _unstable-c-api: + +Unstable C API +============== + +.. index:: single: PyUnstable + +Any API named with the ``PyUnstable`` prefix exposes CPython implementation +details, and may change in every minor release (e.g. from 3.9 to 3.10) without +any deprecation warnings. +However, it will not change in a bugfix release (e.g. from 3.10.0 to 3.10.1). + +It is generally intended for specialized, low-level tools like debuggers. + +Projects that use this API are expected to follow +CPython development and spend extra effort adjusting to changes. Stable Application Binary Interface diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index ed2958f8cd2205..ee64ffdc91662e 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -839,6 +839,8 @@ PyEval_EvalFrameEx:int:throwflag:: PyEval_MergeCompilerFlags:int::: PyEval_MergeCompilerFlags:PyCompilerFlags*:cf:: +PyException_GetArgs:PyObject*::+1: + PyException_GetCause:PyObject*::+1: PyException_GetCause:PyObject*:ex:0: diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 7661249ad522fa..1a0afb6940dab9 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -2538,7 +2538,7 @@ should be logged, or the ``extra`` keyword parameter to indicate additional contextual information to be added to the log). So you cannot directly make logging calls using :meth:`str.format` or :class:`string.Template` syntax, because internally the logging package uses %-formatting to merge the format -string and the variable arguments. There would no changing this while preserving +string and the variable arguments. There would be no changing this while preserving backward compatibility, since all logging calls which are out there in existing code will be using %-format strings. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index f86e784288029c..5138afc2bbe47b 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -524,8 +524,8 @@ Opening network connections When a server's IPv4 path and protocol are working, but the server's IPv6 path and protocol are not working, a dual-stack client application experiences significant connection delay compared to an - IPv4-only client. This is undesirable because it causes the dual- - stack client to have a worse user experience. This document + IPv4-only client. This is undesirable because it causes the + dual-stack client to have a worse user experience. This document specifies requirements for algorithms that reduce this user-visible delay and provides an algorithm. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index fec9b86864c578..6187098a752053 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -40,23 +40,23 @@ decimal floating point arithmetic. It offers several advantages over the people learn at school." -- excerpt from the decimal arithmetic specification. * Decimal numbers can be represented exactly. In contrast, numbers like - :const:`1.1` and :const:`2.2` do not have exact representations in binary + ``1.1`` and ``2.2`` do not have exact representations in binary floating point. End users typically would not expect ``1.1 + 2.2`` to display - as :const:`3.3000000000000003` as it does with binary floating point. + as ``3.3000000000000003`` as it does with binary floating point. * The exactness carries over into arithmetic. In decimal floating point, ``0.1 + 0.1 + 0.1 - 0.3`` is exactly equal to zero. In binary floating point, the result - is :const:`5.5511151231257827e-017`. While near to zero, the differences + is ``5.5511151231257827e-017``. While near to zero, the differences prevent reliable equality testing and differences can accumulate. For this reason, decimal is preferred in accounting applications which have strict equality invariants. * The decimal module incorporates a notion of significant places so that ``1.30 - + 1.20`` is :const:`2.50`. The trailing zero is kept to indicate significance. + + 1.20`` is ``2.50``. The trailing zero is kept to indicate significance. This is the customary presentation for monetary applications. For multiplication, the "schoolbook" approach uses all the figures in the - multiplicands. For instance, ``1.3 * 1.2`` gives :const:`1.56` while ``1.30 * - 1.20`` gives :const:`1.5600`. + multiplicands. For instance, ``1.3 * 1.2`` gives ``1.56`` while ``1.30 * + 1.20`` gives ``1.5600``. * Unlike hardware based binary floating point, the decimal module has a user alterable precision (defaulting to 28 places) which can be as large as needed for @@ -88,8 +88,8 @@ context for arithmetic, and signals. A decimal number is immutable. It has a sign, coefficient digits, and an exponent. To preserve significance, the coefficient digits do not truncate trailing zeros. Decimals also include special values such as -:const:`Infinity`, :const:`-Infinity`, and :const:`NaN`. The standard also -differentiates :const:`-0` from :const:`+0`. +``Infinity``, ``-Infinity``, and ``NaN``. The standard also +differentiates ``-0`` from ``+0``. The context for arithmetic is an environment specifying precision, rounding rules, limits on exponents, flags indicating the results of operations, and trap @@ -139,8 +139,8 @@ precision, rounding, or enabled traps:: Decimal instances can be constructed from integers, strings, floats, or tuples. Construction from an integer or a float performs an exact conversion of the value of that integer or float. Decimal numbers include special values such as -:const:`NaN` which stands for "Not a number", positive and negative -:const:`Infinity`, and :const:`-0`:: +``NaN`` which stands for "Not a number", positive and negative +``Infinity``, and ``-0``:: >>> getcontext().prec = 28 >>> Decimal(10) @@ -250,7 +250,7 @@ And some mathematical functions are also available to Decimal: >>> Decimal('10').log10() Decimal('1') -The :meth:`quantize` method rounds a number to a fixed exponent. This method is +The :meth:`~Decimal.quantize` method rounds a number to a fixed exponent. This method is useful for monetary applications that often round results to a fixed number of places: @@ -299,7 +299,7 @@ enabled: Contexts also have signal flags for monitoring exceptional conditions encountered during computations. The flags remain set until explicitly cleared, so it is best to clear the flags before each set of monitored computations by -using the :meth:`clear_flags` method. :: +using the :meth:`~Context.clear_flags` method. :: >>> setcontext(ExtendedContext) >>> getcontext().clear_flags() @@ -309,12 +309,12 @@ using the :meth:`clear_flags` method. :: Context(prec=9, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999, capitals=1, clamp=0, flags=[Inexact, Rounded], traps=[]) -The *flags* entry shows that the rational approximation to :const:`Pi` was +The *flags* entry shows that the rational approximation to pi was rounded (digits beyond the context precision were thrown away) and that the result is inexact (some of the discarded digits were non-zero). -Individual traps are set using the dictionary in the :attr:`traps` field of a -context: +Individual traps are set using the dictionary in the :attr:`~Context.traps` +attribute of a context: .. doctest:: newcontext @@ -369,7 +369,7 @@ Decimal objects with the fullwidth digits ``'\uff10'`` through ``'\uff19'``. If *value* is a :class:`tuple`, it should have three components, a sign - (:const:`0` for positive or :const:`1` for negative), a :class:`tuple` of + (``0`` for positive or ``1`` for negative), a :class:`tuple` of digits, and an integer exponent. For example, ``Decimal((0, (1, 4, 1, 4), -3))`` returns ``Decimal('1.414')``. @@ -387,7 +387,7 @@ Decimal objects The purpose of the *context* argument is determining what to do if *value* is a malformed string. If the context traps :const:`InvalidOperation`, an exception is raised; otherwise, the constructor returns a new Decimal with the value of - :const:`NaN`. + ``NaN``. Once constructed, :class:`Decimal` objects are immutable. @@ -701,7 +701,7 @@ Decimal objects .. method:: max(other, context=None) Like ``max(self, other)`` except that the context rounding rule is applied - before returning and that :const:`NaN` values are either signaled or + before returning and that ``NaN`` values are either signaled or ignored (depending on the context and whether they are signaling or quiet). @@ -713,7 +713,7 @@ Decimal objects .. method:: min(other, context=None) Like ``min(self, other)`` except that the context rounding rule is applied - before returning and that :const:`NaN` values are either signaled or + before returning and that ``NaN`` values are either signaled or ignored (depending on the context and whether they are signaling or quiet). @@ -744,8 +744,8 @@ Decimal objects .. method:: normalize(context=None) Normalize the number by stripping the rightmost trailing zeros and - converting any result equal to :const:`Decimal('0')` to - :const:`Decimal('0e0')`. Used for producing canonical values for attributes + converting any result equal to ``Decimal('0')`` to + ``Decimal('0e0')``. Used for producing canonical values for attributes of an equivalence class. For example, ``Decimal('32.100')`` and ``Decimal('0.321000e+2')`` both normalize to the equivalent value ``Decimal('32.1')``. @@ -790,7 +790,7 @@ Decimal objects the current thread's context is used. An error is returned whenever the resulting exponent is greater than - :attr:`Emax` or less than :attr:`Etiny`. + :attr:`~Context.Emax` or less than :meth:`~Context.Etiny`. .. method:: radix() @@ -830,7 +830,7 @@ Decimal objects .. method:: same_quantum(other, context=None) Test whether self and other have the same exponent or whether both are - :const:`NaN`. + ``NaN``. This operation is unaffected by context and is quiet: no flags are changed and no rounding is performed. As an exception, the C version may raise @@ -892,11 +892,11 @@ Decimal objects Logical operands ^^^^^^^^^^^^^^^^ -The :meth:`logical_and`, :meth:`logical_invert`, :meth:`logical_or`, -and :meth:`logical_xor` methods expect their arguments to be *logical +The :meth:`~Decimal.logical_and`, :meth:`~Decimal.logical_invert`, :meth:`~Decimal.logical_or`, +and :meth:`~Decimal.logical_xor` methods expect their arguments to be *logical operands*. A *logical operand* is a :class:`Decimal` instance whose exponent and sign are both zero, and whose digits are all either -:const:`0` or :const:`1`. +``0`` or ``1``. .. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -982,7 +982,7 @@ described below. In addition, the module provides three pre-made contexts: exceptions are not raised during computations). Because the traps are disabled, this context is useful for applications that - prefer to have result value of :const:`NaN` or :const:`Infinity` instead of + prefer to have result value of ``NaN`` or ``Infinity`` instead of raising exceptions. This allows an application to complete a run in the presence of conditions that would otherwise halt the program. @@ -1001,8 +1001,8 @@ described below. In addition, the module provides three pre-made contexts: In single threaded environments, it is preferable to not use this context at all. Instead, simply create contexts explicitly as described below. - The default values are :attr:`prec`\ =\ :const:`28`, - :attr:`rounding`\ =\ :const:`ROUND_HALF_EVEN`, + The default values are :attr:`Context.prec`\ =\ ``28``, + :attr:`Context.rounding`\ =\ :const:`ROUND_HALF_EVEN`, and enabled traps for :class:`Overflow`, :class:`InvalidOperation`, and :class:`DivisionByZero`. @@ -1016,7 +1016,7 @@ In addition to the three supplied contexts, new contexts can be created with the default values are copied from the :const:`DefaultContext`. If the *flags* field is not specified or is :const:`None`, all flags are cleared. - *prec* is an integer in the range [:const:`1`, :const:`MAX_PREC`] that sets + *prec* is an integer in the range [``1``, :const:`MAX_PREC`] that sets the precision for arithmetic operations in the context. The *rounding* option is one of the constants listed in the section @@ -1026,20 +1026,20 @@ In addition to the three supplied contexts, new contexts can be created with the contexts should only set traps and leave the flags clear. The *Emin* and *Emax* fields are integers specifying the outer limits allowable - for exponents. *Emin* must be in the range [:const:`MIN_EMIN`, :const:`0`], - *Emax* in the range [:const:`0`, :const:`MAX_EMAX`]. + for exponents. *Emin* must be in the range [:const:`MIN_EMIN`, ``0``], + *Emax* in the range [``0``, :const:`MAX_EMAX`]. - The *capitals* field is either :const:`0` or :const:`1` (the default). If set to - :const:`1`, exponents are printed with a capital :const:`E`; otherwise, a - lowercase :const:`e` is used: :const:`Decimal('6.02e+23')`. + The *capitals* field is either ``0`` or ``1`` (the default). If set to + ``1``, exponents are printed with a capital ``E``; otherwise, a + lowercase ``e`` is used: ``Decimal('6.02e+23')``. - The *clamp* field is either :const:`0` (the default) or :const:`1`. - If set to :const:`1`, the exponent ``e`` of a :class:`Decimal` + The *clamp* field is either ``0`` (the default) or ``1``. + If set to ``1``, the exponent ``e`` of a :class:`Decimal` instance representable in this context is strictly limited to the range ``Emin - prec + 1 <= e <= Emax - prec + 1``. If *clamp* is - :const:`0` then a weaker condition holds: the adjusted exponent of - the :class:`Decimal` instance is at most ``Emax``. When *clamp* is - :const:`1`, a large normal number will, where possible, have its + ``0`` then a weaker condition holds: the adjusted exponent of + the :class:`Decimal` instance is at most :attr:`~Context.Emax`. When *clamp* is + ``1``, a large normal number will, where possible, have its exponent reduced and a corresponding number of zeros added to its coefficient, in order to fit the exponent constraints; this preserves the value of the number but loses information about @@ -1048,13 +1048,13 @@ In addition to the three supplied contexts, new contexts can be created with the >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') Decimal('1.23000E+999') - A *clamp* value of :const:`1` allows compatibility with the + A *clamp* value of ``1`` allows compatibility with the fixed-width decimal interchange formats specified in IEEE 754. The :class:`Context` class defines several general purpose methods as well as a large number of methods for doing arithmetic directly in a given context. In addition, for each of the :class:`Decimal` methods described above (with - the exception of the :meth:`adjusted` and :meth:`as_tuple` methods) there is + the exception of the :meth:`~Decimal.adjusted` and :meth:`~Decimal.as_tuple` methods) there is a corresponding :class:`Context` method. For example, for a :class:`Context` instance ``C`` and :class:`Decimal` instance ``x``, ``C.exp(x)`` is equivalent to ``x.exp(context=C)``. Each :class:`Context` method accepts a @@ -1064,11 +1064,11 @@ In addition to the three supplied contexts, new contexts can be created with the .. method:: clear_flags() - Resets all of the flags to :const:`0`. + Resets all of the flags to ``0``. .. method:: clear_traps() - Resets all of the traps to :const:`0`. + Resets all of the traps to ``0``. .. versionadded:: 3.3 @@ -1483,13 +1483,13 @@ are also included in the pure Python version for compatibility. +---------------------+---------------------+-------------------------------+ | | 32-bit | 64-bit | +=====================+=====================+===============================+ -| .. data:: MAX_PREC | :const:`425000000` | :const:`999999999999999999` | +| .. data:: MAX_PREC | ``425000000`` | ``999999999999999999`` | +---------------------+---------------------+-------------------------------+ -| .. data:: MAX_EMAX | :const:`425000000` | :const:`999999999999999999` | +| .. data:: MAX_EMAX | ``425000000`` | ``999999999999999999`` | +---------------------+---------------------+-------------------------------+ -| .. data:: MIN_EMIN | :const:`-425000000` | :const:`-999999999999999999` | +| .. data:: MIN_EMIN | ``-425000000`` | ``-999999999999999999`` | +---------------------+---------------------+-------------------------------+ -| .. data:: MIN_ETINY | :const:`-849999999` | :const:`-1999999999999999997` | +| .. data:: MIN_ETINY | ``-849999999`` | ``-1999999999999999997`` | +---------------------+---------------------+-------------------------------+ @@ -1514,7 +1514,7 @@ Rounding modes .. data:: ROUND_CEILING - Round towards :const:`Infinity`. + Round towards ``Infinity``. .. data:: ROUND_DOWN @@ -1522,7 +1522,7 @@ Rounding modes .. data:: ROUND_FLOOR - Round towards :const:`-Infinity`. + Round towards ``-Infinity``. .. data:: ROUND_HALF_DOWN @@ -1570,7 +1570,7 @@ condition. Altered an exponent to fit representation constraints. Typically, clamping occurs when an exponent falls outside the context's - :attr:`Emin` and :attr:`Emax` limits. If possible, the exponent is reduced to + :attr:`~Context.Emin` and :attr:`~Context.Emax` limits. If possible, the exponent is reduced to fit by adding zeros to the coefficient. @@ -1584,8 +1584,8 @@ condition. Signals the division of a non-infinite number by zero. Can occur with division, modulo division, or when raising a number to a negative - power. If this signal is not trapped, returns :const:`Infinity` or - :const:`-Infinity` with the sign determined by the inputs to the calculation. + power. If this signal is not trapped, returns ``Infinity`` or + ``-Infinity`` with the sign determined by the inputs to the calculation. .. class:: Inexact @@ -1602,7 +1602,7 @@ condition. An invalid operation was performed. Indicates that an operation was requested that does not make sense. If not - trapped, returns :const:`NaN`. Possible causes include:: + trapped, returns ``NaN``. Possible causes include:: Infinity - Infinity 0 * Infinity @@ -1619,10 +1619,10 @@ condition. Numerical overflow. - Indicates the exponent is larger than :attr:`Emax` after rounding has + Indicates the exponent is larger than :attr:`Context.Emax` after rounding has occurred. If not trapped, the result depends on the rounding mode, either pulling inward to the largest representable finite number or rounding outward - to :const:`Infinity`. In either case, :class:`Inexact` and :class:`Rounded` + to ``Infinity``. In either case, :class:`Inexact` and :class:`Rounded` are also signaled. @@ -1631,14 +1631,14 @@ condition. Rounding occurred though possibly no information was lost. Signaled whenever rounding discards digits; even if those digits are zero - (such as rounding :const:`5.00` to :const:`5.0`). If not trapped, returns + (such as rounding ``5.00`` to ``5.0``). If not trapped, returns the result unchanged. This signal is used to detect loss of significant digits. .. class:: Subnormal - Exponent was lower than :attr:`Emin` prior to rounding. + Exponent was lower than :attr:`~Context.Emin` prior to rounding. Occurs when an operation result is subnormal (the exponent is too small). If not trapped, returns the result unchanged. @@ -1696,7 +1696,7 @@ Mitigating round-off error with increased precision ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The use of decimal floating point eliminates decimal representation error -(making it possible to represent :const:`0.1` exactly); however, some operations +(making it possible to represent ``0.1`` exactly); however, some operations can still incur round-off error when non-zero digits exceed the fixed precision. The effects of round-off error can be amplified by the addition or subtraction @@ -1746,8 +1746,8 @@ Special values ^^^^^^^^^^^^^^ The number system for the :mod:`decimal` module provides special values -including :const:`NaN`, :const:`sNaN`, :const:`-Infinity`, :const:`Infinity`, -and two zeros, :const:`+0` and :const:`-0`. +including ``NaN``, ``sNaN``, ``-Infinity``, ``Infinity``, +and two zeros, ``+0`` and ``-0``. Infinities can be constructed directly with: ``Decimal('Infinity')``. Also, they can arise from dividing by zero when the :exc:`DivisionByZero` signal is @@ -1758,30 +1758,30 @@ The infinities are signed (affine) and can be used in arithmetic operations where they get treated as very large, indeterminate numbers. For instance, adding a constant to infinity gives another infinite result. -Some operations are indeterminate and return :const:`NaN`, or if the +Some operations are indeterminate and return ``NaN``, or if the :exc:`InvalidOperation` signal is trapped, raise an exception. For example, -``0/0`` returns :const:`NaN` which means "not a number". This variety of -:const:`NaN` is quiet and, once created, will flow through other computations -always resulting in another :const:`NaN`. This behavior can be useful for a +``0/0`` returns ``NaN`` which means "not a number". This variety of +``NaN`` is quiet and, once created, will flow through other computations +always resulting in another ``NaN``. This behavior can be useful for a series of computations that occasionally have missing inputs --- it allows the calculation to proceed while flagging specific results as invalid. -A variant is :const:`sNaN` which signals rather than remaining quiet after every +A variant is ``sNaN`` which signals rather than remaining quiet after every operation. This is a useful return value when an invalid result needs to interrupt a calculation for special handling. The behavior of Python's comparison operators can be a little surprising where a -:const:`NaN` is involved. A test for equality where one of the operands is a -quiet or signaling :const:`NaN` always returns :const:`False` (even when doing +``NaN`` is involved. A test for equality where one of the operands is a +quiet or signaling ``NaN`` always returns :const:`False` (even when doing ``Decimal('NaN')==Decimal('NaN')``), while a test for inequality always returns :const:`True`. An attempt to compare two Decimals using any of the ``<``, ``<=``, ``>`` or ``>=`` operators will raise the :exc:`InvalidOperation` signal -if either operand is a :const:`NaN`, and return :const:`False` if this signal is +if either operand is a ``NaN``, and return :const:`False` if this signal is not trapped. Note that the General Decimal Arithmetic specification does not specify the behavior of direct comparisons; these rules for comparisons -involving a :const:`NaN` were taken from the IEEE 854 standard (see Table 3 in -section 5.7). To ensure strict standards-compliance, use the :meth:`compare` -and :meth:`compare-signal` methods instead. +involving a ``NaN`` were taken from the IEEE 854 standard (see Table 3 in +section 5.7). To ensure strict standards-compliance, use the :meth:`~Decimal.compare` +and :meth:`~Decimal.compare_signal` methods instead. The signed zeros can result from calculations that underflow. They keep the sign that would have resulted if the calculation had been carried out to greater @@ -2013,7 +2013,7 @@ Q. In a fixed-point application with two decimal places, some inputs have many places and need to be rounded. Others are not supposed to have excess digits and need to be validated. What methods should be used? -A. The :meth:`quantize` method rounds to a fixed number of decimal places. If +A. The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If the :const:`Inexact` trap is set, it is also useful for validation: >>> TWOPLACES = Decimal(10) ** -2 # same as Decimal('0.01') @@ -2037,7 +2037,7 @@ throughout an application? A. Some operations like addition, subtraction, and multiplication by an integer will automatically preserve fixed point. Others operations, like division and non-integer multiplication, will change the number of decimal places and need to -be followed-up with a :meth:`quantize` step: +be followed-up with a :meth:`~Decimal.quantize` step: >>> a = Decimal('102.72') # Initial fixed-point values >>> b = Decimal('3.17') @@ -2053,7 +2053,7 @@ be followed-up with a :meth:`quantize` step: Decimal('0.03') In developing fixed-point applications, it is convenient to define functions -to handle the :meth:`quantize` step: +to handle the :meth:`~Decimal.quantize` step: >>> def mul(x, y, fp=TWOPLACES): ... return (x * y).quantize(fp) @@ -2066,12 +2066,12 @@ to handle the :meth:`quantize` step: >>> div(b, a) Decimal('0.03') -Q. There are many ways to express the same value. The numbers :const:`200`, -:const:`200.000`, :const:`2E2`, and :const:`.02E+4` all have the same value at +Q. There are many ways to express the same value. The numbers ``200``, +``200.000``, ``2E2``, and ``.02E+4`` all have the same value at various precisions. Is there a way to transform them to a single recognizable canonical value? -A. The :meth:`normalize` method maps all equivalent values to a single +A. The :meth:`~Decimal.normalize` method maps all equivalent values to a single representative: >>> values = map(Decimal, '200 200.000 2E2 .02E+4'.split()) @@ -2083,7 +2083,7 @@ to get a non-exponential representation? A. For some values, exponential notation is the only way to express the number of significant places in the coefficient. For example, expressing -:const:`5.0E+3` as :const:`5000` keeps the value constant but cannot show the +``5.0E+3`` as ``5000`` keeps the value constant but cannot show the original's two-place significance. If an application does not care about tracking significance, it is easy to @@ -2159,12 +2159,12 @@ for medium-sized numbers and the `Number Theoretic Transform `_ for very large numbers. -The context must be adapted for exact arbitrary precision arithmetic. :attr:`Emin` -and :attr:`Emax` should always be set to the maximum values, :attr:`clamp` -should always be 0 (the default). Setting :attr:`prec` requires some care. +The context must be adapted for exact arbitrary precision arithmetic. :attr:`~Context.Emin` +and :attr:`~Context.Emax` should always be set to the maximum values, :attr:`~Context.clamp` +should always be 0 (the default). Setting :attr:`~Context.prec` requires some care. The easiest approach for trying out bignum arithmetic is to use the maximum -value for :attr:`prec` as well [#]_:: +value for :attr:`~Context.prec` as well [#]_:: >>> setcontext(Context(prec=MAX_PREC, Emax=MAX_EMAX, Emin=MIN_EMIN)) >>> x = Decimal(2) ** 256 @@ -2181,7 +2181,7 @@ the available memory will be insufficient:: MemoryError On systems with overallocation (e.g. Linux), a more sophisticated approach is to -adjust :attr:`prec` to the amount of available RAM. Suppose that you have 8GB of +adjust :attr:`~Context.prec` to the amount of available RAM. Suppose that you have 8GB of RAM and expect 10 simultaneous operands using a maximum of 500MB each:: >>> import sys diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index c61bbac892e0e4..fe2e8ab655edf8 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -118,7 +118,8 @@ another rational number, or from a string. .. method:: as_integer_ratio() Return a tuple of two integers, whose ratio is equal - to the Fraction and with a positive denominator. + to the original Fraction. The ratio is in lowest terms + and has a positive denominator. .. versionadded:: 3.8 diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3ff28849025153..f0f374771b0cf1 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -623,7 +623,7 @@ are always available. They are listed here in alphabetical order. .. function:: filter(function, iterable) Construct an iterator from those elements of *iterable* for which *function* - returns true. *iterable* may be either a sequence, a container which + is true. *iterable* may be either a sequence, a container which supports iteration, or an iterator. If *function* is ``None``, the identity function is assumed, that is, all elements of *iterable* that are false are removed. @@ -634,7 +634,7 @@ are always available. They are listed here in alphabetical order. ``None``. See :func:`itertools.filterfalse` for the complementary function that returns - elements of *iterable* for which *function* returns false. + elements of *iterable* for which *function* is false. .. class:: float(x=0.0) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 80a405e87d8d56..d467e50bc7a424 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -86,6 +86,14 @@ The :mod:`functools` module defines the following functions: The cached value can be cleared by deleting the attribute. This allows the *cached_property* method to run again. + The *cached_property* does not prevent a possible race condition in + multi-threaded usage. The getter function could run more than once on the + same instance, with the latest run setting the cached value. If the cached + property is idempotent or otherwise not harmful to run more than once on an + instance, this is fine. If synchronization is needed, implement the necessary + locking inside the decorated getter function or around the cached property + access. + Note, this decorator interferes with the operation of :pep:`412` key-sharing dictionaries. This means that instance dictionaries can take more space than usual. @@ -110,6 +118,14 @@ The :mod:`functools` module defines the following functions: def stdev(self): return statistics.stdev(self._data) + + .. versionchanged:: 3.12 + Prior to Python 3.12, ``cached_property`` included an undocumented lock to + ensure that in multi-threaded usage the getter function was guaranteed to + run only once per instance. However, the lock was per-property, not + per-instance, which could result in unacceptably high lock contention. In + Python 3.12+ this locking is removed. + .. versionadded:: 3.8 diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 9c3be5a250a67e..789e9839d22f71 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -802,8 +802,9 @@ function. .. attribute:: Parameter.kind - Describes how argument values are bound to the parameter. Possible values - (accessible via :class:`Parameter`, like ``Parameter.KEYWORD_ONLY``): + Describes how argument values are bound to the parameter. The possible + values are accessible via :class:`Parameter` (like ``Parameter.KEYWORD_ONLY``), + and support comparison and ordering, in the following order: .. tabularcolumns:: |l|L| diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 8d83d92660d6ef..95da7166842ee6 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -398,7 +398,7 @@ loops that truncate the stream. .. function:: filterfalse(predicate, iterable) Make an iterator that filters elements from iterable returning only those for - which the predicate is ``False``. If *predicate* is ``None``, return the items + which the predicate is false. If *predicate* is ``None``, return the items that are false. Roughly equivalent to:: def filterfalse(predicate, iterable): @@ -831,7 +831,7 @@ which incur interpreter overhead. return next(g, True) and not next(g, False) def quantify(iterable, pred=bool): - "Count how many times the predicate is true" + "Count how many times the predicate is True" return sum(map(pred, iterable)) def ncycles(iterable, n): diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index c4f8781f2ac993..69afadff1f5f42 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -370,11 +370,19 @@ MAP_* Constants MAP_ANONYMOUS MAP_POPULATE MAP_STACK + MAP_ALIGNED_SUPER + MAP_CONCEAL - These are the various flags that can be passed to :meth:`mmap.mmap`. Note that some options might not be present on some systems. + These are the various flags that can be passed to :meth:`mmap.mmap`. :data:`MAP_ALIGNED_SUPER` + is only available at FreeBSD and :data:`MAP_CONCEAL` is only available at OpenBSD. Note + that some options might not be present on some systems. .. versionchanged:: 3.10 - Added MAP_POPULATE constant. + Added :data:`MAP_POPULATE` constant. .. versionadded:: 3.11 - Added MAP_STACK constant. + Added :data:`MAP_STACK` constant. + + .. versionadded:: 3.12 + Added :data:`MAP_ALIGNED_SUPER` constant. + Added :data:`MAP_CONCEAL` constant. diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index a0c9f63ab9f957..69c4dfc422c98e 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -63,7 +63,7 @@ Cross Platform string is returned if the value cannot be determined. -.. function:: platform(aliased=0, terse=0) +.. function:: platform(aliased=False, terse=False) Returns a single string identifying the underlying platform with as much useful information as possible. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index c67f15e953bccc..b2b787c5a8260c 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -15,7 +15,7 @@ module implements all the required locking semantics. The module implements three types of queue, which differ only in the order in which the entries are retrieved. In a :abbr:`FIFO (first-in, first-out)` -queue, the first tasks added are the first retrieved. In a +queue, the first tasks added are the first retrieved. In a :abbr:`LIFO (last-in, first-out)` queue, the most recently added entry is the first retrieved (operating like a stack). With a priority queue, the entries are kept sorted (using the :mod:`heapq` module) and the @@ -57,8 +57,8 @@ The :mod:`queue` module defines the following classes and exceptions: *maxsize* is less than or equal to zero, the queue size is infinite. The lowest valued entries are retrieved first (the lowest valued entry is the - one returned by ``sorted(list(entries))[0]``). A typical pattern for entries - is a tuple in the form: ``(priority_number, data)``. + one that would be returned by ``min(entries)``). A typical pattern for + entries is a tuple in the form: ``(priority_number, data)``. If the *data* elements are not comparable, the data can be wrapped in a class that ignores the data item and only compares the priority number:: @@ -127,8 +127,8 @@ provide the public methods described below. .. method:: Queue.put(item, block=True, timeout=None) - Put *item* into the queue. If optional args *block* is true and *timeout* is - ``None`` (the default), block if necessary until a free slot is available. If + Put *item* into the queue. If optional args *block* is true and *timeout* is + ``None`` (the default), block if necessary until a free slot is available. If *timeout* is a positive number, it blocks at most *timeout* seconds and raises the :exc:`Full` exception if no free slot was available within that time. Otherwise (*block* is false), put an item on the queue if a free slot is @@ -143,7 +143,7 @@ provide the public methods described below. .. method:: Queue.get(block=True, timeout=None) - Remove and return an item from the queue. If optional args *block* is true and + Remove and return an item from the queue. If optional args *block* is true and *timeout* is ``None`` (the default), block if necessary until an item is available. If *timeout* is a positive number, it blocks at most *timeout* seconds and raises the :exc:`Empty` exception if no item was available within that time. @@ -152,7 +152,7 @@ provide the public methods described below. Prior to 3.0 on POSIX systems, and for all versions on Windows, if *block* is true and *timeout* is ``None``, this operation goes into - an uninterruptible wait on an underlying lock. This means that no exceptions + an uninterruptible wait on an underlying lock. This means that no exceptions can occur, and in particular a SIGINT will not trigger a :exc:`KeyboardInterrupt`. @@ -184,7 +184,7 @@ fully processed by daemon consumer threads. The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer thread calls :meth:`task_done` to - indicate that the item was retrieved and all work on it is complete. When the + indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. @@ -227,7 +227,7 @@ SimpleQueue Objects .. method:: SimpleQueue.empty() - Return ``True`` if the queue is empty, ``False`` otherwise. If empty() + Return ``True`` if the queue is empty, ``False`` otherwise. If empty() returns ``False`` it doesn't guarantee that a subsequent call to get() will not block. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index d0a16b95184474..b7510b93d75427 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -271,7 +271,8 @@ The special characters are: * To match a literal ``']'`` inside a set, precede it with a backslash, or place it at the beginning of the set. For example, both ``[()[\]{}]`` and - ``[]()[{}]`` will both match a parenthesis. + ``[]()[{}]`` will match a right bracket, as well as left bracket, braces, + and parentheses. .. .. index:: single: --; in regular expressions .. .. index:: single: &&; in regular expressions diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 98bda6ded30f79..1240f80b0f11f0 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -84,8 +84,8 @@ These are the Boolean operations, ordered by ascending priority: +-------------+---------------------------------+-------+ | Operation | Result | Notes | +=============+=================================+=======+ -| ``x or y`` | if *x* is false, then *y*, else | \(1) | -| | *x* | | +| ``x or y`` | if *x* is true, then *x*, else | \(1) | +| | *y* | | +-------------+---------------------------------+-------+ | ``x and y`` | if *x* is false, then *x*, else | \(2) | | | *y* | | @@ -602,8 +602,8 @@ class`. In addition, it provides a few more methods: .. method:: int.as_integer_ratio() - Return a pair of integers whose ratio is exactly equal to the original - integer and with a positive denominator. The integer ratio of integers + Return a pair of integers whose ratio is equal to the original + integer and has a positive denominator. The integer ratio of integers (whole numbers) is always the integer as the numerator and ``1`` as the denominator. @@ -624,7 +624,7 @@ class`. float also has the following additional methods. .. method:: float.as_integer_ratio() Return a pair of integers whose ratio is exactly equal to the - original float and with a positive denominator. Raises + original float. The ratio is in lowest terms and has a positive denominator. Raises :exc:`OverflowError` on infinities and a :exc:`ValueError` on NaNs. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 69d95f27cb61d9..9c0e32ba16bf68 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -371,7 +371,7 @@ ordering:: >>> from struct import * >>> pack(">bhl", 1, 2, 3) b'\x01\x00\x02\x00\x00\x00\x03' - >>> unpack('>bhl', b'\x01\x00\x02\x00\x00\x00\x03' + >>> unpack('>bhl', b'\x01\x00\x02\x00\x00\x00\x03') (1, 2, 3) >>> calcsize('>bhl') 7 diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 605e2c9a6710c1..23c5bbed0c6f9c 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -568,55 +568,55 @@ always available. .. tabularcolumns:: |l|l|L| - +---------------------+----------------+--------------------------------------------------+ - | attribute | float.h macro | explanation | - +=====================+================+==================================================+ - | :const:`epsilon` | DBL_EPSILON | difference between 1.0 and the least value | - | | | greater than 1.0 that is representable as a float| - | | | | - | | | See also :func:`math.ulp`. | - +---------------------+----------------+--------------------------------------------------+ - | :const:`dig` | DBL_DIG | maximum number of decimal digits that can be | - | | | faithfully represented in a float; see below | - +---------------------+----------------+--------------------------------------------------+ - | :const:`mant_dig` | DBL_MANT_DIG | float precision: the number of base-``radix`` | - | | | digits in the significand of a float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`max` | DBL_MAX | maximum representable positive finite float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`max_exp` | DBL_MAX_EXP | maximum integer *e* such that ``radix**(e-1)`` is| - | | | a representable finite float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`max_10_exp` | DBL_MAX_10_EXP | maximum integer *e* such that ``10**e`` is in the| - | | | range of representable finite floats | - +---------------------+----------------+--------------------------------------------------+ - | :const:`min` | DBL_MIN | minimum representable positive *normalized* float| - | | | | - | | | Use :func:`math.ulp(0.0) ` to get the | - | | | smallest positive *denormalized* representable | - | | | float. | - +---------------------+----------------+--------------------------------------------------+ - | :const:`min_exp` | DBL_MIN_EXP | minimum integer *e* such that ``radix**(e-1)`` is| - | | | a normalized float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`min_10_exp` | DBL_MIN_10_EXP | minimum integer *e* such that ``10**e`` is a | - | | | normalized float | - +---------------------+----------------+--------------------------------------------------+ - | :const:`radix` | FLT_RADIX | radix of exponent representation | - +---------------------+----------------+--------------------------------------------------+ - | :const:`rounds` | FLT_ROUNDS | integer representing the rounding mode for | - | | | floating-point arithmetic. This reflects the | - | | | value of the system FLT_ROUNDS macro at | - | | | interpreter startup time: | - | | | ``-1`` indeterminable, | - | | | ``0`` toward zero, | - | | | ``1`` to nearest, | - | | | ``2`` toward positive infinity, | - | | | ``3`` toward negative infinity | - | | | | - | | | All other values for FLT_ROUNDS characterize | - | | | implementation-defined rounding behavior. | - +---------------------+----------------+--------------------------------------------------+ + +---------------------+---------------------+--------------------------------------------------+ + | attribute | float.h macro | explanation | + +=====================+=====================+==================================================+ + | ``epsilon`` | ``DBL_EPSILON`` | difference between 1.0 and the least value | + | | | greater than 1.0 that is representable as a float| + | | | | + | | | See also :func:`math.ulp`. | + +---------------------+---------------------+--------------------------------------------------+ + | ``dig`` | ``DBL_DIG`` | maximum number of decimal digits that can be | + | | | faithfully represented in a float; see below | + +---------------------+---------------------+--------------------------------------------------+ + | ``mant_dig`` | ``DBL_MANT_DIG`` | float precision: the number of base-``radix`` | + | | | digits in the significand of a float | + +---------------------+---------------------+--------------------------------------------------+ + | ``max`` | ``DBL_MAX`` | maximum representable positive finite float | + +---------------------+---------------------+--------------------------------------------------+ + | ``max_exp`` | ``DBL_MAX_EXP`` | maximum integer *e* such that ``radix**(e-1)`` is| + | | | a representable finite float | + +---------------------+---------------------+--------------------------------------------------+ + | ``max_10_exp`` | ``DBL_MAX_10_EXP`` | maximum integer *e* such that ``10**e`` is in the| + | | | range of representable finite floats | + +---------------------+---------------------+--------------------------------------------------+ + | ``min`` | ``DBL_MIN`` | minimum representable positive *normalized* float| + | | | | + | | | Use :func:`math.ulp(0.0) ` to get the | + | | | smallest positive *denormalized* representable | + | | | float. | + +---------------------+---------------------+--------------------------------------------------+ + | ``min_exp`` | ``DBL_MIN_EXP`` | minimum integer *e* such that ``radix**(e-1)`` is| + | | | a normalized float | + +---------------------+---------------------+--------------------------------------------------+ + | ``min_10_exp`` | ``DBL_MIN_10_EXP`` | minimum integer *e* such that ``10**e`` is a | + | | | normalized float | + +---------------------+---------------------+--------------------------------------------------+ + | ``radix`` | ``FLT_RADIX`` | radix of exponent representation | + +---------------------+---------------------+--------------------------------------------------+ + | ``rounds`` | ``FLT_ROUNDS`` | integer representing the rounding mode for | + | | | floating-point arithmetic. This reflects the | + | | | value of the system ``FLT_ROUNDS`` macro at | + | | | interpreter startup time: | + | | | ``-1`` indeterminable, | + | | | ``0`` toward zero, | + | | | ``1`` to nearest, | + | | | ``2`` toward positive infinity, | + | | | ``3`` toward negative infinity | + | | | | + | | | All other values for ``FLT_ROUNDS`` characterize | + | | | implementation-defined rounding behavior. | + +---------------------+---------------------+--------------------------------------------------+ The attribute :attr:`sys.float_info.dig` needs further explanation. If ``s`` is any string representing a decimal number with at most diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 5add61c759ea8e..b7eb569c18a6c4 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -2444,6 +2444,9 @@ The demo scripts are: | planet_and_moon| simulation of | compound shapes, | | | gravitational system | :class:`Vec2D` | +----------------+------------------------------+-----------------------+ +| rosette | a pattern from the wikipedia | :func:`clone`, | +| | article on turtle graphics | :func:`undo` | ++----------------+------------------------------+-----------------------+ | round_dance | dancing turtles rotating | compound shapes, clone| | | pairwise in opposite | shapesize, tilt, | | | direction | get_shapepoly, update | @@ -2457,9 +2460,6 @@ The demo scripts are: | two_canvases | simple design | turtles on two | | | | canvases | +----------------+------------------------------+-----------------------+ -| wikipedia | a pattern from the wikipedia | :func:`clone`, | -| | article on turtle graphics | :func:`undo` | -+----------------+------------------------------+-----------------------+ | yinyang | another elementary example | :func:`circle` | +----------------+------------------------------+-----------------------+ diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 415413c4cd9747..747ba58bb225d4 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -486,7 +486,7 @@ Coroutine Utility Functions The generator-based coroutine is still a :term:`generator iterator`, but is also considered to be a :term:`coroutine` object and is :term:`awaitable`. However, it may not necessarily implement - the :meth:`__await__` method. + the :meth:`~object.__await__` method. If *gen_func* is a generator function, it will be modified in-place. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 169f7196a74ec6..3395e4bfb95c44 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -91,6 +91,8 @@ annotations. These include: *Introducing* :data:`LiteralString` * :pep:`681`: Data Class Transforms *Introducing* the :func:`@dataclass_transform` decorator +* :pep:`698`: Adding an override decorator to typing + *Introducing* the :func:`@override` decorator .. _type-aliases: @@ -1588,7 +1590,7 @@ These are not used in annotations. They are building blocks for creating generic methods, not their type signatures. For example, :class:`ssl.SSLObject` is a class, therefore it passes an :func:`issubclass` check against :data:`Callable`. However, the - :meth:`ssl.SSLObject.__init__` method exists only to raise a + ``ssl.SSLObject.__init__`` method exists only to raise a :exc:`TypeError` with a more informative message, therefore making it impossible to call (instantiate) :class:`ssl.SSLObject`. @@ -2722,6 +2724,42 @@ Functions and decorators This wraps the decorator with something that wraps the decorated function in :func:`no_type_check`. + +.. decorator:: override + + A decorator for methods that indicates to type checkers that this method + should override a method or attribute with the same name on a base class. + This helps prevent bugs that may occur when a base class is changed without + an equivalent change to a child class. + + For example:: + + class Base: + def log_status(self) + + class Sub(Base): + @override + def log_status(self) -> None: # Okay: overrides Base.log_status + ... + + @override + def done(self) -> None: # Error reported by type checker + ... + + There is no runtime checking of this property. + + The decorator will set the ``__override__`` attribute to ``True`` on + the decorated object. Thus, a check like + ``if getattr(obj, "__override__", False)`` can be used at runtime to determine + whether an object ``obj`` has been marked as an override. If the decorated object + does not support setting attributes, the decorator returns the object unchanged + without raising an exception. + + See :pep:`698` for more details. + + .. versionadded:: 3.12 + + .. decorator:: type_check_only Decorator to mark a class or function to be unavailable at runtime. diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 9defb24a0331ef..e5dc82cf0dc2d9 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -143,6 +143,22 @@ def add_annotations(self, app, doctree): ' (Only some members are part of the stable ABI.)') node.insert(0, emph_node) + # Unstable API annotation. + if name.startswith('PyUnstable'): + warn_node = nodes.admonition( + classes=['unstable-c-api', 'warning']) + message = 'This is ' + emph_node = nodes.emphasis(message, message) + ref_node = addnodes.pending_xref( + 'Unstable API', refdomain="std", + reftarget='unstable-c-api', + reftype='ref', refexplicit="False") + ref_node += nodes.Text('Unstable API') + emph_node += ref_node + emph_node += nodes.Text('. It may change without warning in minor releases.') + warn_node += emph_node + node.insert(0, warn_node) + # Return value annotation if objtype != 'function': continue diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index c62f462a19a2df..c0c021c679147f 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -350,6 +350,14 @@ tempfile The :class:`tempfile.NamedTemporaryFile` function has a new optional parameter *delete_on_close* (Contributed by Evgeny Zorin in :gh:`58451`.) +typing +------ + +* Add :func:`typing.override`, an override decorator telling to static type + checkers to verify that a method overrides some method or attribute of the + same name on a base class, as per :pep:`698`. (Contributed by Steven Troxler in + :gh:`101564`.) + sys --- @@ -761,6 +769,15 @@ Changes in the Python API around process-global resources, which are best managed from the main interpreter. (Contributed by Dong-hee Na in :gh:`99127`.) +* The undocumented locking behavior of :func:`~functools.cached_property` + is removed, because it locked across all instances of the class, leading to high + lock contention. This means that a cached property getter function could now run + more than once for a single instance, if two threads race. For most simple + cached properties (e.g. those that are idempotent and simply calculate a value + based on other attributes of the instance) this will be fine. If + synchronization is needed, implement locking within the cached property getter + function or around multi-threaded access points. + Build Changes ============= @@ -793,6 +810,29 @@ C API Changes New Features ------------ + +* :pep:`697`: Introduced the :ref:`Unstable C API tier `, + intended for low-level tools like debuggers and JIT compilers. + This API may change in each minor release of CPython without deprecation + warnings. + Its contents are marked by the ``PyUnstable_`` prefix in names. + + Code object constructors: + + - ``PyUnstable_Code_New()`` (renamed from ``PyCode_New``) + - ``PyUnstable_Code_NewWithPosOnlyArgs()`` (renamed from ``PyCode_NewWithPosOnlyArgs``) + + Extra storage for code objects (:pep:`523`): + + - ``PyUnstable_Eval_RequestCodeExtraIndex()`` (renamed from ``_PyEval_RequestCodeExtraIndex``) + - ``PyUnstable_Code_GetExtra()`` (renamed from ``_PyCode_GetExtra``) + - ``PyUnstable_Code_SetExtra()`` (renamed from ``_PyCode_SetExtra``) + + The original names will continue to be available until the respective + API changes. + + (Contributed by Petr Viktorin in :gh:`101101`.) + * Added the new limited C API function :c:func:`PyType_FromMetaclass`, which generalizes the existing :c:func:`PyType_FromModuleAndSpec` using an additional metaclass argument. @@ -861,6 +901,19 @@ New Features get a frame variable by its name. (Contributed by Victor Stinner in :gh:`91248`.) +* Add :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException` + for saving and restoring the current exception. + These functions return and accept a single exception object, + rather than the triple arguments of the now-deprecated + :c:func:`PyErr_Fetch` and :c:func:`PyErr_Restore`. + This is less error prone and a bit more efficient. + (Contributed by Mark Shannon in :gh:`101578`.) + +* Add :c:func:`PyException_GetArgs` and :c:func:`PyException_SetArgs` + as convenience functions for retrieving and modifying + the :attr:`~BaseException.args` passed to the exception's constructor. + (Contributed by Mark Shannon in :gh:`101578`.) + Porting to Python 3.12 ---------------------- @@ -984,6 +1037,11 @@ Deprecated (Contributed in :gh:`47146` by Petr Viktorin, based on earlier work by Alexander Belopolsky and Matthias Braun.) +* :c:func:`PyErr_Fetch` and :c:func:`PyErr_Restore` are deprecated. + Use :c:func:`PyErr_GetRaisedException` and + :c:func:`PyErr_SetRaisedException` instead. + (Contributed by Mark Shannon in :gh:`101578`.) + Removed ------- diff --git a/Include/README.rst b/Include/README.rst index f52e690eac9a91..531f09692f783f 100644 --- a/Include/README.rst +++ b/Include/README.rst @@ -1,11 +1,13 @@ The Python C API ================ -The C API is divided into three sections: +The C API is divided into these sections: 1. ``Include/``: Limited API 2. ``Include/cpython/``: CPython implementation details -3. ``Include/internal/``: The internal API +3. ``Include/cpython/``, names with the ``PyUnstable_`` prefix: API that can + change between minor releases +4. ``Include/internal/``, and any name with ``_`` prefix: The internal API Information on changing the C API is available `in the developer guide`_ diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 74665c9fa10580..0fbbee10c2edce 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -22,7 +22,12 @@ PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _P PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds); PyAPI_FUNC(unsigned long) _PyEval_GetSwitchInterval(void); -PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc); +PyAPI_FUNC(Py_ssize_t) PyUnstable_Eval_RequestCodeExtraIndex(freefunc); +// Old name -- remove when this API changes: +_Py_DEPRECATED_EXTERNALLY(3.12) static inline Py_ssize_t +_PyEval_RequestCodeExtraIndex(freefunc f) { + return PyUnstable_Eval_RequestCodeExtraIndex(f); +} PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index fba9296aedc99d..0e4bd8a58c165b 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -178,19 +178,40 @@ static inline int PyCode_GetFirstFree(PyCodeObject *op) { #define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) #define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) -/* Public interface */ -PyAPI_FUNC(PyCodeObject *) PyCode_New( +/* Unstable public interface */ +PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_New( int, int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *, PyObject *); -PyAPI_FUNC(PyCodeObject *) PyCode_NewWithPosOnlyArgs( +PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_NewWithPosOnlyArgs( int, int, int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *, PyObject *); /* same as struct above */ +// Old names -- remove when this API changes: +_Py_DEPRECATED_EXTERNALLY(3.12) static inline PyCodeObject * +PyCode_New( + int a, int b, int c, int d, int e, PyObject *f, PyObject *g, + PyObject *h, PyObject *i, PyObject *j, PyObject *k, + PyObject *l, PyObject *m, PyObject *n, int o, PyObject *p, + PyObject *q) +{ + return PyUnstable_Code_New( + a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q); +} +_Py_DEPRECATED_EXTERNALLY(3.12) static inline PyCodeObject * +PyCode_NewWithPosOnlyArgs( + int a, int poac, int b, int c, int d, int e, PyObject *f, PyObject *g, + PyObject *h, PyObject *i, PyObject *j, PyObject *k, + PyObject *l, PyObject *m, PyObject *n, int o, PyObject *p, + PyObject *q) +{ + return PyUnstable_Code_NewWithPosOnlyArgs( + a, poac, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q); +} /* Creates a new empty code object with the specified source location. */ PyAPI_FUNC(PyCodeObject *) @@ -269,11 +290,21 @@ PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj); PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, PyObject *lnotab); - -PyAPI_FUNC(int) _PyCode_GetExtra(PyObject *code, Py_ssize_t index, - void **extra); -PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index, - void *extra); +PyAPI_FUNC(int) PyUnstable_Code_GetExtra( + PyObject *code, Py_ssize_t index, void **extra); +PyAPI_FUNC(int) PyUnstable_Code_SetExtra( + PyObject *code, Py_ssize_t index, void *extra); +// Old names -- remove when this API changes: +_Py_DEPRECATED_EXTERNALLY(3.12) static inline int +_PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) +{ + return PyUnstable_Code_GetExtra(code, index, extra); +} +_Py_DEPRECATED_EXTERNALLY(3.12) static inline int +_PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) +{ + return PyUnstable_Code_SetExtra(code, index, extra); +} /* Equivalent to getattr(code, 'co_code') in Python. Returns a strong reference to a bytes object. */ diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index 6127ba7babb80f..18b8ce913e6e31 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -13,8 +13,6 @@ extern "C" { and coroutine objects. */ #define _PyGenObject_HEAD(prefix) \ PyObject_HEAD \ - /* The code object backing the generator */ \ - PyCodeObject *prefix##_code; \ /* List of weak reference. */ \ PyObject *prefix##_weakreflist; \ /* Name of the generator. */ \ @@ -28,7 +26,7 @@ extern "C" { char prefix##_running_async; \ /* The frame */ \ int8_t prefix##_frame_state; \ - PyObject *prefix##_iframe[1]; + PyObject *prefix##_iframe[1]; \ typedef struct { /* The gi_ prefix is intended to remind of generator-iterator. */ @@ -46,6 +44,7 @@ PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self); +PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); /* --- PyCoroObject ------------------------------------------------------- */ diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index 67189cf0ade665..fb524770efed7c 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -24,10 +24,11 @@ Bigint { #ifdef Py_USING_MEMORY_DEBUGGER -struct _dtoa_runtime_state { +struct _dtoa_state { int _not_used; }; -#define _dtoa_runtime_state_INIT {0} +#define _dtoa_interp_state_INIT(INTERP) \ + {0} #else // !Py_USING_MEMORY_DEBUGGER @@ -40,7 +41,7 @@ struct _dtoa_runtime_state { #define Bigint_PREALLOC_SIZE \ ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) -struct _dtoa_runtime_state { +struct _dtoa_state { /* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ // XXX This should be freed during runtime fini. struct Bigint *p5s; @@ -48,9 +49,9 @@ struct _dtoa_runtime_state { double preallocated[Bigint_PREALLOC_SIZE]; double *preallocated_next; }; -#define _dtoa_runtime_state_INIT(runtime) \ +#define _dtoa_state_INIT(INTERP) \ { \ - .preallocated_next = runtime.dtoa.preallocated, \ + .preallocated_next = (INTERP)->dtoa.preallocated, \ } #endif // !Py_USING_MEMORY_DEBUGGER diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 81d16b219c305b..5806cf05f174a9 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -209,7 +209,7 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) * frames like the ones in generators and coroutines. */ void -_PyFrame_Clear(_PyInterpreterFrame * frame); +_PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index d0461fa7e82e8b..30c7c4e3bbd067 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -27,8 +27,6 @@ extern "C" { _PyRuntime.cached_objects.NAME struct _Py_cached_objects { - PyObject *str_replace_inf; - PyObject *interned_strings; }; @@ -67,11 +65,14 @@ struct _Py_static_objects { (interp)->cached_objects.NAME struct _Py_interp_cached_objects { - int _not_set; + /* AST */ + PyObject *str_replace_inf; + /* object.__reduce__ */ PyObject *objreduce; PyObject *type_slots_pname; pytype_slotdef *type_slots_ptrs[MAX_EQUIV]; + }; #define _Py_INTERP_STATIC_OBJECT(interp, NAME) \ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 60de31b336f613..7ef9c40153e421 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -16,6 +16,7 @@ extern "C" { #include "pycore_code.h" // struct callable_cache #include "pycore_context.h" // struct _Py_context_state #include "pycore_dict_state.h" // struct _Py_dict_state +#include "pycore_dtoa.h" // struct _dtoa_state #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_floatobject.h" // struct _Py_float_state #include "pycore_function.h" // FUNC_MAX_WATCHERS @@ -139,6 +140,7 @@ struct _is { struct _Py_unicode_state unicode; struct _Py_float_state float_state; struct _Py_long_state long_state; + struct _dtoa_state dtoa; /* Using a cache is very effective since typically only a single slice is created and then deleted again. */ PySliceObject *slice_cache; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 9ef270791576e3..2350eaab5976ca 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -11,7 +11,6 @@ extern "C" { #include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_ceval_state.h" // struct _ceval_runtime_state #include "pycore_dict_state.h" // struct _Py_dict_runtime_state -#include "pycore_dtoa.h" // struct _dtoa_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_function.h" // struct _func_runtime_state @@ -141,7 +140,6 @@ typedef struct pyruntimestate { struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; struct _getargs_runtime_state getargs; - struct _dtoa_runtime_state dtoa; struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index a8d5953ff98b0b..b54adf04761d4e 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -53,7 +53,6 @@ extern "C" { .gilstate = { \ .check_enabled = 1, \ }, \ - .dtoa = _dtoa_runtime_state_INIT(runtime), \ .fileutils = { \ .force_ascii = -1, \ }, \ @@ -94,10 +93,10 @@ extern "C" { }, \ }, \ }, \ - ._main_interpreter = _PyInterpreterState_INIT, \ + ._main_interpreter = _PyInterpreterState_INIT(runtime._main_interpreter), \ } -#define _PyInterpreterState_INIT \ +#define _PyInterpreterState_INIT(INTERP) \ { \ .id_refcount = -1, \ .imports = IMPORTS_INIT, \ @@ -113,6 +112,7 @@ extern "C" { { .threshold = 10, }, \ }, \ }, \ + .dtoa = _dtoa_state_INIT(&(INTERP)), \ .static_objects = { \ .singletons = { \ ._not_used = 1, \ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 4d705740a9a62b..cc5ce2875101ea 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -4,6 +4,8 @@ extern "C" { #endif +#include "pycore_moduleobject.h" + #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif @@ -62,6 +64,20 @@ _PyStaticType_GET_WEAKREFS_LISTPTR(static_builtin_state *state) return &state->tp_weaklist; } +/* Like PyType_GetModuleState, but skips verification + * that type is a heap type with an associated module */ +static inline void * +_PyType_GetModuleState(PyTypeObject *type) +{ + assert(PyType_Check(type)); + assert(type->tp_flags & Py_TPFLAGS_HEAPTYPE); + PyHeapTypeObject *et = (PyHeapTypeObject *)type; + assert(et->ht_module); + PyModuleObject *mod = (PyModuleObject *)(et->ht_module); + assert(mod != NULL); + return mod->md_state; +} + struct types_state { struct type_cache type_cache; size_t num_builtins_initialized; diff --git a/Include/pyport.h b/Include/pyport.h index 40092c2f81ad48..eef0fe1bfd71d8 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -323,6 +323,15 @@ extern "C" { #define Py_DEPRECATED(VERSION_UNUSED) #endif +// _Py_DEPRECATED_EXTERNALLY(version) +// Deprecated outside CPython core. +#ifdef Py_BUILD_CORE +#define _Py_DEPRECATED_EXTERNALLY(VERSION_UNUSED) +#else +#define _Py_DEPRECATED_EXTERNALLY(version) Py_DEPRECATED(version) +#endif + + #if defined(__clang__) #define _Py_COMP_DIAG_PUSH _Pragma("clang diagnostic push") #define _Py_COMP_DIAG_IGNORE_DEPR_DECLS \ diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 052b7bca28ddd3..00e77749e25e77 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -11,7 +11,7 @@ __all__ = ["version", "bootstrap"] _PACKAGE_NAMES = ('setuptools', 'pip') _SETUPTOOLS_VERSION = "65.5.0" -_PIP_VERSION = "23.0" +_PIP_VERSION = "23.0.1" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION, "py3"), ("pip", _PIP_VERSION, "py3"), diff --git a/Lib/ensurepip/_bundled/pip-23.0-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-23.0.1-py3-none-any.whl similarity index 93% rename from Lib/ensurepip/_bundled/pip-23.0-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-23.0.1-py3-none-any.whl index bb9aebf474cfe9..a855dc40e8630d 100644 Binary files a/Lib/ensurepip/_bundled/pip-23.0-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-23.0.1-py3-none-any.whl differ diff --git a/Lib/fileinput.py b/Lib/fileinput.py index e234dc9ea65f15..1b25f28f3d3432 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -399,7 +399,7 @@ def isstdin(self): def hook_compressed(filename, mode, *, encoding=None, errors=None): - if encoding is None: # EncodingWarning is emitted in FileInput() already. + if encoding is None and "b" not in mode: # EncodingWarning is emitted in FileInput() already. encoding = "locale" ext = os.path.splitext(filename)[1] if ext == '.gz': diff --git a/Lib/fractions.py b/Lib/fractions.py index 49a3f2841a2ed4..c95db0730e5b6d 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -183,7 +183,7 @@ class Fraction(numbers.Rational): __slots__ = ('_numerator', '_denominator') # We're immutable, so use __new__ not __init__ - def __new__(cls, numerator=0, denominator=None, *, _normalize=True): + def __new__(cls, numerator=0, denominator=None): """Constructs a Rational. Takes a string like '3/2' or '1.5', another Rational instance, a @@ -279,12 +279,11 @@ def __new__(cls, numerator=0, denominator=None, *, _normalize=True): if denominator == 0: raise ZeroDivisionError('Fraction(%s, 0)' % numerator) - if _normalize: - g = math.gcd(numerator, denominator) - if denominator < 0: - g = -g - numerator //= g - denominator //= g + g = math.gcd(numerator, denominator) + if denominator < 0: + g = -g + numerator //= g + denominator //= g self._numerator = numerator self._denominator = denominator return self @@ -301,7 +300,7 @@ def from_float(cls, f): elif not isinstance(f, float): raise TypeError("%s.from_float() only takes floats, not %r (%s)" % (cls.__name__, f, type(f).__name__)) - return cls(*f.as_integer_ratio()) + return cls._from_coprime_ints(*f.as_integer_ratio()) @classmethod def from_decimal(cls, dec): @@ -313,17 +312,28 @@ def from_decimal(cls, dec): raise TypeError( "%s.from_decimal() only takes Decimals, not %r (%s)" % (cls.__name__, dec, type(dec).__name__)) - return cls(*dec.as_integer_ratio()) + return cls._from_coprime_ints(*dec.as_integer_ratio()) + + @classmethod + def _from_coprime_ints(cls, numerator, denominator, /): + """Convert a pair of ints to a rational number, for internal use. + + The ratio of integers should be in lowest terms and the denominator + should be positive. + """ + obj = super(Fraction, cls).__new__(cls) + obj._numerator = numerator + obj._denominator = denominator + return obj def is_integer(self): """Return True if the Fraction is an integer.""" return self._denominator == 1 def as_integer_ratio(self): - """Return the integer ratio as a tuple. + """Return a pair of integers, whose ratio is equal to the original Fraction. - Return a tuple of two integers, whose ratio is equal to the - Fraction and with a positive denominator. + The ratio is in lowest terms and has a positive denominator. """ return (self._numerator, self._denominator) @@ -380,9 +390,9 @@ def limit_denominator(self, max_denominator=1000000): # the distance from p1/q1 to self is d/(q1*self._denominator). So we # need to compare 2*(q0+k*q1) with self._denominator/d. if 2*d*(q0+k*q1) <= self._denominator: - return Fraction(p1, q1, _normalize=False) + return Fraction._from_coprime_ints(p1, q1) else: - return Fraction(p0+k*p1, q0+k*q1, _normalize=False) + return Fraction._from_coprime_ints(p0+k*p1, q0+k*q1) @property def numerator(a): @@ -703,13 +713,13 @@ def _add(a, b): nb, db = b._numerator, b._denominator g = math.gcd(da, db) if g == 1: - return Fraction(na * db + da * nb, da * db, _normalize=False) + return Fraction._from_coprime_ints(na * db + da * nb, da * db) s = da // g t = na * (db // g) + nb * s g2 = math.gcd(t, g) if g2 == 1: - return Fraction(t, s * db, _normalize=False) - return Fraction(t // g2, s * (db // g2), _normalize=False) + return Fraction._from_coprime_ints(t, s * db) + return Fraction._from_coprime_ints(t // g2, s * (db // g2)) __add__, __radd__ = _operator_fallbacks(_add, operator.add) @@ -719,13 +729,13 @@ def _sub(a, b): nb, db = b._numerator, b._denominator g = math.gcd(da, db) if g == 1: - return Fraction(na * db - da * nb, da * db, _normalize=False) + return Fraction._from_coprime_ints(na * db - da * nb, da * db) s = da // g t = na * (db // g) - nb * s g2 = math.gcd(t, g) if g2 == 1: - return Fraction(t, s * db, _normalize=False) - return Fraction(t // g2, s * (db // g2), _normalize=False) + return Fraction._from_coprime_ints(t, s * db) + return Fraction._from_coprime_ints(t // g2, s * (db // g2)) __sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub) @@ -741,15 +751,17 @@ def _mul(a, b): if g2 > 1: nb //= g2 da //= g2 - return Fraction(na * nb, db * da, _normalize=False) + return Fraction._from_coprime_ints(na * nb, db * da) __mul__, __rmul__ = _operator_fallbacks(_mul, operator.mul) def _div(a, b): """a / b""" # Same as _mul(), with inversed b. - na, da = a._numerator, a._denominator nb, db = b._numerator, b._denominator + if nb == 0: + raise ZeroDivisionError('Fraction(%s, 0)' % db) + na, da = a._numerator, a._denominator g1 = math.gcd(na, nb) if g1 > 1: na //= g1 @@ -761,7 +773,7 @@ def _div(a, b): n, d = na * db, nb * da if d < 0: n, d = -n, -d - return Fraction(n, d, _normalize=False) + return Fraction._from_coprime_ints(n, d) __truediv__, __rtruediv__ = _operator_fallbacks(_div, operator.truediv) @@ -798,17 +810,17 @@ def __pow__(a, b): if b.denominator == 1: power = b.numerator if power >= 0: - return Fraction(a._numerator ** power, - a._denominator ** power, - _normalize=False) - elif a._numerator >= 0: - return Fraction(a._denominator ** -power, - a._numerator ** -power, - _normalize=False) + return Fraction._from_coprime_ints(a._numerator ** power, + a._denominator ** power) + elif a._numerator > 0: + return Fraction._from_coprime_ints(a._denominator ** -power, + a._numerator ** -power) + elif a._numerator == 0: + raise ZeroDivisionError('Fraction(%s, 0)' % + a._denominator ** -power) else: - return Fraction((-a._denominator) ** -power, - (-a._numerator) ** -power, - _normalize=False) + return Fraction._from_coprime_ints((-a._denominator) ** -power, + (-a._numerator) ** -power) else: # A fractional power will generally produce an # irrational number. @@ -832,15 +844,15 @@ def __rpow__(b, a): def __pos__(a): """+a: Coerces a subclass instance to Fraction""" - return Fraction(a._numerator, a._denominator, _normalize=False) + return Fraction._from_coprime_ints(a._numerator, a._denominator) def __neg__(a): """-a""" - return Fraction(-a._numerator, a._denominator, _normalize=False) + return Fraction._from_coprime_ints(-a._numerator, a._denominator) def __abs__(a): """abs(a)""" - return Fraction(abs(a._numerator), a._denominator, _normalize=False) + return Fraction._from_coprime_ints(abs(a._numerator), a._denominator) def __int__(a, _index=operator.index): """int(a)""" diff --git a/Lib/functools.py b/Lib/functools.py index 43ead512e1ea4e..aaf4291150fbbf 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -959,15 +959,12 @@ def __isabstractmethod__(self): ### cached_property() - computed once per instance, cached as attribute ################################################################################ -_NOT_FOUND = object() - class cached_property: def __init__(self, func): self.func = func self.attrname = None self.__doc__ = func.__doc__ - self.lock = RLock() def __set_name__(self, owner, name): if self.attrname is None: @@ -992,21 +989,15 @@ def __get__(self, instance, owner=None): f"instance to cache {self.attrname!r} property." ) raise TypeError(msg) from None - val = cache.get(self.attrname, _NOT_FOUND) - if val is _NOT_FOUND: - with self.lock: - # check if another thread filled cache while we awaited lock - val = cache.get(self.attrname, _NOT_FOUND) - if val is _NOT_FOUND: - val = self.func(instance) - try: - cache[self.attrname] = val - except TypeError: - msg = ( - f"The '__dict__' attribute on {type(instance).__name__!r} instance " - f"does not support item assignment for caching {self.attrname!r} property." - ) - raise TypeError(msg) from None + val = self.func(instance) + try: + cache[self.attrname] = val + except TypeError: + msg = ( + f"The '__dict__' attribute on {type(instance).__name__!r} instance " + f"does not support item assignment for caching {self.attrname!r} property." + ) + raise TypeError(msg) from None return val __class_getitem__ = classmethod(GenericAlias) diff --git a/Lib/idlelib/dynoption.py b/Lib/idlelib/dynoption.py index 9c6ffa435a1089..d5dfc3eda13f60 100644 --- a/Lib/idlelib/dynoption.py +++ b/Lib/idlelib/dynoption.py @@ -2,24 +2,19 @@ OptionMenu widget modified to allow dynamic menu reconfiguration and setting of highlightthickness """ -import copy - from tkinter import OptionMenu, _setit, StringVar, Button class DynOptionMenu(OptionMenu): - """ - unlike OptionMenu, our kwargs can include highlightthickness + """Add SetMenu and highlightthickness to OptionMenu. + + Highlightthickness adds space around menu button. """ def __init__(self, master, variable, value, *values, **kwargs): - # TODO copy value instead of whole dict - kwargsCopy=copy.copy(kwargs) - if 'highlightthickness' in list(kwargs.keys()): - del(kwargs['highlightthickness']) + highlightthickness = kwargs.pop('highlightthickness', None) OptionMenu.__init__(self, master, variable, value, *values, **kwargs) - self.config(highlightthickness=kwargsCopy.get('highlightthickness')) - #self.menu=self['menu'] - self.variable=variable - self.command=kwargs.get('command') + self['highlightthickness'] = highlightthickness + self.variable = variable + self.command = kwargs.get('command') def SetMenu(self,valueList,value=None): """ @@ -38,14 +33,15 @@ def _dyn_option_menu(parent): # htest # from tkinter import Toplevel # + StringVar, Button top = Toplevel(parent) - top.title("Tets dynamic option menu") + top.title("Test dynamic option menu") x, y = map(int, parent.geometry().split('+')[1:]) top.geometry("200x100+%d+%d" % (x + 250, y + 175)) top.focus_set() var = StringVar(top) var.set("Old option set") #Set the default value - dyn = DynOptionMenu(top,var, "old1","old2","old3","old4") + dyn = DynOptionMenu(top, var, "old1","old2","old3","old4", + highlightthickness=5) dyn.pack() def update(): @@ -54,5 +50,6 @@ def update(): button.pack() if __name__ == '__main__': + # Only module without unittests because of intention to replace. from idlelib.idle_test.htest import run run(_dyn_option_menu) diff --git a/Lib/imp.py b/Lib/imp.py index fc42c15765852e..fe850f6a001814 100644 --- a/Lib/imp.py +++ b/Lib/imp.py @@ -338,8 +338,8 @@ def load_dynamic(name, path, file=None): # Issue #24748: Skip the sys.modules check in _load_module_shim; # always load new extension - spec = importlib.machinery.ModuleSpec( - name=name, loader=loader, origin=path) + spec = importlib.util.spec_from_file_location( + name, path, loader=loader) return _load(spec) else: diff --git a/Lib/platform.py b/Lib/platform.py index 2dfaf76252db51..f2b0d1d1bd3f5d 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1246,7 +1246,7 @@ def python_compiler(): _platform_cache = {} -def platform(aliased=0, terse=0): +def platform(aliased=False, terse=False): """ Returns a single string identifying the underlying platform with as much useful information as possible (but no more :). diff --git a/Lib/test/cmath_testcases.txt b/Lib/test/cmath_testcases.txt index dd7e458ddcb7b1..0165e17634f41c 100644 --- a/Lib/test/cmath_testcases.txt +++ b/Lib/test/cmath_testcases.txt @@ -1536,6 +1536,7 @@ sqrt0141 sqrt -1.797e+308 -9.9999999999999999e+306 -> 3.7284476432057307e+152 -1 sqrt0150 sqrt 1.7976931348623157e+308 0.0 -> 1.3407807929942596355e+154 0.0 sqrt0151 sqrt 2.2250738585072014e-308 0.0 -> 1.4916681462400413487e-154 0.0 sqrt0152 sqrt 5e-324 0.0 -> 2.2227587494850774834e-162 0.0 +sqrt0153 sqrt 5e-324 1.0 -> 0.7071067811865476 0.7071067811865476 -- special values sqrt1000 sqrt 0.0 0.0 -> 0.0 0.0 @@ -1744,6 +1745,7 @@ cosh0023 cosh 2.218885944363501 2.0015727395883687 -> -1.94294321081968 4.129026 -- large real part cosh0030 cosh 710.5 2.3519999999999999 -> -1.2967465239355998e+308 1.3076707908857333e+308 cosh0031 cosh -710.5 0.69999999999999996 -> 1.4085466381392499e+308 -1.1864024666450239e+308 +cosh0032 cosh 720.0 0.0 -> inf 0.0 overflow -- Additional real values (mpmath) cosh0050 cosh 1e-150 0.0 -> 1.0 0.0 @@ -1853,6 +1855,7 @@ sinh0023 sinh 0.043713693678420068 0.22512549887532657 -> 0.042624198673416713 0 -- large real part sinh0030 sinh 710.5 -2.3999999999999999 -> -1.3579970564885919e+308 -1.24394470907798e+308 sinh0031 sinh -710.5 0.80000000000000004 -> -1.2830671601735164e+308 1.3210954193997678e+308 +sinh0032 sinh 720.0 0.0 -> inf 0.0 overflow -- Additional real values (mpmath) sinh0050 sinh 1e-100 0.0 -> 1.00000000000000002e-100 0.0 diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 65ae7a227baafe..190fe8723b1fb5 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -50,18 +50,13 @@ class CompilationStepTestCase(unittest.TestCase): HAS_TARGET = set(dis.hasjrel + dis.hasjabs + dis.hasexc) HAS_ARG_OR_TARGET = HAS_ARG.union(HAS_TARGET) - def setUp(self): - self.last_label = 0 - - def Label(self): - self.last_label += 1 - return self.last_label + class Label: + pass def assertInstructionsMatch(self, actual_, expected_): # get two lists where each entry is a label or - # an instruction tuple. Compare them, while mapping - # each actual label to a corresponding expected label - # based on their locations. + # an instruction tuple. Normalize the labels to the + # instruction count of the target, and compare the lists. self.assertIsInstance(actual_, list) self.assertIsInstance(expected_, list) @@ -82,39 +77,35 @@ def assertInstructionsMatch(self, actual_, expected_): act = act[:len(exp)] self.assertEqual(exp, act) + def resolveAndRemoveLabels(self, insts): + idx = 0 + res = [] + for item in insts: + assert isinstance(item, (self.Label, tuple)) + if isinstance(item, self.Label): + item.value = idx + else: + idx += 1 + res.append(item) + + return res + def normalize_insts(self, insts): """ Map labels to instruction index. - Remove labels which are not used as jump targets. Map opcodes to opnames. """ - labels_map = {} - targets = set() - idx = 1 - for item in insts: - assert isinstance(item, (int, tuple)) - if isinstance(item, tuple): - opcode, oparg, *_ = item - if dis.opmap.get(opcode, opcode) in self.HAS_TARGET: - targets.add(oparg) - idx += 1 - elif isinstance(item, int): - assert item not in labels_map, "label reused" - labels_map[item] = idx - + insts = self.resolveAndRemoveLabels(insts) res = [] for item in insts: - if isinstance(item, int) and item in targets: - if not res or labels_map[item] != res[-1]: - res.append(labels_map[item]) - elif isinstance(item, tuple): - opcode, oparg, *loc = item - opcode = dis.opmap.get(opcode, opcode) - if opcode in self.HAS_TARGET: - arg = labels_map[oparg] - else: - arg = oparg if opcode in self.HAS_TARGET else None - opcode = dis.opname[opcode] - res.append((opcode, arg, *loc)) + assert isinstance(item, tuple) + opcode, oparg, *loc = item + opcode = dis.opmap.get(opcode, opcode) + if isinstance(oparg, self.Label): + arg = oparg.value + else: + arg = oparg if opcode in self.HAS_ARG else None + opcode = dis.opname[opcode] + res.append((opcode, arg, *loc)) return res @@ -129,20 +120,18 @@ class CfgOptimizationTestCase(CompilationStepTestCase): def complete_insts_info(self, insts): # fill in omitted fields in location, and oparg 0 for ops with no arg. - instructions = [] + res = [] for item in insts: - if isinstance(item, int): - instructions.append(item) - else: - assert isinstance(item, tuple) - inst = list(reversed(item)) - opcode = dis.opmap[inst.pop()] - oparg = inst.pop() if opcode in self.HAS_ARG_OR_TARGET else 0 - loc = inst + [-1] * (4 - len(inst)) - instructions.append((opcode, oparg, *loc)) - return instructions + assert isinstance(item, tuple) + inst = list(reversed(item)) + opcode = dis.opmap[inst.pop()] + oparg = inst.pop() if opcode in self.HAS_ARG_OR_TARGET else 0 + loc = inst + [-1] * (4 - len(inst)) + res.append((opcode, oparg, *loc)) + return res def get_optimized(self, insts, consts): + insts = self.normalize_insts(insts) insts = self.complete_insts_info(insts) insts = optimize_cfg(insts, consts) return insts, consts diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index f46f21da8da351..916e22a527a8e0 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -40,6 +40,12 @@ def test_float(self): self.assertEqual(float(True), 1.0) self.assertIsNot(float(True), True) + def test_complex(self): + self.assertEqual(complex(False), 0j) + self.assertEqual(complex(False), False) + self.assertEqual(complex(True), 1+0j) + self.assertEqual(complex(True), True) + def test_math(self): self.assertEqual(+False, 0) self.assertIsNot(+False, False) @@ -313,6 +319,26 @@ def __len__(self): return -1 self.assertRaises(ValueError, bool, Eggs()) + def test_interpreter_convert_to_bool_raises(self): + class SymbolicBool: + def __bool__(self): + raise TypeError + + class Symbol: + def __gt__(self, other): + return SymbolicBool() + + x = Symbol() + + with self.assertRaises(TypeError): + if x > 0: + msg = "x > 0 was true" + else: + msg = "x > 0 was false" + + # This used to create negative refcounts, see gh-102250 + del x + def test_from_bytes(self): self.assertIs(bool.from_bytes(b'\x00'*8, 'big'), False) self.assertIs(bool.from_bytes(b'abcd', 'little'), True) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index c97ed1cea0d113..e4dd7fc2100b62 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -844,6 +844,10 @@ def test_refleaks_in___init__(self): bzd.__init__() self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + def test_uninitialized_BZ2Decompressor_crash(self): + self.assertEqual(BZ2Decompressor.__new__(BZ2Decompressor). + decompress(bytes()), b'') + class CompressDecompressTest(BaseTest): def testCompress(self): diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py new file mode 100644 index 00000000000000..b543a1a565a56f --- /dev/null +++ b/Lib/test/test_capi/test_exceptions.py @@ -0,0 +1,145 @@ +import re +import sys +import unittest + +from test import support +from test.support import import_helper +from test.support.script_helper import assert_python_failure + +from .test_misc import decode_stderr + +# Skip this test if the _testcapi module isn't available. +_testcapi = import_helper.import_module('_testcapi') + +class Test_Exceptions(unittest.TestCase): + + def test_exception(self): + raised_exception = ValueError("5") + new_exc = TypeError("TEST") + try: + raise raised_exception + except ValueError as e: + orig_sys_exception = sys.exception() + orig_exception = _testcapi.set_exception(new_exc) + new_sys_exception = sys.exception() + new_exception = _testcapi.set_exception(orig_exception) + reset_sys_exception = sys.exception() + + self.assertEqual(orig_exception, e) + + self.assertEqual(orig_exception, raised_exception) + self.assertEqual(orig_sys_exception, orig_exception) + self.assertEqual(reset_sys_exception, orig_exception) + self.assertEqual(new_exception, new_exc) + self.assertEqual(new_sys_exception, new_exception) + else: + self.fail("Exception not raised") + + def test_exc_info(self): + raised_exception = ValueError("5") + new_exc = TypeError("TEST") + try: + raise raised_exception + except ValueError as e: + tb = e.__traceback__ + orig_sys_exc_info = sys.exc_info() + orig_exc_info = _testcapi.set_exc_info(new_exc.__class__, new_exc, None) + new_sys_exc_info = sys.exc_info() + new_exc_info = _testcapi.set_exc_info(*orig_exc_info) + reset_sys_exc_info = sys.exc_info() + + self.assertEqual(orig_exc_info[1], e) + + self.assertSequenceEqual(orig_exc_info, (raised_exception.__class__, raised_exception, tb)) + self.assertSequenceEqual(orig_sys_exc_info, orig_exc_info) + self.assertSequenceEqual(reset_sys_exc_info, orig_exc_info) + self.assertSequenceEqual(new_exc_info, (new_exc.__class__, new_exc, None)) + self.assertSequenceEqual(new_sys_exc_info, new_exc_info) + else: + self.assertTrue(False) + + +class Test_FatalError(unittest.TestCase): + + def check_fatal_error(self, code, expected, not_expected=()): + with support.SuppressCrashReport(): + rc, out, err = assert_python_failure('-sSI', '-c', code) + + err = decode_stderr(err) + self.assertIn('Fatal Python error: test_fatal_error: MESSAGE\n', + err) + + match = re.search(r'^Extension modules:(.*) \(total: ([0-9]+)\)$', + err, re.MULTILINE) + if not match: + self.fail(f"Cannot find 'Extension modules:' in {err!r}") + modules = set(match.group(1).strip().split(', ')) + total = int(match.group(2)) + + for name in expected: + self.assertIn(name, modules) + for name in not_expected: + self.assertNotIn(name, modules) + self.assertEqual(len(modules), total) + + @support.requires_subprocess() + def test_fatal_error(self): + # By default, stdlib extension modules are ignored, + # but not test modules. + expected = ('_testcapi',) + not_expected = ('sys',) + code = 'import _testcapi, sys; _testcapi.fatal_error(b"MESSAGE")' + self.check_fatal_error(code, expected, not_expected) + + # Mark _testcapi as stdlib module, but not sys + expected = ('sys',) + not_expected = ('_testcapi',) + code = """if True: + import _testcapi, sys + sys.stdlib_module_names = frozenset({"_testcapi"}) + _testcapi.fatal_error(b"MESSAGE") + """ + self.check_fatal_error(code, expected) + + +class Test_ErrSetAndRestore(unittest.TestCase): + + def test_err_set_raised(self): + with self.assertRaises(ValueError): + _testcapi.err_set_raised(ValueError()) + v = ValueError() + try: + _testcapi.err_set_raised(v) + except ValueError as ex: + self.assertIs(v, ex) + + def test_err_restore(self): + with self.assertRaises(ValueError): + _testcapi.err_restore(ValueError) + with self.assertRaises(ValueError): + _testcapi.err_restore(ValueError, 1) + with self.assertRaises(ValueError): + _testcapi.err_restore(ValueError, 1, None) + with self.assertRaises(ValueError): + _testcapi.err_restore(ValueError, ValueError()) + try: + _testcapi.err_restore(KeyError, "hi") + except KeyError as k: + self.assertEqual("hi", k.args[0]) + try: + 1/0 + except Exception as e: + tb = e.__traceback__ + with self.assertRaises(ValueError): + _testcapi.err_restore(ValueError, 1, tb) + with self.assertRaises(TypeError): + _testcapi.err_restore(ValueError, 1, 0) + try: + _testcapi.err_restore(ValueError, 1, tb) + except ValueError as v: + self.assertEqual(1, v.args[0]) + self.assertIs(tb, v.__traceback__.tb_next) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index ad099c61463b66..c34ee578b5c83f 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -8,7 +8,6 @@ import os import pickle import random -import re import subprocess import sys import textwrap @@ -91,51 +90,6 @@ def test_no_FatalError_infinite_loop(self): def test_memoryview_from_NULL_pointer(self): self.assertRaises(ValueError, _testcapi.make_memoryview_from_NULL_pointer) - def test_exception(self): - raised_exception = ValueError("5") - new_exc = TypeError("TEST") - try: - raise raised_exception - except ValueError as e: - orig_sys_exception = sys.exception() - orig_exception = _testcapi.set_exception(new_exc) - new_sys_exception = sys.exception() - new_exception = _testcapi.set_exception(orig_exception) - reset_sys_exception = sys.exception() - - self.assertEqual(orig_exception, e) - - self.assertEqual(orig_exception, raised_exception) - self.assertEqual(orig_sys_exception, orig_exception) - self.assertEqual(reset_sys_exception, orig_exception) - self.assertEqual(new_exception, new_exc) - self.assertEqual(new_sys_exception, new_exception) - else: - self.fail("Exception not raised") - - def test_exc_info(self): - raised_exception = ValueError("5") - new_exc = TypeError("TEST") - try: - raise raised_exception - except ValueError as e: - tb = e.__traceback__ - orig_sys_exc_info = sys.exc_info() - orig_exc_info = _testcapi.set_exc_info(new_exc.__class__, new_exc, None) - new_sys_exc_info = sys.exc_info() - new_exc_info = _testcapi.set_exc_info(*orig_exc_info) - reset_sys_exc_info = sys.exc_info() - - self.assertEqual(orig_exc_info[1], e) - - self.assertSequenceEqual(orig_exc_info, (raised_exception.__class__, raised_exception, tb)) - self.assertSequenceEqual(orig_sys_exc_info, orig_exc_info) - self.assertSequenceEqual(reset_sys_exc_info, orig_exc_info) - self.assertSequenceEqual(new_exc_info, (new_exc.__class__, new_exc, None)) - self.assertSequenceEqual(new_sys_exc_info, new_exc_info) - else: - self.assertTrue(False) - @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_seq_bytes_to_charp_array(self): # Issue #15732: crash in _PySequence_BytesToCharpArray() @@ -837,46 +791,6 @@ def __index__(self): self.assertRaises(TypeError, pynumber_tobase, '123', 10) self.assertRaises(SystemError, pynumber_tobase, 123, 0) - def check_fatal_error(self, code, expected, not_expected=()): - with support.SuppressCrashReport(): - rc, out, err = assert_python_failure('-sSI', '-c', code) - - err = decode_stderr(err) - self.assertIn('Fatal Python error: test_fatal_error: MESSAGE\n', - err) - - match = re.search(r'^Extension modules:(.*) \(total: ([0-9]+)\)$', - err, re.MULTILINE) - if not match: - self.fail(f"Cannot find 'Extension modules:' in {err!r}") - modules = set(match.group(1).strip().split(', ')) - total = int(match.group(2)) - - for name in expected: - self.assertIn(name, modules) - for name in not_expected: - self.assertNotIn(name, modules) - self.assertEqual(len(modules), total) - - @support.requires_subprocess() - def test_fatal_error(self): - # By default, stdlib extension modules are ignored, - # but not test modules. - expected = ('_testcapi',) - not_expected = ('sys',) - code = 'import _testcapi, sys; _testcapi.fatal_error(b"MESSAGE")' - self.check_fatal_error(code, expected, not_expected) - - # Mark _testcapi as stdlib module, but not sys - expected = ('sys',) - not_expected = ('_testcapi',) - code = textwrap.dedent(''' - import _testcapi, sys - sys.stdlib_module_names = frozenset({"_testcapi"}) - _testcapi.fatal_error(b"MESSAGE") - ''') - self.check_fatal_error(code, expected) - def test_pyobject_repr_from_null(self): s = _testcapi.pyobject_repr_from_null() self.assertEqual(s, '') @@ -1213,6 +1127,11 @@ def test_pendingcalls_non_threaded(self): self.pendingcalls_submit(l, n) self.pendingcalls_wait(l, n) + def test_gen_get_code(self): + def genf(): yield + gen = genf() + self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) + class SubinterpreterTest(unittest.TestCase): @@ -1636,44 +1555,5 @@ def func2(x=None): self.do_test(func2) -class Test_ErrSetAndRestore(unittest.TestCase): - - def test_err_set_raised(self): - with self.assertRaises(ValueError): - _testcapi.err_set_raised(ValueError()) - v = ValueError() - try: - _testcapi.err_set_raised(v) - except ValueError as ex: - self.assertIs(v, ex) - - def test_err_restore(self): - with self.assertRaises(ValueError): - _testcapi.err_restore(ValueError) - with self.assertRaises(ValueError): - _testcapi.err_restore(ValueError, 1) - with self.assertRaises(ValueError): - _testcapi.err_restore(ValueError, 1, None) - with self.assertRaises(ValueError): - _testcapi.err_restore(ValueError, ValueError()) - try: - _testcapi.err_restore(KeyError, "hi") - except KeyError as k: - self.assertEqual("hi", k.args[0]) - try: - 1/0 - except Exception as e: - tb = e.__traceback__ - with self.assertRaises(ValueError): - _testcapi.err_restore(ValueError, 1, tb) - with self.assertRaises(TypeError): - _testcapi.err_restore(ValueError, 1, 0) - try: - _testcapi.err_restore(ValueError, 1, tb) - except ValueError as v: - self.assertEqual(1, v.args[0]) - self.assertIs(tb, v.__traceback__.tb_next) - - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 9fa08dc4ff3fa7..cd2c6939105d40 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -607,6 +607,14 @@ def test_complex_near_zero(self): self.assertIsClose(0.001-0.001j, 0.001+0.001j, abs_tol=2e-03) self.assertIsNotClose(0.001-0.001j, 0.001+0.001j, abs_tol=1e-03) + def test_complex_special(self): + self.assertIsNotClose(INF, INF*1j) + self.assertIsNotClose(INF*1j, INF) + self.assertIsNotClose(INF, -INF) + self.assertIsNotClose(-INF, INF) + self.assertIsNotClose(0, INF) + self.assertIsNotClose(0, INF*1j) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 9c2ac83e1b69e3..0cd1fb3f9728e5 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -752,15 +752,15 @@ def f(): py = ctypes.pythonapi freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) - RequestCodeExtraIndex = py._PyEval_RequestCodeExtraIndex + RequestCodeExtraIndex = py.PyUnstable_Eval_RequestCodeExtraIndex RequestCodeExtraIndex.argtypes = (freefunc,) RequestCodeExtraIndex.restype = ctypes.c_ssize_t - SetExtra = py._PyCode_SetExtra + SetExtra = py.PyUnstable_Code_SetExtra SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) SetExtra.restype = ctypes.c_int - GetExtra = py._PyCode_GetExtra + GetExtra = py.PyUnstable_Code_GetExtra GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.POINTER(ctypes.c_voidp)) GetExtra.restype = ctypes.c_int diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index f2e14c1e628c01..022753e0c99483 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -37,11 +37,11 @@ def test_for_loop(self): ('GET_ITER', None, 1), loop_lbl := self.Label(), ('FOR_ITER', exit_lbl := self.Label(), 1), - ('STORE_NAME', None, 1), + ('STORE_NAME', 1, 1), ('PUSH_NULL', None, 2), - ('LOAD_NAME', None, 2), - ('LOAD_NAME', None, 2), - ('CALL', None, 2), + ('LOAD_NAME', 2, 2), + ('LOAD_NAME', 1, 2), + ('CALL', 1, 2), ('POP_TOP', None), ('JUMP', loop_lbl), exit_lbl, diff --git a/Lib/test/test_except_star.py b/Lib/test/test_except_star.py index 9de72dbd5a3264..c5167c5bba38af 100644 --- a/Lib/test/test_except_star.py +++ b/Lib/test/test_except_star.py @@ -208,44 +208,38 @@ def assertMetadataNotEqual(self, e1, e2): class TestExceptStarSplitSemantics(ExceptStarTest): def doSplitTestNamed(self, exc, T, match_template, rest_template): - initial_exc_info = sys.exc_info() - exc_info = match = rest = None + initial_sys_exception = sys.exception() + sys_exception = match = rest = None try: try: raise exc except* T as e: - exc_info = sys.exc_info() + sys_exception = sys.exception() match = e except BaseException as e: rest = e - if match_template: - self.assertEqual(exc_info[1], match) - else: - self.assertIsNone(exc_info) + self.assertEqual(sys_exception, match) self.assertExceptionIsLike(match, match_template) self.assertExceptionIsLike(rest, rest_template) - self.assertEqual(sys.exc_info(), initial_exc_info) + self.assertEqual(sys.exception(), initial_sys_exception) def doSplitTestUnnamed(self, exc, T, match_template, rest_template): - initial_exc_info = sys.exc_info() - exc_info = match = rest = None + initial_sys_exception = sys.exception() + sys_exception = match = rest = None try: try: raise exc except* T: - exc_info = sys.exc_info() - match = sys.exc_info()[1] + sys_exception = match = sys.exception() else: if rest_template: self.fail("Exception not raised") except BaseException as e: rest = e self.assertExceptionIsLike(match, match_template) - if match_template: - self.assertEqual(exc_info[0], type(match_template)) self.assertExceptionIsLike(rest, rest_template) - self.assertEqual(sys.exc_info(), initial_exc_info) + self.assertEqual(sys.exception(), initial_sys_exception) def doSplitTestInExceptHandler(self, exc, T, match_template, rest_template): try: @@ -409,11 +403,11 @@ def test_multiple_matches_unnamed(self): try: raise ExceptionGroup("mmu", [OSError("os"), BlockingIOError("io")]) except* BlockingIOError: - e = sys.exc_info()[1] + e = sys.exception() self.assertExceptionIsLike(e, ExceptionGroup("mmu", [BlockingIOError("io")])) except* OSError: - e = sys.exc_info()[1] + e = sys.exception() self.assertExceptionIsLike(e, ExceptionGroup("mmu", [OSError("os")])) else: @@ -434,7 +428,7 @@ def test_first_match_wins_unnamed(self): try: raise ExceptionGroup("fstu", [BlockingIOError("io")]) except* OSError: - e = sys.exc_info()[1] + e = sys.exception() self.assertExceptionIsLike(e, ExceptionGroup("fstu", [BlockingIOError("io")])) except* BlockingIOError: @@ -452,7 +446,7 @@ def test_nested_except_stars(self): pass else: self.fail("Exception not raised") - e = sys.exc_info()[1] + e = sys.exception() self.assertExceptionIsLike(e, ExceptionGroup("n", [BlockingIOError("io")])) else: @@ -766,7 +760,7 @@ def test_raise_unnamed(self): try: raise orig except* OSError: - e = sys.exc_info()[1] + e = sys.exception() raise TypeError(3) from e except ExceptionGroup as e: exc = e @@ -821,7 +815,7 @@ def test_raise_handle_all_raise_one_unnamed(self): try: raise orig except* (TypeError, ValueError) as e: - e = sys.exc_info()[1] + e = sys.exception() raise SyntaxError(3) from e except ExceptionGroup as e: exc = e @@ -882,10 +876,10 @@ def test_raise_handle_all_raise_two_unnamed(self): try: raise orig except* TypeError: - e = sys.exc_info()[1] + e = sys.exception() raise SyntaxError(3) from e except* ValueError: - e = sys.exc_info()[1] + e = sys.exception() raise SyntaxError(4) from e except ExceptionGroup as e: exc = e @@ -982,7 +976,7 @@ def derive(self, excs): class TestExceptStarCleanup(ExceptStarTest): - def test_exc_info_restored(self): + def test_sys_exception_restored(self): try: try: raise ValueError(42) @@ -997,7 +991,7 @@ def test_exc_info_restored(self): self.assertExceptionIsLike(exc, ZeroDivisionError('division by zero')) self.assertExceptionIsLike(exc.__context__, ValueError(42)) - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertEqual(sys.exception(), None) class TestExceptStar_WeirdLeafExceptions(ExceptStarTest): diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 113c7802821dd4..26de67d924272a 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -202,7 +202,7 @@ def test_fcntl_f_pipesize(self): pipesize_default = fcntl.fcntl(test_pipe_w, fcntl.F_GETPIPE_SZ) pipesize = pipesize_default // 2 # A new value to detect change. if pipesize < 512: # the POSIX minimum - raise unittest.SkitTest( + raise unittest.SkipTest( 'default pipesize too small to perform test.') fcntl.fcntl(test_pipe_w, fcntl.F_SETPIPE_SZ, pipesize) self.assertEqual(fcntl.fcntl(test_pipe_w, fcntl.F_GETPIPE_SZ), diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index ac20c74baa09e2..786d9186634305 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -855,29 +855,29 @@ def setUp(self): self.fake_open = InvocationRecorder() def test_empty_string(self): - self.do_test_use_builtin_open("", 1) + self.do_test_use_builtin_open_text("", "r") def test_no_ext(self): - self.do_test_use_builtin_open("abcd", 2) + self.do_test_use_builtin_open_text("abcd", "r") @unittest.skipUnless(gzip, "Requires gzip and zlib") def test_gz_ext_fake(self): original_open = gzip.open gzip.open = self.fake_open try: - result = fileinput.hook_compressed("test.gz", "3") + result = fileinput.hook_compressed("test.gz", "r") finally: gzip.open = original_open self.assertEqual(self.fake_open.invocation_count, 1) - self.assertEqual(self.fake_open.last_invocation, (("test.gz", "3"), {})) + self.assertEqual(self.fake_open.last_invocation, (("test.gz", "r"), {})) @unittest.skipUnless(gzip, "Requires gzip and zlib") def test_gz_with_encoding_fake(self): original_open = gzip.open gzip.open = lambda filename, mode: io.BytesIO(b'Ex-binary string') try: - result = fileinput.hook_compressed("test.gz", "3", encoding="utf-8") + result = fileinput.hook_compressed("test.gz", "r", encoding="utf-8") finally: gzip.open = original_open self.assertEqual(list(result), ['Ex-binary string']) @@ -887,23 +887,40 @@ def test_bz2_ext_fake(self): original_open = bz2.BZ2File bz2.BZ2File = self.fake_open try: - result = fileinput.hook_compressed("test.bz2", "4") + result = fileinput.hook_compressed("test.bz2", "r") finally: bz2.BZ2File = original_open self.assertEqual(self.fake_open.invocation_count, 1) - self.assertEqual(self.fake_open.last_invocation, (("test.bz2", "4"), {})) + self.assertEqual(self.fake_open.last_invocation, (("test.bz2", "r"), {})) def test_blah_ext(self): - self.do_test_use_builtin_open("abcd.blah", "5") + self.do_test_use_builtin_open_binary("abcd.blah", "rb") def test_gz_ext_builtin(self): - self.do_test_use_builtin_open("abcd.Gz", "6") + self.do_test_use_builtin_open_binary("abcd.Gz", "rb") def test_bz2_ext_builtin(self): - self.do_test_use_builtin_open("abcd.Bz2", "7") + self.do_test_use_builtin_open_binary("abcd.Bz2", "rb") - def do_test_use_builtin_open(self, filename, mode): + def test_binary_mode_encoding(self): + self.do_test_use_builtin_open_binary("abcd", "rb") + + def test_text_mode_encoding(self): + self.do_test_use_builtin_open_text("abcd", "r") + + def do_test_use_builtin_open_binary(self, filename, mode): + original_open = self.replace_builtin_open(self.fake_open) + try: + result = fileinput.hook_compressed(filename, mode) + finally: + self.replace_builtin_open(original_open) + + self.assertEqual(self.fake_open.invocation_count, 1) + self.assertEqual(self.fake_open.last_invocation, + ((filename, mode), {'encoding': None, 'errors': None})) + + def do_test_use_builtin_open_text(self, filename, mode): original_open = self.replace_builtin_open(self.fake_open) try: result = fileinput.hook_compressed(filename, mode) diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 3bc6b409e05dc3..e112f49d2e7944 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -488,6 +488,7 @@ def testArithmetic(self): self.assertEqual(F(5, 6), F(2, 3) * F(5, 4)) self.assertEqual(F(1, 4), F(1, 10) / F(2, 5)) self.assertEqual(F(-15, 8), F(3, 4) / F(-2, 5)) + self.assertRaises(ZeroDivisionError, operator.truediv, F(1), F(0)) self.assertTypedEquals(2, F(9, 10) // F(2, 5)) self.assertTypedEquals(10**23, F(10**23, 1) // F(1)) self.assertEqual(F(5, 6), F(7, 3) % F(3, 2)) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 730ab1f595f22c..57db96d37ee369 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2931,21 +2931,6 @@ def get_cost(self): cached_cost = py_functools.cached_property(get_cost) -class CachedCostItemWait: - - def __init__(self, event): - self._cost = 1 - self.lock = py_functools.RLock() - self.event = event - - @py_functools.cached_property - def cost(self): - self.event.wait(1) - with self.lock: - self._cost += 1 - return self._cost - - class CachedCostItemWithSlots: __slots__ = ('_cost') @@ -2970,27 +2955,6 @@ def test_cached_attribute_name_differs_from_func_name(self): self.assertEqual(item.get_cost(), 4) self.assertEqual(item.cached_cost, 3) - @threading_helper.requires_working_threading() - def test_threaded(self): - go = threading.Event() - item = CachedCostItemWait(go) - - num_threads = 3 - - orig_si = sys.getswitchinterval() - sys.setswitchinterval(1e-6) - try: - threads = [ - threading.Thread(target=lambda: item.cost) - for k in range(num_threads) - ] - with threading_helper.start_threads(threads): - go.set() - finally: - sys.setswitchinterval(orig_si) - - self.assertEqual(item.cost, 2) - def test_object_with_slots(self): item = CachedCostItemWithSlots() with self.assertRaisesRegex( diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 2292bb20939599..03e3adba221e57 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -1,4 +1,5 @@ import gc +import json import importlib import importlib.util import os @@ -11,6 +12,7 @@ from test.support import script_helper from test.support import warnings_helper import textwrap +import types import unittest import warnings imp = warnings_helper.import_deprecated('imp') @@ -39,6 +41,169 @@ def requires_load_dynamic(meth): 'imp.load_dynamic() required')(meth) +class ModuleSnapshot(types.SimpleNamespace): + """A representation of a module for testing. + + Fields: + + * id - the module's object ID + * module - the actual module or an adequate substitute + * __file__ + * __spec__ + * name + * origin + * ns - a copy (dict) of the module's __dict__ (or None) + * ns_id - the object ID of the module's __dict__ + * cached - the sys.modules[mod.__spec__.name] entry (or None) + * cached_id - the object ID of the sys.modules entry (or None) + + In cases where the value is not available (e.g. due to serialization), + the value will be None. + """ + _fields = tuple('id module ns ns_id cached cached_id'.split()) + + @classmethod + def from_module(cls, mod): + name = mod.__spec__.name + cached = sys.modules.get(name) + return cls( + id=id(mod), + module=mod, + ns=types.SimpleNamespace(**mod.__dict__), + ns_id=id(mod.__dict__), + cached=cached, + cached_id=id(cached), + ) + + SCRIPT = textwrap.dedent(''' + {imports} + + name = {name!r} + + {prescript} + + mod = {name} + + {body} + + {postscript} + ''') + IMPORTS = textwrap.dedent(''' + import sys + ''').strip() + SCRIPT_BODY = textwrap.dedent(''' + # Capture the snapshot data. + cached = sys.modules.get(name) + snapshot = dict( + id=id(mod), + module=dict( + __file__=mod.__file__, + __spec__=dict( + name=mod.__spec__.name, + origin=mod.__spec__.origin, + ), + ), + ns=None, + ns_id=id(mod.__dict__), + cached=None, + cached_id=id(cached) if cached else None, + ) + ''').strip() + CLEANUP_SCRIPT = textwrap.dedent(''' + # Clean up the module. + sys.modules.pop(name, None) + ''').strip() + + @classmethod + def build_script(cls, name, *, + prescript=None, + import_first=False, + postscript=None, + postcleanup=False, + ): + if postcleanup is True: + postcleanup = cls.CLEANUP_SCRIPT + elif isinstance(postcleanup, str): + postcleanup = textwrap.dedent(postcleanup).strip() + postcleanup = cls.CLEANUP_SCRIPT + os.linesep + postcleanup + else: + postcleanup = '' + prescript = textwrap.dedent(prescript).strip() if prescript else '' + postscript = textwrap.dedent(postscript).strip() if postscript else '' + + if postcleanup: + if postscript: + postscript = postscript + os.linesep * 2 + postcleanup + else: + postscript = postcleanup + + if import_first: + prescript += textwrap.dedent(f''' + + # Now import the module. + assert name not in sys.modules + import {name}''') + + return cls.SCRIPT.format( + imports=cls.IMPORTS.strip(), + name=name, + prescript=prescript.strip(), + body=cls.SCRIPT_BODY.strip(), + postscript=postscript, + ) + + @classmethod + def parse(cls, text): + raw = json.loads(text) + mod = raw['module'] + mod['__spec__'] = types.SimpleNamespace(**mod['__spec__']) + raw['module'] = types.SimpleNamespace(**mod) + return cls(**raw) + + @classmethod + def from_subinterp(cls, name, interpid=None, *, pipe=None, **script_kwds): + if pipe is not None: + return cls._from_subinterp(name, interpid, pipe, script_kwds) + pipe = os.pipe() + try: + return cls._from_subinterp(name, interpid, pipe, script_kwds) + finally: + r, w = pipe + os.close(r) + os.close(w) + + @classmethod + def _from_subinterp(cls, name, interpid, pipe, script_kwargs): + r, w = pipe + + # Build the script. + postscript = textwrap.dedent(f''' + # Send the result over the pipe. + import json + import os + os.write({w}, json.dumps(snapshot).encode()) + + ''') + _postscript = script_kwargs.get('postscript') + if _postscript: + _postscript = textwrap.dedent(_postscript).lstrip() + postscript += _postscript + script_kwargs['postscript'] = postscript.strip() + script = cls.build_script(name, **script_kwargs) + + # Run the script. + if interpid is None: + ret = support.run_in_subinterp(script) + if ret != 0: + raise AssertionError(f'{ret} != 0') + else: + _interpreters.run_string(interpid, script) + + # Parse the results. + text = os.read(r, 1000) + return cls.parse(text.decode()) + + class LockTests(unittest.TestCase): """Very basic test of import lock functions.""" @@ -263,288 +428,6 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) - @unittest.skip('known refleak (temporarily skipping)') - @requires_subinterpreters - @requires_load_dynamic - def test_singlephase_multiple_interpreters(self): - # Currently, for every single-phrase init module loaded - # in multiple interpreters, those interpreters share a - # PyModuleDef for that object, which can be a problem. - - # This single-phase module has global state, which is shared - # by the interpreters. - import _testsinglephase - name = _testsinglephase.__name__ - filename = _testsinglephase.__file__ - - del sys.modules[name] - _testsinglephase._clear_globals() - _testinternalcapi.clear_extension(name, filename) - init_count = _testsinglephase.initialized_count() - assert init_count == -1, (init_count,) - - def clean_up(): - _testsinglephase._clear_globals() - _testinternalcapi.clear_extension(name, filename) - self.addCleanup(clean_up) - - interp1 = _interpreters.create(isolated=False) - self.addCleanup(_interpreters.destroy, interp1) - interp2 = _interpreters.create(isolated=False) - self.addCleanup(_interpreters.destroy, interp2) - - script = textwrap.dedent(f''' - import _testsinglephase - - expected = %d - init_count = _testsinglephase.initialized_count() - if init_count != expected: - raise Exception(init_count) - - lookedup = _testsinglephase.look_up_self() - if lookedup is not _testsinglephase: - raise Exception((_testsinglephase, lookedup)) - - # Attrs set in the module init func are in m_copy. - _initialized = _testsinglephase._initialized - initialized = _testsinglephase.initialized() - if _initialized != initialized: - raise Exception((_initialized, initialized)) - - # Attrs set after loading are not in m_copy. - if hasattr(_testsinglephase, 'spam'): - raise Exception(_testsinglephase.spam) - _testsinglephase.spam = expected - ''') - - # Use an interpreter that gets destroyed right away. - ret = support.run_in_subinterp(script % 1) - self.assertEqual(ret, 0) - - # The module's init func gets run again. - # The module's globals did not get destroyed. - _interpreters.run_string(interp1, script % 2) - - # The module's init func is not run again. - # The second interpreter copies the module's m_copy. - # However, globals are still shared. - _interpreters.run_string(interp2, script % 2) - - @unittest.skip('known refleak (temporarily skipping)') - @requires_load_dynamic - def test_singlephase_variants(self): - # Exercise the most meaningful variants described in Python/import.c. - self.maxDiff = None - - basename = '_testsinglephase' - fileobj, pathname, _ = imp.find_module(basename) - fileobj.close() - - def clean_up(): - import _testsinglephase - _testsinglephase._clear_globals() - self.addCleanup(clean_up) - - def add_ext_cleanup(name): - def clean_up(): - _testinternalcapi.clear_extension(name, pathname) - self.addCleanup(clean_up) - - modules = {} - def load(name): - assert name not in modules - module = imp.load_dynamic(name, pathname) - self.assertNotIn(module, modules.values()) - modules[name] = module - return module - - def re_load(name, module): - assert sys.modules[name] is module - before = type(module)(module.__name__) - before.__dict__.update(vars(module)) - - reloaded = imp.load_dynamic(name, pathname) - - return before, reloaded - - def check_common(name, module): - summed = module.sum(1, 2) - lookedup = module.look_up_self() - initialized = module.initialized() - cached = sys.modules[name] - - # module.__name__ might not match, but the spec will. - self.assertEqual(module.__spec__.name, name) - if initialized is not None: - self.assertIsInstance(initialized, float) - self.assertGreater(initialized, 0) - self.assertEqual(summed, 3) - self.assertTrue(issubclass(module.error, Exception)) - self.assertEqual(module.int_const, 1969) - self.assertEqual(module.str_const, 'something different') - self.assertIs(cached, module) - - return lookedup, initialized, cached - - def check_direct(name, module, lookedup): - # The module has its own PyModuleDef, with a matching name. - self.assertEqual(module.__name__, name) - self.assertIs(lookedup, module) - - def check_indirect(name, module, lookedup, orig): - # The module re-uses another's PyModuleDef, with a different name. - assert orig is not module - assert orig.__name__ != name - self.assertNotEqual(module.__name__, name) - self.assertIs(lookedup, module) - - def check_basic(module, initialized): - init_count = module.initialized_count() - - self.assertIsNot(initialized, None) - self.assertIsInstance(init_count, int) - self.assertGreater(init_count, 0) - - return init_count - - def check_common_reloaded(name, module, cached, before, reloaded): - recached = sys.modules[name] - - self.assertEqual(reloaded.__spec__.name, name) - self.assertEqual(reloaded.__name__, before.__name__) - self.assertEqual(before.__dict__, module.__dict__) - self.assertIs(recached, reloaded) - - def check_basic_reloaded(module, lookedup, initialized, init_count, - before, reloaded): - relookedup = reloaded.look_up_self() - reinitialized = reloaded.initialized() - reinit_count = reloaded.initialized_count() - - self.assertIs(reloaded, module) - self.assertIs(reloaded.__dict__, module.__dict__) - # It only happens to be the same but that's good enough here. - # We really just want to verify that the re-loaded attrs - # didn't change. - self.assertIs(relookedup, lookedup) - self.assertEqual(reinitialized, initialized) - self.assertEqual(reinit_count, init_count) - - def check_with_reinit_reloaded(module, lookedup, initialized, - before, reloaded): - relookedup = reloaded.look_up_self() - reinitialized = reloaded.initialized() - - self.assertIsNot(reloaded, module) - self.assertIsNot(reloaded, module) - self.assertNotEqual(reloaded.__dict__, module.__dict__) - self.assertIs(relookedup, reloaded) - if initialized is None: - self.assertIs(reinitialized, None) - else: - self.assertGreater(reinitialized, initialized) - - # Check the "basic" module. - - name = basename - add_ext_cleanup(name) - expected_init_count = 1 - with self.subTest(name): - mod = load(name) - lookedup, initialized, cached = check_common(name, mod) - check_direct(name, mod, lookedup) - init_count = check_basic(mod, initialized) - self.assertEqual(init_count, expected_init_count) - - before, reloaded = re_load(name, mod) - check_common_reloaded(name, mod, cached, before, reloaded) - check_basic_reloaded(mod, lookedup, initialized, init_count, - before, reloaded) - basic = mod - - # Check its indirect variants. - - name = f'{basename}_basic_wrapper' - add_ext_cleanup(name) - expected_init_count += 1 - with self.subTest(name): - mod = load(name) - lookedup, initialized, cached = check_common(name, mod) - check_indirect(name, mod, lookedup, basic) - init_count = check_basic(mod, initialized) - self.assertEqual(init_count, expected_init_count) - - before, reloaded = re_load(name, mod) - check_common_reloaded(name, mod, cached, before, reloaded) - check_basic_reloaded(mod, lookedup, initialized, init_count, - before, reloaded) - - # Currently PyState_AddModule() always replaces the cached module. - self.assertIs(basic.look_up_self(), mod) - self.assertEqual(basic.initialized_count(), expected_init_count) - - # The cached module shouldn't be changed after this point. - basic_lookedup = mod - - # Check its direct variant. - - name = f'{basename}_basic_copy' - add_ext_cleanup(name) - expected_init_count += 1 - with self.subTest(name): - mod = load(name) - lookedup, initialized, cached = check_common(name, mod) - check_direct(name, mod, lookedup) - init_count = check_basic(mod, initialized) - self.assertEqual(init_count, expected_init_count) - - before, reloaded = re_load(name, mod) - check_common_reloaded(name, mod, cached, before, reloaded) - check_basic_reloaded(mod, lookedup, initialized, init_count, - before, reloaded) - - # This should change the cached module for _testsinglephase. - self.assertIs(basic.look_up_self(), basic_lookedup) - self.assertEqual(basic.initialized_count(), expected_init_count) - - # Check the non-basic variant that has no state. - - name = f'{basename}_with_reinit' - add_ext_cleanup(name) - with self.subTest(name): - mod = load(name) - lookedup, initialized, cached = check_common(name, mod) - self.assertIs(initialized, None) - check_direct(name, mod, lookedup) - - before, reloaded = re_load(name, mod) - check_common_reloaded(name, mod, cached, before, reloaded) - check_with_reinit_reloaded(mod, lookedup, initialized, - before, reloaded) - - # This should change the cached module for _testsinglephase. - self.assertIs(basic.look_up_self(), basic_lookedup) - self.assertEqual(basic.initialized_count(), expected_init_count) - - # Check the basic variant that has state. - - name = f'{basename}_with_state' - add_ext_cleanup(name) - with self.subTest(name): - mod = load(name) - lookedup, initialized, cached = check_common(name, mod) - self.assertIsNot(initialized, None) - check_direct(name, mod, lookedup) - - before, reloaded = re_load(name, mod) - check_common_reloaded(name, mod, cached, before, reloaded) - check_with_reinit_reloaded(mod, lookedup, initialized, - before, reloaded) - - # This should change the cached module for _testsinglephase. - self.assertIs(basic.look_up_self(), basic_lookedup) - self.assertEqual(basic.initialized_count(), expected_init_count) - @requires_load_dynamic def test_load_dynamic_ImportError_path(self): # Issue #1559549 added `name` and `path` attributes to ImportError @@ -737,6 +620,669 @@ def check_get_builtins(): check_get_builtins() +class TestSinglePhaseSnapshot(ModuleSnapshot): + + @classmethod + def from_module(cls, mod): + self = super().from_module(mod) + self.summed = mod.sum(1, 2) + self.lookedup = mod.look_up_self() + self.lookedup_id = id(self.lookedup) + self.state_initialized = mod.state_initialized() + if hasattr(mod, 'initialized_count'): + self.init_count = mod.initialized_count() + return self + + SCRIPT_BODY = ModuleSnapshot.SCRIPT_BODY + textwrap.dedent(f''' + snapshot['module'].update(dict( + int_const=mod.int_const, + str_const=mod.str_const, + _module_initialized=mod._module_initialized, + )) + snapshot.update(dict( + summed=mod.sum(1, 2), + lookedup_id=id(mod.look_up_self()), + state_initialized=mod.state_initialized(), + init_count=mod.initialized_count(), + has_spam=hasattr(mod, 'spam'), + spam=getattr(mod, 'spam', None), + )) + ''').rstrip() + + @classmethod + def parse(cls, text): + self = super().parse(text) + if not self.has_spam: + del self.spam + del self.has_spam + return self + + +@requires_load_dynamic +class SinglephaseInitTests(unittest.TestCase): + + NAME = '_testsinglephase' + + @classmethod + def setUpClass(cls): + if '-R' in sys.argv or '--huntrleaks' in sys.argv: + # https://github.com/python/cpython/issues/102251 + raise unittest.SkipTest('unresolved refleaks (see gh-102251)') + fileobj, filename, _ = imp.find_module(cls.NAME) + fileobj.close() + cls.FILE = filename + + # Start fresh. + cls.clean_up() + + def tearDown(self): + # Clean up the module. + self.clean_up() + + @classmethod + def clean_up(cls): + name = cls.NAME + filename = cls.FILE + if name in sys.modules: + if hasattr(sys.modules[name], '_clear_globals'): + assert sys.modules[name].__file__ == filename + sys.modules[name]._clear_globals() + del sys.modules[name] + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, filename) + + ######################### + # helpers + + def add_module_cleanup(self, name): + def clean_up(): + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, self.FILE) + self.addCleanup(clean_up) + + def load(self, name): + try: + already_loaded = self.already_loaded + except AttributeError: + already_loaded = self.already_loaded = {} + assert name not in already_loaded + mod = imp.load_dynamic(name, self.FILE) + self.assertNotIn(mod, already_loaded.values()) + already_loaded[name] = mod + return types.SimpleNamespace( + name=name, + module=mod, + snapshot=TestSinglePhaseSnapshot.from_module(mod), + ) + + def re_load(self, name, mod): + assert sys.modules[name] is mod + assert mod.__dict__ == mod.__dict__ + reloaded = imp.load_dynamic(name, self.FILE) + return types.SimpleNamespace( + name=name, + module=reloaded, + snapshot=TestSinglePhaseSnapshot.from_module(reloaded), + ) + + # subinterpreters + + def add_subinterpreter(self): + interpid = _interpreters.create(isolated=False) + _interpreters.run_string(interpid, textwrap.dedent(''' + import sys + import _testinternalcapi + ''')) + def clean_up(): + _interpreters.run_string(interpid, textwrap.dedent(f''' + name = {self.NAME!r} + if name in sys.modules: + sys.modules[name]._clear_globals() + _testinternalcapi.clear_extension(name, {self.FILE!r}) + ''')) + _interpreters.destroy(interpid) + self.addCleanup(clean_up) + return interpid + + def import_in_subinterp(self, interpid=None, *, + postscript=None, + postcleanup=False, + ): + name = self.NAME + + if postcleanup: + import_ = 'import _testinternalcapi' if interpid is None else '' + postcleanup = f''' + {import_} + mod._clear_globals() + _testinternalcapi.clear_extension(name, {self.FILE!r}) + ''' + + try: + pipe = self._pipe + except AttributeError: + r, w = pipe = self._pipe = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + + snapshot = TestSinglePhaseSnapshot.from_subinterp( + name, + interpid, + pipe=pipe, + import_first=True, + postscript=postscript, + postcleanup=postcleanup, + ) + + return types.SimpleNamespace( + name=name, + module=None, + snapshot=snapshot, + ) + + # checks + + def check_common(self, loaded): + isolated = False + + mod = loaded.module + if not mod: + # It came from a subinterpreter. + isolated = True + mod = loaded.snapshot.module + # mod.__name__ might not match, but the spec will. + self.assertEqual(mod.__spec__.name, loaded.name) + self.assertEqual(mod.__file__, self.FILE) + self.assertEqual(mod.__spec__.origin, self.FILE) + if not isolated: + self.assertTrue(issubclass(mod.error, Exception)) + self.assertEqual(mod.int_const, 1969) + self.assertEqual(mod.str_const, 'something different') + self.assertIsInstance(mod._module_initialized, float) + self.assertGreater(mod._module_initialized, 0) + + snap = loaded.snapshot + self.assertEqual(snap.summed, 3) + if snap.state_initialized is not None: + self.assertIsInstance(snap.state_initialized, float) + self.assertGreater(snap.state_initialized, 0) + if isolated: + # The "looked up" module is interpreter-specific + # (interp->imports.modules_by_index was set for the module). + self.assertEqual(snap.lookedup_id, snap.id) + self.assertEqual(snap.cached_id, snap.id) + with self.assertRaises(AttributeError): + snap.spam + else: + self.assertIs(snap.lookedup, mod) + self.assertIs(snap.cached, mod) + + def check_direct(self, loaded): + # The module has its own PyModuleDef, with a matching name. + self.assertEqual(loaded.module.__name__, loaded.name) + self.assertIs(loaded.snapshot.lookedup, loaded.module) + + def check_indirect(self, loaded, orig): + # The module re-uses another's PyModuleDef, with a different name. + assert orig is not loaded.module + assert orig.__name__ != loaded.name + self.assertNotEqual(loaded.module.__name__, loaded.name) + self.assertIs(loaded.snapshot.lookedup, loaded.module) + + def check_basic(self, loaded, expected_init_count): + # m_size == -1 + # The module loads fresh the first time and copies m_copy after. + snap = loaded.snapshot + self.assertIsNot(snap.state_initialized, None) + self.assertIsInstance(snap.init_count, int) + self.assertGreater(snap.init_count, 0) + self.assertEqual(snap.init_count, expected_init_count) + + def check_with_reinit(self, loaded): + # m_size >= 0 + # The module loads fresh every time. + pass + + def check_fresh(self, loaded): + """ + The module had not been loaded before (at least since fully reset). + """ + snap = loaded.snapshot + # The module's init func was run. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # _PyRuntime.imports.extensions was set. + self.assertEqual(snap.init_count, 1) + # The global state was initialized. + # The module attrs were initialized from that state. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + + def check_semi_fresh(self, loaded, base, prev): + """ + The module had been loaded before and then reset + (but the module global state wasn't). + """ + snap = loaded.snapshot + # The module's init func was run again. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # The module globals did not get reset. + self.assertNotEqual(snap.id, base.snapshot.id) + self.assertNotEqual(snap.id, prev.snapshot.id) + self.assertEqual(snap.init_count, prev.snapshot.init_count + 1) + # The global state was updated. + # The module attrs were initialized from that state. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + self.assertNotEqual(snap.state_initialized, + base.snapshot.state_initialized) + self.assertNotEqual(snap.state_initialized, + prev.snapshot.state_initialized) + + def check_copied(self, loaded, base): + """ + The module had been loaded before and never reset. + """ + snap = loaded.snapshot + # The module's init func was not run again. + # The interpreter copied m_copy, as set by the other interpreter, + # with objects owned by the other interpreter. + # The module globals did not get reset. + self.assertNotEqual(snap.id, base.snapshot.id) + self.assertEqual(snap.init_count, base.snapshot.init_count) + # The global state was not updated since the init func did not run. + # The module attrs were not directly initialized from that state. + # The state and module attrs still match the previous loading. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + self.assertEqual(snap.state_initialized, + base.snapshot.state_initialized) + + ######################### + # the tests + + def test_cleared_globals(self): + loaded = self.load(self.NAME) + _testsinglephase = loaded.module + init_before = _testsinglephase.state_initialized() + + _testsinglephase._clear_globals() + init_after = _testsinglephase.state_initialized() + init_count = _testsinglephase.initialized_count() + + self.assertGreater(init_before, 0) + self.assertEqual(init_after, 0) + self.assertEqual(init_count, -1) + + def test_variants(self): + # Exercise the most meaningful variants described in Python/import.c. + self.maxDiff = None + + # Check the "basic" module. + + name = self.NAME + expected_init_count = 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_direct(loaded) + self.check_basic(loaded, expected_init_count) + basic = loaded.module + + # Check its indirect variants. + + name = f'{self.NAME}_basic_wrapper' + self.add_module_cleanup(name) + expected_init_count += 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_indirect(loaded, basic) + self.check_basic(loaded, expected_init_count) + + # Currently PyState_AddModule() always replaces the cached module. + self.assertIs(basic.look_up_self(), loaded.module) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # The cached module shouldn't change after this point. + basic_lookedup = loaded.module + + # Check its direct variant. + + name = f'{self.NAME}_basic_copy' + self.add_module_cleanup(name) + expected_init_count += 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_direct(loaded) + self.check_basic(loaded, expected_init_count) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the non-basic variant that has no state. + + name = f'{self.NAME}_with_reinit' + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.assertIs(loaded.snapshot.state_initialized, None) + self.check_direct(loaded) + self.check_with_reinit(loaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the basic variant that has state. + + name = f'{self.NAME}_with_state' + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.assertIsNot(loaded.snapshot.state_initialized, None) + self.check_direct(loaded) + self.check_with_reinit(loaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + def test_basic_reloaded(self): + # m_copy is copied into the existing module object. + # Global state is not changed. + self.maxDiff = None + + for name in [ + self.NAME, # the "basic" module + f'{self.NAME}_basic_wrapper', # the indirect variant + f'{self.NAME}_basic_copy', # the direct variant + ]: + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + reloaded = self.re_load(name, loaded.module) + + self.check_common(loaded) + self.check_common(reloaded) + + # Make sure the original __dict__ did not get replaced. + self.assertEqual(id(loaded.module.__dict__), + loaded.snapshot.ns_id) + self.assertEqual(loaded.snapshot.ns.__dict__, + loaded.module.__dict__) + + self.assertEqual(reloaded.module.__spec__.name, reloaded.name) + self.assertEqual(reloaded.module.__name__, + reloaded.snapshot.ns.__name__) + + self.assertIs(reloaded.module, loaded.module) + self.assertIs(reloaded.module.__dict__, loaded.module.__dict__) + # It only happens to be the same but that's good enough here. + # We really just want to verify that the re-loaded attrs + # didn't change. + self.assertIs(reloaded.snapshot.lookedup, + loaded.snapshot.lookedup) + self.assertEqual(reloaded.snapshot.state_initialized, + loaded.snapshot.state_initialized) + self.assertEqual(reloaded.snapshot.init_count, + loaded.snapshot.init_count) + + self.assertIs(reloaded.snapshot.cached, reloaded.module) + + def test_with_reinit_reloaded(self): + # The module's m_init func is run again. + self.maxDiff = None + + # Keep a reference around. + basic = self.load(self.NAME) + + for name in [ + f'{self.NAME}_with_reinit', # m_size == 0 + f'{self.NAME}_with_state', # m_size > 0 + ]: + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + reloaded = self.re_load(name, loaded.module) + + self.check_common(loaded) + self.check_common(reloaded) + + # Make sure the original __dict__ did not get replaced. + self.assertEqual(id(loaded.module.__dict__), + loaded.snapshot.ns_id) + self.assertEqual(loaded.snapshot.ns.__dict__, + loaded.module.__dict__) + + self.assertEqual(reloaded.module.__spec__.name, reloaded.name) + self.assertEqual(reloaded.module.__name__, + reloaded.snapshot.ns.__name__) + + self.assertIsNot(reloaded.module, loaded.module) + self.assertNotEqual(reloaded.module.__dict__, + loaded.module.__dict__) + self.assertIs(reloaded.snapshot.lookedup, reloaded.module) + if loaded.snapshot.state_initialized is None: + self.assertIs(reloaded.snapshot.state_initialized, None) + else: + self.assertGreater(reloaded.snapshot.state_initialized, + loaded.snapshot.state_initialized) + + self.assertIs(reloaded.snapshot.cached, reloaded.module) + + # Currently, for every single-phrase init module loaded + # in multiple interpreters, those interpreters share a + # PyModuleDef for that object, which can be a problem. + # Also, we test with a single-phase module that has global state, + # which is shared by all interpreters. + + @requires_subinterpreters + def test_basic_multiple_interpreters_main_no_reset(self): + # without resetting; already loaded in main interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + main_loaded = self.load(self.NAME) + _testsinglephase = main_loaded.module + # Attrs set after loading are not in m_copy. + _testsinglephase.spam = 'spam, spam, spam, spam, eggs, and spam' + + self.check_common(main_loaded) + self.check_fresh(main_loaded) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # At this point: + # * alive in 1 interpreter (main) + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset, at least) + # * m_copy was copied from the main interpreter (was NULL) + # * module's global state was initialized + + # Use an interpreter that gets destroyed right away. + loaded = self.import_in_subinterp() + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 1 interpreter (main) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy is NULL (claered when the interpreter was destroyed) + # (was from main interpreter) + # * module's global state was updated, not reset + + # Use a subinterpreter that sticks around. + loaded = self.import_in_subinterp(interpid1) + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 2 interpreters (main, interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 + # * module's global state was updated, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded = self.import_in_subinterp(interpid2) + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 3 interpreters (main, interp1, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was updated, not reset + + @requires_subinterpreters + def test_basic_multiple_interpreters_deleted_no_reset(self): + # without resetting; already loaded in a deleted interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # First, load in the main interpreter but then completely clear it. + loaded_main = self.load(self.NAME) + loaded_main.module._clear_globals() + _testinternalcapi.clear_extension(self.NAME, self.FILE) + + # At this point: + # * alive in 0 interpreters + # * module def loaded already + # * module def was in _PyRuntime.imports.extensions, but cleared + # * mod init func ran for the first time (since reset, at least) + # * m_copy was set, but cleared (was NULL) + # * module's global state was initialized but cleared + + # Start with an interpreter that gets destroyed right away. + base = self.import_in_subinterp(postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''') + self.check_common(base) + self.check_fresh(base) + + # At this point: + # * alive in 0 interpreters + # * module def in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy is NULL (claered when the interpreter was destroyed) + # * module's global state was initialized, not reset + + # Use a subinterpreter that sticks around. + loaded_interp1 = self.import_in_subinterp(interpid1) + self.check_common(loaded_interp1) + self.check_semi_fresh(loaded_interp1, loaded_main, base) + + # At this point: + # * alive in 1 interpreter (interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 (was NULL) + # * module's global state was updated, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded_interp2 = self.import_in_subinterp(interpid2) + self.check_common(loaded_interp2) + self.check_copied(loaded_interp2, loaded_interp1) + + # At this point: + # * alive in 2 interpreters (interp1, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was updated, not reset + + @requires_subinterpreters + @requires_load_dynamic + def test_basic_multiple_interpreters_reset_each(self): + # resetting between each interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # Use an interpreter that gets destroyed right away. + loaded = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''', + postcleanup=True, + ) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 0 interpreters + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset, at least) + # * m_copy is NULL (claered when the interpreter was destroyed) + # * module's global state was initialized, not reset + + # Use a subinterpreter that sticks around. + loaded = self.import_in_subinterp(interpid1, postcleanup=True) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 1 interpreter (interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 (was NULL) + # * module's global state was initialized, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded = self.import_in_subinterp(interpid2, postcleanup=True) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 2 interpreters (interp2, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was initialized, not reset + + class ReloadTests(unittest.TestCase): """Very basic tests to make sure that imp.reload() operates just like diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index acbdcb5f302060..6ab9b7a7230309 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -7,6 +7,9 @@ from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ import pickle import collections.abc +import functools +import contextlib +import builtins # Test result of triple loop (too big to inline) TRIPLETS = [(0, 0, 0), (0, 0, 1), (0, 0, 2), @@ -91,6 +94,12 @@ def __call__(self): raise IndexError # Emergency stop return i +class EmptyIterClass: + def __len__(self): + return 0 + def __getitem__(self, i): + raise StopIteration + # Main test suite class TestCase(unittest.TestCase): @@ -238,6 +247,78 @@ def test_mutating_seq_class_exhausted_iter(self): self.assertEqual(list(empit), [5, 6]) self.assertEqual(list(a), [0, 1, 2, 3, 4, 5, 6]) + def test_reduce_mutating_builtins_iter(self): + # This is a reproducer of issue #101765 + # where iter `__reduce__` calls could lead to a segfault or SystemError + # depending on the order of C argument evaluation, which is undefined + + # Backup builtins + builtins_dict = builtins.__dict__ + orig = {"iter": iter, "reversed": reversed} + + def run(builtin_name, item, sentinel=None): + it = iter(item) if sentinel is None else iter(item, sentinel) + + class CustomStr: + def __init__(self, name, iterator): + self.name = name + self.iterator = iterator + def __hash__(self): + return hash(self.name) + def __eq__(self, other): + # Here we exhaust our iterator, possibly changing + # its `it_seq` pointer to NULL + # The `__reduce__` call should correctly get + # the pointers after this call + list(self.iterator) + return other == self.name + + # del is required here + # to not prematurely call __eq__ from + # the hash collision with the old key + del builtins_dict[builtin_name] + builtins_dict[CustomStr(builtin_name, it)] = orig[builtin_name] + + return it.__reduce__() + + types = [ + (EmptyIterClass(),), + (bytes(8),), + (bytearray(8),), + ((1, 2, 3),), + (lambda: 0, 0), + (tuple[int],) # GenericAlias + ] + + try: + run_iter = functools.partial(run, "iter") + # The returned value of `__reduce__` should not only be valid + # but also *empty*, as `it` was exhausted during `__eq__` + # i.e "xyz" returns (iter, ("",)) + self.assertEqual(run_iter("xyz"), (orig["iter"], ("",))) + self.assertEqual(run_iter([1, 2, 3]), (orig["iter"], ([],))) + + # _PyEval_GetBuiltin is also called for `reversed` in a branch of + # listiter_reduce_general + self.assertEqual( + run("reversed", orig["reversed"](list(range(8)))), + (iter, ([],)) + ) + + for case in types: + self.assertEqual(run_iter(*case), (orig["iter"], ((),))) + finally: + # Restore original builtins + for key, func in orig.items(): + # need to suppress KeyErrors in case + # a failed test deletes the key without setting anything + with contextlib.suppress(KeyError): + # del is required here + # to not invoke our custom __eq__ from + # the hash collision with the old key + del builtins_dict[key] + builtins_dict[key] = func + # Test a new_style class with __iter__ but no next() method def test_new_style_iter_class(self): class IterClass(object): diff --git a/Lib/test/test_keyword.py b/Lib/test/test_keyword.py index 3e2a8b3fb7f4c3..f329f88fa01d51 100644 --- a/Lib/test/test_keyword.py +++ b/Lib/test/test_keyword.py @@ -20,18 +20,36 @@ def test_changing_the_kwlist_does_not_affect_iskeyword(self): keyword.kwlist = ['its', 'all', 'eggs', 'beans', 'and', 'a', 'slice'] self.assertFalse(keyword.iskeyword('eggs')) + def test_changing_the_softkwlist_does_not_affect_issoftkeyword(self): + oldlist = keyword.softkwlist + self.addCleanup(setattr, keyword, "softkwlist", oldlist) + keyword.softkwlist = ["foo", "bar", "spam", "egs", "case"] + self.assertFalse(keyword.issoftkeyword("spam")) + def test_all_keywords_fail_to_be_used_as_names(self): for key in keyword.kwlist: with self.assertRaises(SyntaxError): exec(f"{key} = 42") + def test_all_soft_keywords_can_be_used_as_names(self): + for key in keyword.softkwlist: + exec(f"{key} = 42") + def test_async_and_await_are_keywords(self): self.assertIn("async", keyword.kwlist) self.assertIn("await", keyword.kwlist) + def test_match_and_case_are_soft_keywords(self): + self.assertIn("match", keyword.softkwlist) + self.assertIn("case", keyword.softkwlist) + self.assertIn("_", keyword.softkwlist) + def test_keywords_are_sorted(self): self.assertListEqual(sorted(keyword.kwlist), keyword.kwlist) + def test_softkeywords_are_sorted(self): + self.assertListEqual(sorted(keyword.softkwlist), keyword.softkwlist) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 18f474ba2a8bdc..ac53bdda2f1747 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -380,6 +380,10 @@ def test_refleaks_in_decompressor___init__(self): lzd.__init__() self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + def test_uninitialized_LZMADecompressor_crash(self): + self.assertEqual(LZMADecompressor.__new__(LZMADecompressor). + decompress(bytes()), b'') + class CompressDecompressFunctionTestCase(unittest.TestCase): diff --git a/Lib/test/test_numeric_tower.py b/Lib/test/test_numeric_tower.py index 9cd85e13634c2b..337682d6bac96c 100644 --- a/Lib/test/test_numeric_tower.py +++ b/Lib/test/test_numeric_tower.py @@ -145,7 +145,7 @@ def test_fractions(self): # The numbers ABC doesn't enforce that the "true" division # of integers produces a float. This tests that the # Rational.__float__() method has required type conversions. - x = F(DummyIntegral(1), DummyIntegral(2), _normalize=False) + x = F._from_coprime_ints(DummyIntegral(1), DummyIntegral(2)) self.assertRaises(TypeError, lambda: x.numerator/x.denominator) self.assertEqual(float(x), 0.5) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 707ff821b31a8a..aea234e38705a8 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -984,6 +984,7 @@ def cfg_optimization_test(self, insts, expected_insts, if expected_consts is None: expected_consts = consts opt_insts, opt_consts = self.get_optimized(insts, consts) + expected_insts = self.normalize_insts(expected_insts) self.assertInstructionsMatch(opt_insts, expected_insts) self.assertEqual(opt_consts, expected_consts) @@ -996,11 +997,11 @@ def test_conditional_jump_forward_non_const_condition(self): ('LOAD_CONST', 3, 14), ] expected = [ - ('LOAD_NAME', '1', 11), + ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), - ('LOAD_CONST', '2', 13), + ('LOAD_CONST', 2, 13), lbl, - ('LOAD_CONST', '3', 14) + ('LOAD_CONST', 3, 14) ] self.cfg_optimization_test(insts, expected, consts=list(range(5))) @@ -1018,7 +1019,7 @@ def test_conditional_jump_forward_const_condition(self): expected = [ ('NOP', None, 11), ('NOP', None, 12), - ('LOAD_CONST', '3', 14) + ('LOAD_CONST', 3, 14) ] self.cfg_optimization_test(insts, expected, consts=list(range(5))) @@ -1031,9 +1032,9 @@ def test_conditional_jump_backward_non_const_condition(self): ] expected = [ lbl := self.Label(), - ('LOAD_NAME', '1', 11), + ('LOAD_NAME', 1, 11), ('POP_JUMP_IF_TRUE', lbl, 12), - ('LOAD_CONST', '2', 13) + ('LOAD_CONST', 2, 13) ] self.cfg_optimization_test(insts, expected, consts=list(range(5))) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index abd0dd8b25699b..727b0e6dc578c2 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -718,7 +718,7 @@ def test_pipesizes(self): os.close(test_pipe_w) pipesize = pipesize_default // 2 if pipesize < 512: # the POSIX minimum - raise unittest.SkitTest( + raise unittest.SkipTest( 'default pipesize too small to perform test.') p = subprocess.Popen( [sys.executable, "-c", diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index ab1a0659471857..b839985def9a12 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -445,46 +445,47 @@ def g456(): t.start() entered_g.wait() - # At this point, t has finished its entered_g.set(), although it's - # impossible to guess whether it's still on that line or has moved on - # to its leave_g.wait(). - self.assertEqual(len(thread_info), 1) - thread_id = thread_info[0] - - d = sys._current_frames() - for tid in d: - self.assertIsInstance(tid, int) - self.assertGreater(tid, 0) - - main_id = threading.get_ident() - self.assertIn(main_id, d) - self.assertIn(thread_id, d) - - # Verify that the captured main-thread frame is _this_ frame. - frame = d.pop(main_id) - self.assertTrue(frame is sys._getframe()) - - # Verify that the captured thread frame is blocked in g456, called - # from f123. This is a little tricky, since various bits of - # threading.py are also in the thread's call stack. - frame = d.pop(thread_id) - stack = traceback.extract_stack(frame) - for i, (filename, lineno, funcname, sourceline) in enumerate(stack): - if funcname == "f123": - break - else: - self.fail("didn't find f123() on thread's call stack") - - self.assertEqual(sourceline, "g456()") + try: + # At this point, t has finished its entered_g.set(), although it's + # impossible to guess whether it's still on that line or has moved on + # to its leave_g.wait(). + self.assertEqual(len(thread_info), 1) + thread_id = thread_info[0] + + d = sys._current_frames() + for tid in d: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + + main_id = threading.get_ident() + self.assertIn(main_id, d) + self.assertIn(thread_id, d) + + # Verify that the captured main-thread frame is _this_ frame. + frame = d.pop(main_id) + self.assertTrue(frame is sys._getframe()) + + # Verify that the captured thread frame is blocked in g456, called + # from f123. This is a little tricky, since various bits of + # threading.py are also in the thread's call stack. + frame = d.pop(thread_id) + stack = traceback.extract_stack(frame) + for i, (filename, lineno, funcname, sourceline) in enumerate(stack): + if funcname == "f123": + break + else: + self.fail("didn't find f123() on thread's call stack") - # And the next record must be for g456(). - filename, lineno, funcname, sourceline = stack[i+1] - self.assertEqual(funcname, "g456") - self.assertIn(sourceline, ["leave_g.wait()", "entered_g.set()"]) + self.assertEqual(sourceline, "g456()") - # Reap the spawned thread. - leave_g.set() - t.join() + # And the next record must be for g456(). + filename, lineno, funcname, sourceline = stack[i+1] + self.assertEqual(funcname, "g456") + self.assertIn(sourceline, ["leave_g.wait()", "entered_g.set()"]) + finally: + # Reap the spawned thread. + leave_g.set() + t.join() @threading_helper.reap_threads @threading_helper.requires_working_threading() @@ -516,43 +517,44 @@ def g456(): t.start() entered_g.wait() - # At this point, t has finished its entered_g.set(), although it's - # impossible to guess whether it's still on that line or has moved on - # to its leave_g.wait(). - self.assertEqual(len(thread_info), 1) - thread_id = thread_info[0] - - d = sys._current_exceptions() - for tid in d: - self.assertIsInstance(tid, int) - self.assertGreater(tid, 0) - - main_id = threading.get_ident() - self.assertIn(main_id, d) - self.assertIn(thread_id, d) - self.assertEqual((None, None, None), d.pop(main_id)) - - # Verify that the captured thread frame is blocked in g456, called - # from f123. This is a little tricky, since various bits of - # threading.py are also in the thread's call stack. - exc_type, exc_value, exc_tb = d.pop(thread_id) - stack = traceback.extract_stack(exc_tb.tb_frame) - for i, (filename, lineno, funcname, sourceline) in enumerate(stack): - if funcname == "f123": - break - else: - self.fail("didn't find f123() on thread's call stack") - - self.assertEqual(sourceline, "g456()") + try: + # At this point, t has finished its entered_g.set(), although it's + # impossible to guess whether it's still on that line or has moved on + # to its leave_g.wait(). + self.assertEqual(len(thread_info), 1) + thread_id = thread_info[0] + + d = sys._current_exceptions() + for tid in d: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + + main_id = threading.get_ident() + self.assertIn(main_id, d) + self.assertIn(thread_id, d) + self.assertEqual((None, None, None), d.pop(main_id)) + + # Verify that the captured thread frame is blocked in g456, called + # from f123. This is a little tricky, since various bits of + # threading.py are also in the thread's call stack. + exc_type, exc_value, exc_tb = d.pop(thread_id) + stack = traceback.extract_stack(exc_tb.tb_frame) + for i, (filename, lineno, funcname, sourceline) in enumerate(stack): + if funcname == "f123": + break + else: + self.fail("didn't find f123() on thread's call stack") - # And the next record must be for g456(). - filename, lineno, funcname, sourceline = stack[i+1] - self.assertEqual(funcname, "g456") - self.assertTrue(sourceline.startswith("if leave_g.wait(")) + self.assertEqual(sourceline, "g456()") - # Reap the spawned thread. - leave_g.set() - t.join() + # And the next record must be for g456(). + filename, lineno, funcname, sourceline = stack[i+1] + self.assertEqual(funcname, "g456") + self.assertTrue(sourceline.startswith("if leave_g.wait(")) + finally: + # Reap the spawned thread. + leave_g.set() + t.join() def test_attributes(self): self.assertIsInstance(sys.api_version, int) @@ -1460,7 +1462,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('P2P4P4c7P2ic??2P')) + check(get_gen(), size('PP4P4c7P2ic??2P')) # iterator check(iter('abc'), size('lP')) # callable-iterator diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index f15a800976681c..75b60e9a50e78a 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -225,18 +225,19 @@ def test_add_dir_getmember(self): self.add_dir_and_getmember('bar') self.add_dir_and_getmember('a'*101) - @unittest.skipIf( - (hasattr(os, 'getuid') and os.getuid() > 0o777_7777) or - (hasattr(os, 'getgid') and os.getgid() > 0o777_7777), - "uid or gid too high for USTAR format." - ) + @unittest.skipUnless(hasattr(os, "getuid") and hasattr(os, "getgid"), + "Missing getuid or getgid implementation") def add_dir_and_getmember(self, name): + def filter(tarinfo): + tarinfo.uid = tarinfo.gid = 100 + return tarinfo + with os_helper.temp_cwd(): with tarfile.open(tmpname, 'w') as tar: tar.format = tarfile.USTAR_FORMAT try: os.mkdir(name) - tar.add(name) + tar.add(name, filter=filter) finally: os.rmdir(name) with tarfile.open(tmpname) as tar: diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 7a460d94469fe7..d61dc6e2fbd70b 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -23,6 +23,7 @@ from typing import assert_type, cast, runtime_checkable from typing import get_type_hints from typing import get_origin, get_args +from typing import override from typing import is_typeddict from typing import reveal_type from typing import dataclass_transform @@ -4166,6 +4167,43 @@ def cached(self): ... self.assertIs(True, Methods.cached.__final__) +class OverrideDecoratorTests(BaseTestCase): + def test_override(self): + class Base: + def normal_method(self): ... + @staticmethod + def static_method_good_order(): ... + @staticmethod + def static_method_bad_order(): ... + @staticmethod + def decorator_with_slots(): ... + + class Derived(Base): + @override + def normal_method(self): + return 42 + + @staticmethod + @override + def static_method_good_order(): + return 42 + + @override + @staticmethod + def static_method_bad_order(): + return 42 + + + self.assertIsSubclass(Derived, Base) + instance = Derived() + self.assertEqual(instance.normal_method(), 42) + self.assertIs(True, instance.normal_method.__override__) + self.assertEqual(Derived.static_method_good_order(), 42) + self.assertIs(True, Derived.static_method_good_order.__override__) + self.assertEqual(Derived.static_method_bad_order(), 42) + self.assertIs(False, hasattr(Derived.static_method_bad_order, "__override__")) + + class CastTests(BaseTestCase): def test_basics(self): diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 498c0382d2137b..633d596ac3de3f 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1827,6 +1827,7 @@ def test_HTTPError_interface(self): def test_gh_98778(self): x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) self.assertEqual(getattr(x, "__notes__", ()), ()) + self.assertIsInstance(x.fp.read(), bytes) def test_parse_proxy(self): parse_proxy_test_cases = [ diff --git a/Lib/test/test_zipfile/_context.py b/Lib/test/test_zipfile/_context.py deleted file mode 100644 index 348798aa08d452..00000000000000 --- a/Lib/test/test_zipfile/_context.py +++ /dev/null @@ -1,30 +0,0 @@ -import contextlib -import time - - -class DeadlineExceeded(Exception): - pass - - -class TimedContext(contextlib.ContextDecorator): - """ - A context that will raise DeadlineExceeded if the - max duration is reached during the execution. - - >>> TimedContext(1)(time.sleep)(.1) - >>> TimedContext(0)(time.sleep)(.1) - Traceback (most recent call last): - ... - tests._context.DeadlineExceeded: (..., 0) - """ - - def __init__(self, max_duration: int): - self.max_duration = max_duration - - def __enter__(self): - self.start = time.monotonic() - - def __exit__(self, *err): - duration = time.monotonic() - self.start - if duration > self.max_duration: - raise DeadlineExceeded(duration, self.max_duration) diff --git a/Lib/test/test_zipfile/_func_timeout_compat.py b/Lib/test/test_zipfile/_func_timeout_compat.py deleted file mode 100644 index b1f2b2698a8538..00000000000000 --- a/Lib/test/test_zipfile/_func_timeout_compat.py +++ /dev/null @@ -1,8 +0,0 @@ -try: - from func_timeout import func_set_timeout as set_timeout -except ImportError: # pragma: no cover - # provide a fallback that doesn't actually time out - from ._context import TimedContext as set_timeout - - -__all__ = ['set_timeout'] diff --git a/Lib/test/test_zipfile/_itertools.py b/Lib/test/test_zipfile/_itertools.py index 74f01fe5ba3de2..f735dd21733006 100644 --- a/Lib/test/test_zipfile/_itertools.py +++ b/Lib/test/test_zipfile/_itertools.py @@ -1,4 +1,6 @@ import itertools +from collections import deque +from itertools import islice # from jaraco.itertools 6.3.0 @@ -39,3 +41,39 @@ def always_iterable(obj, base_type=(str, bytes)): return iter(obj) except TypeError: return iter((obj,)) + + +# from more_itertools v9.0.0 +def consume(iterator, n=None): + """Advance *iterable* by *n* steps. If *n* is ``None``, consume it + entirely. + Efficiently exhausts an iterator without returning values. Defaults to + consuming the whole iterator, but an optional second argument may be + provided to limit consumption. + >>> i = (x for x in range(10)) + >>> next(i) + 0 + >>> consume(i, 3) + >>> next(i) + 4 + >>> consume(i) + >>> next(i) + Traceback (most recent call last): + File "", line 1, in + StopIteration + If the iterator has fewer items remaining than the provided limit, the + whole iterator will be consumed. + >>> i = (x for x in range(3)) + >>> consume(i, 5) + >>> next(i) + Traceback (most recent call last): + File "", line 1, in + StopIteration + """ + # Use functions that consume iterators at C speed. + if n is None: + # feed the entire iterator into a zero-length deque + deque(iterator, maxlen=0) + else: + # advance to the empty slice starting at position n + next(islice(iterator, n, n), None) diff --git a/Lib/test/test_zipfile/_support.py b/Lib/test/test_zipfile/_support.py new file mode 100644 index 00000000000000..1afdf3b3a773d7 --- /dev/null +++ b/Lib/test/test_zipfile/_support.py @@ -0,0 +1,9 @@ +import importlib +import unittest + + +def import_or_skip(name): + try: + return importlib.import_module(name) + except ImportError: # pragma: no cover + raise unittest.SkipTest(f'Unable to import {name}') diff --git a/Lib/test/test_zipfile/test_complexity.py b/Lib/test/test_zipfile/test_complexity.py new file mode 100644 index 00000000000000..3432dc39e56c4e --- /dev/null +++ b/Lib/test/test_zipfile/test_complexity.py @@ -0,0 +1,24 @@ +import unittest +import string +import zipfile + +from ._functools import compose +from ._itertools import consume + +from ._support import import_or_skip + + +big_o = import_or_skip('big_o') + + +class TestComplexity(unittest.TestCase): + def test_implied_dirs_performance(self): + best, others = big_o.big_o( + compose(consume, zipfile.CompleteDirs._implied_dirs), + lambda size: [ + '/'.join(string.ascii_lowercase + str(n)) for n in range(size) + ], + max_n=1000, + min_n=1, + ) + assert best <= big_o.complexities.Linear diff --git a/Lib/test/test_zipfile/test_path.py b/Lib/test/test_zipfile/test_path.py index 92fda690b2fc50..aff91e53995875 100644 --- a/Lib/test/test_zipfile/test_path.py +++ b/Lib/test/test_zipfile/test_path.py @@ -3,7 +3,6 @@ import contextlib import pathlib import pickle -import string import sys import unittest import zipfile @@ -12,7 +11,6 @@ from ._itertools import Counter from ._test_params import parameterize, Invoked -from ._func_timeout_compat import set_timeout from test.support.os_helper import temp_dir @@ -22,9 +20,6 @@ class itertools: Counter = Counter -consume = tuple - - def add_dirs(zf): """ Given a writable zip file zf, inject directory entries for @@ -330,11 +325,6 @@ def test_joinpath_constant_time(self): # Check the file iterated all items assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES - @set_timeout(3) - def test_implied_dirs_performance(self): - data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)] - zipfile.CompleteDirs._implied_dirs(data) - @pass_alpharep def test_read_does_not_close(self, alpharep): alpharep = self.zipfile_ondisk(alpharep) @@ -512,7 +502,7 @@ def test_pickle(self, alpharep, path_type, subpath): saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) restored_1 = pickle.loads(saved_1) first, *rest = restored_1.iterdir() - assert first.read_text().startswith('content of ') + assert first.read_text(encoding='utf-8').startswith('content of ') @pass_alpharep def test_extract_orig_with_implied_dirs(self, alpharep): @@ -524,3 +514,12 @@ def test_extract_orig_with_implied_dirs(self, alpharep): # wrap the zipfile for its side effect zipfile.Path(zf) zf.extractall(source_path.parent) + + @pass_alpharep + def test_getinfo_missing(self, alpharep): + """ + Validate behavior of getinfo on original zipfile after wrapping. + """ + zipfile.Path(alpharep) + with self.assertRaises(KeyError): + alpharep.getinfo('does-not-exist') diff --git a/Lib/typing.py b/Lib/typing.py index bdf51bb5f41595..8d40e923bb1d08 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -138,6 +138,7 @@ def _idfunc(_, x): 'NoReturn', 'NotRequired', 'overload', + 'override', 'ParamSpecArgs', 'ParamSpecKwargs', 'Required', @@ -2657,6 +2658,7 @@ class Other(Leaf): # Error reported by type checker # Internal type variable used for Type[]. CT_co = TypeVar('CT_co', covariant=True, bound=type) + # A useful type variable with constraints. This represents string types. # (This one *is* for export!) AnyStr = TypeVar('AnyStr', bytes, str) @@ -2748,6 +2750,8 @@ def new_user(user_class: Type[U]) -> U: At this point the type checker knows that joe has type BasicUser. """ +# Internal type variable for callables. Not for export. +F = TypeVar("F", bound=Callable[..., Any]) @runtime_checkable class SupportsInt(Protocol): @@ -3448,3 +3452,40 @@ def decorator(cls_or_fn): } return cls_or_fn return decorator + + + +def override(method: F, /) -> F: + """Indicate that a method is intended to override a method in a base class. + + Usage: + + class Base: + def method(self) -> None: ... + pass + + class Child(Base): + @override + def method(self) -> None: + super().method() + + When this decorator is applied to a method, the type checker will + validate that it overrides a method or attribute with the same name on a + base class. This helps prevent bugs that may occur when a base class is + changed without an equivalent change to a child class. + + There is no runtime checking of this property. The decorator sets the + ``__override__`` attribute to ``True`` on the decorated object to allow + runtime introspection. + + See PEP 698 for details. + + """ + try: + method.__override__ = True + except (AttributeError, TypeError): + # Skip the attribute silently if it is not writable. + # AttributeError happens if the object has __slots__ or a + # read-only property, TypeError if it's a builtin class. + pass + return method diff --git a/Lib/urllib/error.py b/Lib/urllib/error.py index feec0e7f848e46..a9cd1ecadd63f2 100644 --- a/Lib/urllib/error.py +++ b/Lib/urllib/error.py @@ -43,7 +43,7 @@ def __init__(self, url, code, msg, hdrs, fp): self.fp = fp self.filename = url if fp is None: - fp = io.StringIO() + fp = io.BytesIO() self.__super_init(fp, hdrs, url, code) def __str__(self): diff --git a/Lib/zipfile/_path.py b/Lib/zipfile/_path.py index c2c804e96d6b6c..fd49a3ea91db59 100644 --- a/Lib/zipfile/_path.py +++ b/Lib/zipfile/_path.py @@ -86,6 +86,11 @@ class CompleteDirs(InitializedState, zipfile.ZipFile): """ A ZipFile subclass that ensures that implied directories are always included in the namelist. + + >>> list(CompleteDirs._implied_dirs(['foo/bar.txt', 'foo/bar/baz.txt'])) + ['foo/', 'foo/bar/'] + >>> list(CompleteDirs._implied_dirs(['foo/bar.txt', 'foo/bar/baz.txt', 'foo/bar/'])) + ['foo/'] """ @staticmethod @@ -215,7 +220,7 @@ class Path: Read text: - >>> c.read_text() + >>> c.read_text(encoding='utf-8') 'content of c' existence: diff --git a/Makefile.pre.in b/Makefile.pre.in index b28c6067a535b8..b12a1bc060af90 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -578,17 +578,21 @@ LIBEXPAT_HEADERS= \ LIBHACL_SHA2_OBJS= \ Modules/_hacl/Hacl_Streaming_SHA2.o -LIBHACL_SHA2_HEADERS= \ - Modules/_hacl/Hacl_Streaming_SHA2.h \ +LIBHACL_HEADERS= \ Modules/_hacl/include/krml/FStar_UInt128_Verified.h \ Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \ Modules/_hacl/include/krml/fstar_uint128_struct_endianness.h \ Modules/_hacl/include/krml/internal/target.h \ Modules/_hacl/include/krml/lowstar_endianness.h \ Modules/_hacl/include/krml/types.h \ - Modules/_hacl/internal/Hacl_SHA2_Generic.h \ + Modules/_hacl/Hacl_Streaming_Types.h \ Modules/_hacl/python_hacl_namespaces.h +LIBHACL_SHA2_HEADERS= \ + Modules/_hacl/Hacl_Streaming_SHA2.h \ + Modules/_hacl/internal/Hacl_SHA2_Generic.h \ + $(LIBHACL_HEADERS) + ######################################################################### # Rules @@ -2635,8 +2639,8 @@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h -MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h -MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h +MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_MD5.h Modules/_hacl/Hacl_Hash_MD5.c +MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) Modules/_hacl/Hacl_Hash_SHA1.h Modules/_hacl/Hacl_Hash_SHA1.c MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A) MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c diff --git a/Misc/ACKS b/Misc/ACKS index ca92608868f23f..2da3d0ab29b81d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -927,8 +927,10 @@ Tyler Kieft Mads Kiilerich Jason Killen Derek D. Kim +Gihwan Kim Jan Kim Taek Joo Kim +Yeojin Kim Sam Kimbrel Tomohiko Kinebuchi James King @@ -1228,6 +1230,7 @@ The Dragon De Monsyne Bastien Montagne Skip Montanaro Peter Moody +HyunKyun Moon Alan D. Moore Nicolai Moore Paul Moore @@ -1305,6 +1308,7 @@ Jon Oberheide Milan Oberkirch Pascal Oberndoerfer Géry Ogam +Seonkyo Ok Jeffrey Ollie Adam Olsen Bryan Olson @@ -1844,6 +1848,7 @@ Tom Tromey John Tromp Diane Trout Jason Trowbridge +Steven Troxler Brent Tubbs Anthony Tuininga Erno Tukia @@ -1897,6 +1902,7 @@ Kurt Vile Norman Vine Pauli Virtanen Frank Visser +Long Vo Johannes Vogel Michael Vogt Radu Voicilas diff --git a/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst b/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst new file mode 100644 index 00000000000000..63a640a9cdf733 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst @@ -0,0 +1,2 @@ +On Android, python.pc now correctly reports the library to link to, the same +as python-config.sh. diff --git a/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst b/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst new file mode 100644 index 00000000000000..20db25ddd0c1f6 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst @@ -0,0 +1,3 @@ +Introduced the *Unstable C API tier*, marking APi that is allowed to change +in minor releases without a deprecation period. +See :pep:`689` for details. diff --git a/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst b/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst index fc694f6e051b53..27294a9e5179c4 100644 --- a/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst +++ b/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst @@ -1,13 +1,10 @@ -Add new C-API functions for saving and restoring the current exception: -``PyErr_GetRaisedException`` and ``PyErr_SetRaisedException``. -These functions take and return a single exception rather than -the triple of ``PyErr_Fetch`` and ``PyErr_Restore``. +Add :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException` +for saving and restoring the current exception. +These functions return and accept a single exception object, +rather than the triple arguments of the now-deprecated +:c:func:`PyErr_Fetch` and :c:func:`PyErr_Restore`. This is less error prone and a bit more efficient. -The three arguments forms of saving and restoring the -current exception: ``PyErr_Fetch`` and ``PyErr_Restore`` -are deprecated. - -Also add ``PyException_GetArgs`` and ``PyException_SetArgs`` -as convenience functions to help dealing with -exceptions in the C API. +Add :c:func:`PyException_GetArgs` and :c:func:`PyException_SetArgs` +as convenience functions for retrieving and modifying +the :attr:`~BaseException.args` passed to the exception's constructor. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst deleted file mode 100644 index 72291d042a0394..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-03-10-21-48-05.bpo-46978.f5QFfw.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed docstrings for in-place operators of built-in types. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst new file mode 100644 index 00000000000000..2addef27b8ea4d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst @@ -0,0 +1,3 @@ +Remove gi_code field from generator (and coroutine and async generator) +objects as it is redundant. The frame already includes a reference to the +code object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst new file mode 100644 index 00000000000000..cc99779a944ec6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst @@ -0,0 +1 @@ +Fix SystemError / segmentation fault in iter ``__reduce__`` when internal access of ``builtins.__dict__`` keys mutates the iter object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst new file mode 100644 index 00000000000000..53bbc860c53f37 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst @@ -0,0 +1,2 @@ +Restrict the scope of the :opcode:`FOR_ITER_RANGE` instruction to the scope of the +original :opcode:`FOR_ITER` instruction, to allow instrumentation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst new file mode 100644 index 00000000000000..514a8ef26594dc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst @@ -0,0 +1,2 @@ +Use ``GetCurrentProcessId`` on Windows when ``getpid`` is unavailable. Patch by +Max Bachmann. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst new file mode 100644 index 00000000000000..68c43688c3df03 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst @@ -0,0 +1 @@ +Fix deadlock at shutdown when clearing thread states if any finalizer tries to acquire the runtime head lock. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst new file mode 100644 index 00000000000000..17ab0cd4367991 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst @@ -0,0 +1 @@ +Fixed a segfault occurring when the interpreter calls a ``__bool__`` method that raises. diff --git a/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst b/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst new file mode 100644 index 00000000000000..8909753c7f9ee6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst @@ -0,0 +1,6 @@ +Fix segfaults when creating :class:`lzma.LZMADecompressor` and +:class:`bz2.BZ2Decompressor` objects without calling ``__init__()``, and fix +leakage of locks and internal buffers when calling the ``__init__()`` +methods of :class:`lzma.LZMADecompressor`, :class:`lzma.LZMACompressor`, +:class:`bz2.BZ2Compressor`, and :class:`bz2.BZ2Decompressor` objects +multiple times. diff --git a/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst b/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst new file mode 100644 index 00000000000000..2f6a4153062e5a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst @@ -0,0 +1 @@ +Add a new decorator :func:`typing.override`. See :pep:`698` for details. Patch by Steven Troxler. diff --git a/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst b/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst new file mode 100644 index 00000000000000..b577d93d28c2ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst @@ -0,0 +1,2 @@ +Optimize :class:`fractions.Fraction` for small components. The private argument +``_normalize`` of the :class:`fractions.Fraction` constructor has been removed. diff --git a/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst b/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst new file mode 100644 index 00000000000000..a17927500bd9a5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst @@ -0,0 +1 @@ +Remove locking behavior from :func:`functools.cached_property`. diff --git a/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst b/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst new file mode 100644 index 00000000000000..f9dfd46d1ed430 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst @@ -0,0 +1 @@ +Upgrade pip wheel bundled with ensurepip (pip 23.0.1) diff --git a/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst b/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst new file mode 100644 index 00000000000000..55841da44b1146 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst @@ -0,0 +1,2 @@ +The default value of ``fp`` becomes :class:`io.BytesIO` if :exc:`~urllib.error.HTTPError` +is initialized without a designated ``fp`` parameter. Patch by Long Vo. diff --git a/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst b/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst new file mode 100644 index 00000000000000..a3d4119e7cbdce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst @@ -0,0 +1,2 @@ +For the binary mode, :func:`fileinput.hookcompressed` doesn't set the ``encoding`` value +even if the value is ``None``. Patch by Gihwan Kim. diff --git a/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst b/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst new file mode 100644 index 00000000000000..48acce1d863ea6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst @@ -0,0 +1,2 @@ +Add :data:`mmap.MAP_ALIGNED_SUPER` FreeBSD and :data:`mmap.MAP_CONCEAL` +OpenBSD constants to :mod:`mmap`. Patch by Yeojin Kim. diff --git a/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst b/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst new file mode 100644 index 00000000000000..2667ff120fd402 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst @@ -0,0 +1 @@ +:meth:`platform.platform` now has boolean default arguments. diff --git a/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst b/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst new file mode 100644 index 00000000000000..723d8a43a09f9e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst @@ -0,0 +1,2 @@ +Replace builtin hashlib implementations of MD5 and SHA1 with verified ones +from the HACL* project. diff --git a/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst b/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst new file mode 100644 index 00000000000000..63e36046d26dfe --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst @@ -0,0 +1,2 @@ +Fix deadlock on shutdown if ``test_current_{exception,frames}`` fails. Patch +by Jacob Bower. diff --git a/Misc/python.pc.in b/Misc/python.pc.in index 87e04decc2a2d5..027dba38585a89 100644 --- a/Misc/python.pc.in +++ b/Misc/python.pc.in @@ -9,5 +9,5 @@ Description: Build a C extension for Python Requires: Version: @VERSION@ Libs.private: @LIBS@ -Libs: +Libs: -L${libdir} @LIBPYTHON@ Cflags: -I${includedir}/python@VERSION@@ABIFLAGS@ diff --git a/Modules/Setup b/Modules/Setup index 1d5183bc2df118..1c6f2f7ea5182d 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -163,8 +163,8 @@ PYTHONPATH=$(COREPYTHONPATH) # hashing builtins #_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c -#_md5 md5module.c -#_sha1 sha1module.c +#_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE +#_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE #_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a #_sha3 _sha3/sha3module.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 8f5e14a4e80e22..b12290d436cbeb 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -77,8 +77,8 @@ @MODULE_READLINE_TRUE@readline readline.c # hashing builtins, can be disabled with --without-builtin-hashlib-hashes -@MODULE__MD5_TRUE@_md5 md5module.c -@MODULE__SHA1_TRUE@_sha1 sha1module.c +@MODULE__MD5_TRUE@_md5 md5module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_MD5.c -D_BSD_SOURCE -D_DEFAULT_SOURCE +@MODULE__SHA1_TRUE@_sha1 sha1module.c -I$(srcdir)/Modules/_hacl/include _hacl/Hacl_Hash_SHA1.c -D_BSD_SOURCE -D_DEFAULT_SOURCE @MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a @MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c @MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_abc.c b/Modules/_abc.c index e146d4fd0cac39..9d6654b4e58aad 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -79,7 +79,7 @@ abc_data_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } - state = PyType_GetModuleState(type); + state = _PyType_GetModuleState(type); if (state == NULL) { Py_DECREF(self); return NULL; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 055dded05431df..13d98eedf32f0e 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -76,7 +76,7 @@ get_asyncio_state(PyObject *mod) static inline asyncio_state * get_asyncio_state_by_cls(PyTypeObject *cls) { - asyncio_state *state = (asyncio_state *)PyType_GetModuleState(cls); + asyncio_state *state = (asyncio_state *)_PyType_GetModuleState(cls); assert(state != NULL); return state; } @@ -1422,7 +1422,6 @@ _asyncio_Future__make_cancelled_error_impl(FutureObj *self) static void FutureObj_finalize(FutureObj *fut) { - PyObject *error_type, *error_value, *error_traceback; PyObject *context; PyObject *message = NULL; PyObject *func; @@ -1434,7 +1433,7 @@ FutureObj_finalize(FutureObj *fut) fut->fut_log_tb = 0; /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); context = PyDict_New(); if (context == NULL) { @@ -1476,7 +1475,7 @@ FutureObj_finalize(FutureObj *fut) Py_XDECREF(message); /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } static PyMethodDef FutureType_methods[] = { @@ -2491,14 +2490,13 @@ TaskObj_finalize(TaskObj *task) PyObject *context; PyObject *message = NULL; PyObject *func; - PyObject *error_type, *error_value, *error_traceback; if (task->task_state != STATE_PENDING || !task->task_log_destroy_pending) { goto done; } /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); context = PyDict_New(); if (context == NULL) { @@ -2541,7 +2539,7 @@ TaskObj_finalize(TaskObj *task) Py_XDECREF(message); /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); done: FutureObj_finalize((FutureObj*)task); @@ -2766,8 +2764,6 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) } if (gen_status == PYGEN_RETURN || gen_status == PYGEN_ERROR) { - PyObject *et, *ev, *tb; - if (result != NULL) { /* The error is StopIteration and that means that the underlying coroutine has resolved */ @@ -2794,52 +2790,39 @@ task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) if (PyErr_ExceptionMatches(state->asyncio_CancelledError)) { /* CancelledError */ - PyErr_Fetch(&et, &ev, &tb); - assert(et); - PyErr_NormalizeException(&et, &ev, &tb); - if (tb != NULL) { - PyException_SetTraceback(ev, tb); - Py_DECREF(tb); - } - Py_XDECREF(et); + + PyObject *exc = PyErr_GetRaisedException(); + assert(exc); FutureObj *fut = (FutureObj*)task; /* transfer ownership */ - fut->fut_cancelled_exc = ev; + fut->fut_cancelled_exc = exc; return future_cancel(state, fut, NULL); } /* Some other exception; pop it and call Task.set_exception() */ - PyErr_Fetch(&et, &ev, &tb); - assert(et); - PyErr_NormalizeException(&et, &ev, &tb); - if (tb != NULL) { - PyException_SetTraceback(ev, tb); - } + PyObject *exc = PyErr_GetRaisedException(); + assert(exc); - o = future_set_exception(state, (FutureObj*)task, ev); + o = future_set_exception(state, (FutureObj*)task, exc); if (!o) { /* An exception in Task.set_exception() */ - Py_DECREF(et); - Py_XDECREF(tb); - Py_XDECREF(ev); + Py_DECREF(exc); goto fail; } assert(o == Py_None); Py_DECREF(o); - if (PyErr_GivenExceptionMatches(et, PyExc_KeyboardInterrupt) || - PyErr_GivenExceptionMatches(et, PyExc_SystemExit)) + if (PyErr_GivenExceptionMatches(exc, PyExc_KeyboardInterrupt) || + PyErr_GivenExceptionMatches(exc, PyExc_SystemExit)) { /* We've got a KeyboardInterrupt or a SystemError; re-raise it */ - PyErr_Restore(et, ev, tb); + PyErr_SetRaisedException(exc); goto fail; } - Py_DECREF(et); - Py_XDECREF(tb); - Py_XDECREF(ev); + Py_DECREF(exc); Py_RETURN_NONE; } @@ -3059,10 +3042,9 @@ task_step(asyncio_state *state, TaskObj *task, PyObject *exc) res = task_step_impl(state, task, exc); if (res == NULL) { - PyObject *et, *ev, *tb; - PyErr_Fetch(&et, &ev, &tb); + PyObject *exc = PyErr_GetRaisedException(); leave_task(state, task->task_loop, (PyObject*)task); - _PyErr_ChainExceptions(et, ev, tb); /* Normalizes (et, ev, tb) */ + _PyErr_ChainExceptions1(exc); return NULL; } else { @@ -3079,7 +3061,6 @@ task_step(asyncio_state *state, TaskObj *task, PyObject *exc) static PyObject * task_wakeup(TaskObj *task, PyObject *o) { - PyObject *et, *ev, *tb; PyObject *result; assert(o); @@ -3111,18 +3092,12 @@ task_wakeup(TaskObj *task, PyObject *o) /* exception raised */ } - PyErr_Fetch(&et, &ev, &tb); - assert(et); - PyErr_NormalizeException(&et, &ev, &tb); - if (tb != NULL) { - PyException_SetTraceback(ev, tb); - } + PyObject *exc = PyErr_GetRaisedException(); + assert(exc); - result = task_step(state, task, ev); + result = task_step(state, task, exc); - Py_DECREF(et); - Py_XDECREF(tb); - Py_XDECREF(ev); + Py_DECREF(exc); return result; } diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 9ceb3ae46fe56d..d3bec535ee512d 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -66,7 +66,7 @@ internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t if (sq_item == NULL) { return -1; } - if (Py_EnterRecursiveCall("in _bisect.bisect_right") < 0) { + if (Py_EnterRecursiveCall("in _bisect.bisect_right")) { return -1; } PyTypeObject *tp = Py_TYPE(item); @@ -246,7 +246,7 @@ internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t h if (sq_item == NULL) { return -1; } - if (Py_EnterRecursiveCall("in _bisect.bisect_left") < 0) { + if (Py_EnterRecursiveCall("in _bisect.bisect_left")) { return -1; } PyTypeObject *tp = Py_TYPE(item); diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 9304c13fbed5fc..8e7b8e8078af4e 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -15,6 +15,29 @@ #error "The maximum block size accepted by libbzip2 is UINT32_MAX." #endif +typedef struct { + PyTypeObject *bz2_compressor_type; + PyTypeObject *bz2_decompressor_type; +} _bz2_state; + +static inline _bz2_state * +get_module_state(PyObject *module) +{ + void *state = PyModule_GetState(module); + assert(state != NULL); + return (_bz2_state *)state; +} + +static struct PyModuleDef _bz2module; + +static inline _bz2_state * +find_module_state_by_def(PyTypeObject *type) +{ + PyObject *module = PyType_GetModuleByDef(type, &_bz2module); + assert(module != NULL); + return get_module_state(module); +} + /* On success, return value >= 0 On failure, return -1 */ static inline Py_ssize_t @@ -214,12 +237,14 @@ compress(BZ2Compressor *c, char *data, size_t len, int action) /*[clinic input] module _bz2 -class _bz2.BZ2Compressor "BZ2Compressor *" "&BZ2Compressor_Type" -class _bz2.BZ2Decompressor "BZ2Decompressor *" "&BZ2Decompressor_Type" +class _bz2.BZ2Compressor "BZ2Compressor *" "clinic_state()->bz2_compressor_type" +class _bz2.BZ2Decompressor "BZ2Decompressor *" "clinic_state()->bz2_decompressor_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=dc7d7992a79f9cb7]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=92348121632b94c4]*/ +#define clinic_state() (find_module_state_by_def(type)) #include "clinic/_bz2module.c.h" +#undef clinic_state /*[clinic input] _bz2.BZ2Compressor.compress @@ -295,24 +320,43 @@ BZ2_Free(void* ctx, void *ptr) PyMem_RawFree(ptr); } +/*[clinic input] +@classmethod +_bz2.BZ2Compressor.__new__ + + compresslevel: int = 9 + Compression level, as a number between 1 and 9. + / -/* Argument Clinic is not used since the Argument Clinic always want to - check the type which would be wrong here */ -static int -_bz2_BZ2Compressor___init___impl(BZ2Compressor *self, int compresslevel) +Create a compressor object for compressing data incrementally. + +For one-shot compression, use the compress() function instead. +[clinic start generated code]*/ + +static PyObject * +_bz2_BZ2Compressor_impl(PyTypeObject *type, int compresslevel) +/*[clinic end generated code: output=83346c96beaacad7 input=d4500d2a52c8b263]*/ { int bzerror; + BZ2Compressor *self; if (!(1 <= compresslevel && compresslevel <= 9)) { PyErr_SetString(PyExc_ValueError, "compresslevel must be between 1 and 9"); - return -1; + return NULL; + } + + assert(type != NULL && type->tp_alloc != NULL); + self = (BZ2Compressor *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; } self->lock = PyThread_allocate_lock(); if (self->lock == NULL) { + Py_DECREF(self); PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return -1; + return NULL; } self->bzs.opaque = NULL; @@ -322,49 +366,11 @@ _bz2_BZ2Compressor___init___impl(BZ2Compressor *self, int compresslevel) if (catch_bz2_error(bzerror)) goto error; - return 0; + return (PyObject *)self; error: - PyThread_free_lock(self->lock); - self->lock = NULL; - return -1; -} - -PyDoc_STRVAR(_bz2_BZ2Compressor___init____doc__, -"BZ2Compressor(compresslevel=9, /)\n" -"--\n" -"\n" -"Create a compressor object for compressing data incrementally.\n" -"\n" -" compresslevel\n" -" Compression level, as a number between 1 and 9.\n" -"\n" -"For one-shot compression, use the compress() function instead."); - -static int -_bz2_BZ2Compressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int return_value = -1; - int compresslevel = 9; - - if (!_PyArg_NoKeywords("BZ2Compressor", kwargs)) { - goto exit; - } - if (!_PyArg_CheckPositional("BZ2Compressor", PyTuple_GET_SIZE(args), 0, 1)) { - goto exit; - } - if (PyTuple_GET_SIZE(args) < 1) { - goto skip_optional; - } - compresslevel = _PyLong_AsInt(PyTuple_GET_ITEM(args, 0)); - if (compresslevel == -1 && PyErr_Occurred()) { - goto exit; - } -skip_optional: - return_value = _bz2_BZ2Compressor___init___impl((BZ2Compressor *)self, compresslevel); - -exit: - return return_value; + Py_DECREF(self); + return NULL; } static void @@ -395,9 +401,8 @@ static PyMethodDef BZ2Compressor_methods[] = { static PyType_Slot bz2_compressor_type_slots[] = { {Py_tp_dealloc, BZ2Compressor_dealloc}, {Py_tp_methods, BZ2Compressor_methods}, - {Py_tp_init, _bz2_BZ2Compressor___init__}, - {Py_tp_new, PyType_GenericNew}, - {Py_tp_doc, (char *)_bz2_BZ2Compressor___init____doc__}, + {Py_tp_new, _bz2_BZ2Compressor}, + {Py_tp_doc, (char *)_bz2_BZ2Compressor__doc__}, {Py_tp_traverse, BZ2Compressor_traverse}, {0, 0} }; @@ -624,28 +629,40 @@ _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, return result; } -/* Argument Clinic is not used since the Argument Clinic always want to - check the type which would be wrong here */ -static int -_bz2_BZ2Decompressor___init___impl(BZ2Decompressor *self) +/*[clinic input] +@classmethod +_bz2.BZ2Decompressor.__new__ + +Create a decompressor object for decompressing data incrementally. + +For one-shot decompression, use the decompress() function instead. +[clinic start generated code]*/ + +static PyObject * +_bz2_BZ2Decompressor_impl(PyTypeObject *type) +/*[clinic end generated code: output=5150d51ccaab220e input=b87413ce51853528]*/ { + BZ2Decompressor *self; int bzerror; - PyThread_type_lock lock = PyThread_allocate_lock(); - if (lock == NULL) { - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return -1; + assert(type != NULL && type->tp_alloc != NULL); + self = (BZ2Decompressor *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; } - if (self->lock != NULL) { - PyThread_free_lock(self->lock); + + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + Py_DECREF(self); + PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); + return NULL; } - self->lock = lock; self->needs_input = 1; self->bzs_avail_in_real = 0; self->input_buffer = NULL; self->input_buffer_size = 0; - Py_XSETREF(self->unused_data, PyBytes_FromStringAndSize(NULL, 0)); + self->unused_data = PyBytes_FromStringAndSize(NULL, 0); if (self->unused_data == NULL) goto error; @@ -653,40 +670,13 @@ _bz2_BZ2Decompressor___init___impl(BZ2Decompressor *self) if (catch_bz2_error(bzerror)) goto error; - return 0; + return (PyObject *)self; error: - Py_CLEAR(self->unused_data); - PyThread_free_lock(self->lock); - self->lock = NULL; - return -1; -} - -static int -_bz2_BZ2Decompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int return_value = -1; - - if (!_PyArg_NoPositional("BZ2Decompressor", args)) { - goto exit; - } - if (!_PyArg_NoKeywords("BZ2Decompressor", kwargs)) { - goto exit; - } - return_value = _bz2_BZ2Decompressor___init___impl((BZ2Decompressor *)self); - -exit: - return return_value; + Py_DECREF(self); + return NULL; } -PyDoc_STRVAR(_bz2_BZ2Decompressor___init____doc__, -"BZ2Decompressor()\n" -"--\n" -"\n" -"Create a decompressor object for decompressing data incrementally.\n" -"\n" -"For one-shot decompression, use the decompress() function instead."); - static void BZ2Decompressor_dealloc(BZ2Decompressor *self) { @@ -738,10 +728,9 @@ static PyMemberDef BZ2Decompressor_members[] = { static PyType_Slot bz2_decompressor_type_slots[] = { {Py_tp_dealloc, BZ2Decompressor_dealloc}, {Py_tp_methods, BZ2Decompressor_methods}, - {Py_tp_init, _bz2_BZ2Decompressor___init__}, - {Py_tp_doc, (char *)_bz2_BZ2Decompressor___init____doc__}, + {Py_tp_doc, (char *)_bz2_BZ2Decompressor__doc__}, {Py_tp_members, BZ2Decompressor_members}, - {Py_tp_new, PyType_GenericNew}, + {Py_tp_new, _bz2_BZ2Decompressor}, {Py_tp_traverse, BZ2Decompressor_traverse}, {0, 0} }; @@ -762,31 +751,52 @@ static PyType_Spec bz2_decompressor_type_spec = { static int _bz2_exec(PyObject *module) { - PyTypeObject *bz2_compressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + _bz2_state *state = get_module_state(module); + state->bz2_compressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &bz2_compressor_type_spec, NULL); - if (bz2_compressor_type == NULL) { + if (state->bz2_compressor_type == NULL) { return -1; } - int rc = PyModule_AddType(module, bz2_compressor_type); - Py_DECREF(bz2_compressor_type); - if (rc < 0) { + if (PyModule_AddType(module, state->bz2_compressor_type) < 0) { return -1; } - PyTypeObject *bz2_decompressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + state->bz2_decompressor_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, &bz2_decompressor_type_spec, NULL); - if (bz2_decompressor_type == NULL) { + if (state->bz2_decompressor_type == NULL) { return -1; } - rc = PyModule_AddType(module, bz2_decompressor_type); - Py_DECREF(bz2_decompressor_type); - if (rc < 0) { + if (PyModule_AddType(module, state->bz2_decompressor_type) < 0) { return -1; } return 0; } +static int +_bz2_traverse(PyObject *module, visitproc visit, void *arg) +{ + _bz2_state *state = get_module_state(module); + Py_VISIT(state->bz2_compressor_type); + Py_VISIT(state->bz2_decompressor_type); + return 0; +} + +static int +_bz2_clear(PyObject *module) +{ + _bz2_state *state = get_module_state(module); + Py_CLEAR(state->bz2_compressor_type); + Py_CLEAR(state->bz2_decompressor_type); + return 0; +} + +static void +_bz2_free(void *module) +{ + (void)_bz2_clear((PyObject *)module); +} + static struct PyModuleDef_Slot _bz2_slots[] = { {Py_mod_exec, _bz2_exec}, {0, NULL} @@ -795,6 +805,10 @@ static struct PyModuleDef_Slot _bz2_slots[] = { static struct PyModuleDef _bz2module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_bz2", + .m_size = sizeof(_bz2_state), + .m_traverse = _bz2_traverse, + .m_clear = _bz2_clear, + .m_free = _bz2_free, .m_slots = _bz2_slots, }; diff --git a/Modules/_hacl/Hacl_Hash_MD5.c b/Modules/_hacl/Hacl_Hash_MD5.c new file mode 100644 index 00000000000000..2c613066d9f682 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_MD5.c @@ -0,0 +1,1472 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_MD5.h" + +static uint32_t +_h0[4U] = + { (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U }; + +static uint32_t +_t[64U] = + { + (uint32_t)0xd76aa478U, (uint32_t)0xe8c7b756U, (uint32_t)0x242070dbU, (uint32_t)0xc1bdceeeU, + (uint32_t)0xf57c0fafU, (uint32_t)0x4787c62aU, (uint32_t)0xa8304613U, (uint32_t)0xfd469501U, + (uint32_t)0x698098d8U, (uint32_t)0x8b44f7afU, (uint32_t)0xffff5bb1U, (uint32_t)0x895cd7beU, + (uint32_t)0x6b901122U, (uint32_t)0xfd987193U, (uint32_t)0xa679438eU, (uint32_t)0x49b40821U, + (uint32_t)0xf61e2562U, (uint32_t)0xc040b340U, (uint32_t)0x265e5a51U, (uint32_t)0xe9b6c7aaU, + (uint32_t)0xd62f105dU, (uint32_t)0x02441453U, (uint32_t)0xd8a1e681U, (uint32_t)0xe7d3fbc8U, + (uint32_t)0x21e1cde6U, (uint32_t)0xc33707d6U, (uint32_t)0xf4d50d87U, (uint32_t)0x455a14edU, + (uint32_t)0xa9e3e905U, (uint32_t)0xfcefa3f8U, (uint32_t)0x676f02d9U, (uint32_t)0x8d2a4c8aU, + (uint32_t)0xfffa3942U, (uint32_t)0x8771f681U, (uint32_t)0x6d9d6122U, (uint32_t)0xfde5380cU, + (uint32_t)0xa4beea44U, (uint32_t)0x4bdecfa9U, (uint32_t)0xf6bb4b60U, (uint32_t)0xbebfbc70U, + (uint32_t)0x289b7ec6U, (uint32_t)0xeaa127faU, (uint32_t)0xd4ef3085U, (uint32_t)0x4881d05U, + (uint32_t)0xd9d4d039U, (uint32_t)0xe6db99e5U, (uint32_t)0x1fa27cf8U, (uint32_t)0xc4ac5665U, + (uint32_t)0xf4292244U, (uint32_t)0x432aff97U, (uint32_t)0xab9423a7U, (uint32_t)0xfc93a039U, + (uint32_t)0x655b59c3U, (uint32_t)0x8f0ccc92U, (uint32_t)0xffeff47dU, (uint32_t)0x85845dd1U, + (uint32_t)0x6fa87e4fU, (uint32_t)0xfe2ce6e0U, (uint32_t)0xa3014314U, (uint32_t)0x4e0811a1U, + (uint32_t)0xf7537e82U, (uint32_t)0xbd3af235U, (uint32_t)0x2ad7d2bbU, (uint32_t)0xeb86d391U + }; + +void Hacl_Hash_Core_MD5_legacy_init(uint32_t *s) +{ + KRML_MAYBE_FOR4(i, (uint32_t)0U, (uint32_t)4U, (uint32_t)1U, s[i] = _h0[i];); +} + +static void legacy_update(uint32_t *abcd, uint8_t *x) +{ + uint32_t aa = abcd[0U]; + uint32_t bb = abcd[1U]; + uint32_t cc = abcd[2U]; + uint32_t dd = abcd[3U]; + uint32_t va = abcd[0U]; + uint32_t vb0 = abcd[1U]; + uint32_t vc0 = abcd[2U]; + uint32_t vd0 = abcd[3U]; + uint8_t *b0 = x; + uint32_t u = load32_le(b0); + uint32_t xk = u; + uint32_t ti0 = _t[0U]; + uint32_t + v = + vb0 + + + ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) + << (uint32_t)7U + | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> (uint32_t)25U); + abcd[0U] = v; + uint32_t va0 = abcd[3U]; + uint32_t vb1 = abcd[0U]; + uint32_t vc1 = abcd[1U]; + uint32_t vd1 = abcd[2U]; + uint8_t *b1 = x + (uint32_t)4U; + uint32_t u0 = load32_le(b1); + uint32_t xk0 = u0; + uint32_t ti1 = _t[1U]; + uint32_t + v0 = + vb1 + + + ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) + << (uint32_t)12U + | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> (uint32_t)20U); + abcd[3U] = v0; + uint32_t va1 = abcd[2U]; + uint32_t vb2 = abcd[3U]; + uint32_t vc2 = abcd[0U]; + uint32_t vd2 = abcd[1U]; + uint8_t *b2 = x + (uint32_t)8U; + uint32_t u1 = load32_le(b2); + uint32_t xk1 = u1; + uint32_t ti2 = _t[2U]; + uint32_t + v1 = + vb2 + + + ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) + << (uint32_t)17U + | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> (uint32_t)15U); + abcd[2U] = v1; + uint32_t va2 = abcd[1U]; + uint32_t vb3 = abcd[2U]; + uint32_t vc3 = abcd[3U]; + uint32_t vd3 = abcd[0U]; + uint8_t *b3 = x + (uint32_t)12U; + uint32_t u2 = load32_le(b3); + uint32_t xk2 = u2; + uint32_t ti3 = _t[3U]; + uint32_t + v2 = + vb3 + + + ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) + << (uint32_t)22U + | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> (uint32_t)10U); + abcd[1U] = v2; + uint32_t va3 = abcd[0U]; + uint32_t vb4 = abcd[1U]; + uint32_t vc4 = abcd[2U]; + uint32_t vd4 = abcd[3U]; + uint8_t *b4 = x + (uint32_t)16U; + uint32_t u3 = load32_le(b4); + uint32_t xk3 = u3; + uint32_t ti4 = _t[4U]; + uint32_t + v3 = + vb4 + + + ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) + << (uint32_t)7U + | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> (uint32_t)25U); + abcd[0U] = v3; + uint32_t va4 = abcd[3U]; + uint32_t vb5 = abcd[0U]; + uint32_t vc5 = abcd[1U]; + uint32_t vd5 = abcd[2U]; + uint8_t *b5 = x + (uint32_t)20U; + uint32_t u4 = load32_le(b5); + uint32_t xk4 = u4; + uint32_t ti5 = _t[5U]; + uint32_t + v4 = + vb5 + + + ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) + << (uint32_t)12U + | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> (uint32_t)20U); + abcd[3U] = v4; + uint32_t va5 = abcd[2U]; + uint32_t vb6 = abcd[3U]; + uint32_t vc6 = abcd[0U]; + uint32_t vd6 = abcd[1U]; + uint8_t *b6 = x + (uint32_t)24U; + uint32_t u5 = load32_le(b6); + uint32_t xk5 = u5; + uint32_t ti6 = _t[6U]; + uint32_t + v5 = + vb6 + + + ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) + << (uint32_t)17U + | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> (uint32_t)15U); + abcd[2U] = v5; + uint32_t va6 = abcd[1U]; + uint32_t vb7 = abcd[2U]; + uint32_t vc7 = abcd[3U]; + uint32_t vd7 = abcd[0U]; + uint8_t *b7 = x + (uint32_t)28U; + uint32_t u6 = load32_le(b7); + uint32_t xk6 = u6; + uint32_t ti7 = _t[7U]; + uint32_t + v6 = + vb7 + + + ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) + << (uint32_t)22U + | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> (uint32_t)10U); + abcd[1U] = v6; + uint32_t va7 = abcd[0U]; + uint32_t vb8 = abcd[1U]; + uint32_t vc8 = abcd[2U]; + uint32_t vd8 = abcd[3U]; + uint8_t *b8 = x + (uint32_t)32U; + uint32_t u7 = load32_le(b8); + uint32_t xk7 = u7; + uint32_t ti8 = _t[8U]; + uint32_t + v7 = + vb8 + + + ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) + << (uint32_t)7U + | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> (uint32_t)25U); + abcd[0U] = v7; + uint32_t va8 = abcd[3U]; + uint32_t vb9 = abcd[0U]; + uint32_t vc9 = abcd[1U]; + uint32_t vd9 = abcd[2U]; + uint8_t *b9 = x + (uint32_t)36U; + uint32_t u8 = load32_le(b9); + uint32_t xk8 = u8; + uint32_t ti9 = _t[9U]; + uint32_t + v8 = + vb9 + + + ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) + << (uint32_t)12U + | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> (uint32_t)20U); + abcd[3U] = v8; + uint32_t va9 = abcd[2U]; + uint32_t vb10 = abcd[3U]; + uint32_t vc10 = abcd[0U]; + uint32_t vd10 = abcd[1U]; + uint8_t *b10 = x + (uint32_t)40U; + uint32_t u9 = load32_le(b10); + uint32_t xk9 = u9; + uint32_t ti10 = _t[10U]; + uint32_t + v9 = + vb10 + + + ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) + << (uint32_t)17U + | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> (uint32_t)15U); + abcd[2U] = v9; + uint32_t va10 = abcd[1U]; + uint32_t vb11 = abcd[2U]; + uint32_t vc11 = abcd[3U]; + uint32_t vd11 = abcd[0U]; + uint8_t *b11 = x + (uint32_t)44U; + uint32_t u10 = load32_le(b11); + uint32_t xk10 = u10; + uint32_t ti11 = _t[11U]; + uint32_t + v10 = + vb11 + + + ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) + << (uint32_t)22U + | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> (uint32_t)10U); + abcd[1U] = v10; + uint32_t va11 = abcd[0U]; + uint32_t vb12 = abcd[1U]; + uint32_t vc12 = abcd[2U]; + uint32_t vd12 = abcd[3U]; + uint8_t *b12 = x + (uint32_t)48U; + uint32_t u11 = load32_le(b12); + uint32_t xk11 = u11; + uint32_t ti12 = _t[12U]; + uint32_t + v11 = + vb12 + + + ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) + << (uint32_t)7U + | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> (uint32_t)25U); + abcd[0U] = v11; + uint32_t va12 = abcd[3U]; + uint32_t vb13 = abcd[0U]; + uint32_t vc13 = abcd[1U]; + uint32_t vd13 = abcd[2U]; + uint8_t *b13 = x + (uint32_t)52U; + uint32_t u12 = load32_le(b13); + uint32_t xk12 = u12; + uint32_t ti13 = _t[13U]; + uint32_t + v12 = + vb13 + + + ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) + << (uint32_t)12U + | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> (uint32_t)20U); + abcd[3U] = v12; + uint32_t va13 = abcd[2U]; + uint32_t vb14 = abcd[3U]; + uint32_t vc14 = abcd[0U]; + uint32_t vd14 = abcd[1U]; + uint8_t *b14 = x + (uint32_t)56U; + uint32_t u13 = load32_le(b14); + uint32_t xk13 = u13; + uint32_t ti14 = _t[14U]; + uint32_t + v13 = + vb14 + + + ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) + << (uint32_t)17U + | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> (uint32_t)15U); + abcd[2U] = v13; + uint32_t va14 = abcd[1U]; + uint32_t vb15 = abcd[2U]; + uint32_t vc15 = abcd[3U]; + uint32_t vd15 = abcd[0U]; + uint8_t *b15 = x + (uint32_t)60U; + uint32_t u14 = load32_le(b15); + uint32_t xk14 = u14; + uint32_t ti15 = _t[15U]; + uint32_t + v14 = + vb15 + + + ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) + << (uint32_t)22U + | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> (uint32_t)10U); + abcd[1U] = v14; + uint32_t va15 = abcd[0U]; + uint32_t vb16 = abcd[1U]; + uint32_t vc16 = abcd[2U]; + uint32_t vd16 = abcd[3U]; + uint8_t *b16 = x + (uint32_t)4U; + uint32_t u15 = load32_le(b16); + uint32_t xk15 = u15; + uint32_t ti16 = _t[16U]; + uint32_t + v15 = + vb16 + + + ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) + << (uint32_t)5U + | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> (uint32_t)27U); + abcd[0U] = v15; + uint32_t va16 = abcd[3U]; + uint32_t vb17 = abcd[0U]; + uint32_t vc17 = abcd[1U]; + uint32_t vd17 = abcd[2U]; + uint8_t *b17 = x + (uint32_t)24U; + uint32_t u16 = load32_le(b17); + uint32_t xk16 = u16; + uint32_t ti17 = _t[17U]; + uint32_t + v16 = + vb17 + + + ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) + << (uint32_t)9U + | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> (uint32_t)23U); + abcd[3U] = v16; + uint32_t va17 = abcd[2U]; + uint32_t vb18 = abcd[3U]; + uint32_t vc18 = abcd[0U]; + uint32_t vd18 = abcd[1U]; + uint8_t *b18 = x + (uint32_t)44U; + uint32_t u17 = load32_le(b18); + uint32_t xk17 = u17; + uint32_t ti18 = _t[18U]; + uint32_t + v17 = + vb18 + + + ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) + << (uint32_t)14U + | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> (uint32_t)18U); + abcd[2U] = v17; + uint32_t va18 = abcd[1U]; + uint32_t vb19 = abcd[2U]; + uint32_t vc19 = abcd[3U]; + uint32_t vd19 = abcd[0U]; + uint8_t *b19 = x; + uint32_t u18 = load32_le(b19); + uint32_t xk18 = u18; + uint32_t ti19 = _t[19U]; + uint32_t + v18 = + vb19 + + + ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) + << (uint32_t)20U + | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> (uint32_t)12U); + abcd[1U] = v18; + uint32_t va19 = abcd[0U]; + uint32_t vb20 = abcd[1U]; + uint32_t vc20 = abcd[2U]; + uint32_t vd20 = abcd[3U]; + uint8_t *b20 = x + (uint32_t)20U; + uint32_t u19 = load32_le(b20); + uint32_t xk19 = u19; + uint32_t ti20 = _t[20U]; + uint32_t + v19 = + vb20 + + + ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) + << (uint32_t)5U + | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> (uint32_t)27U); + abcd[0U] = v19; + uint32_t va20 = abcd[3U]; + uint32_t vb21 = abcd[0U]; + uint32_t vc21 = abcd[1U]; + uint32_t vd21 = abcd[2U]; + uint8_t *b21 = x + (uint32_t)40U; + uint32_t u20 = load32_le(b21); + uint32_t xk20 = u20; + uint32_t ti21 = _t[21U]; + uint32_t + v20 = + vb21 + + + ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) + << (uint32_t)9U + | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> (uint32_t)23U); + abcd[3U] = v20; + uint32_t va21 = abcd[2U]; + uint32_t vb22 = abcd[3U]; + uint32_t vc22 = abcd[0U]; + uint32_t vd22 = abcd[1U]; + uint8_t *b22 = x + (uint32_t)60U; + uint32_t u21 = load32_le(b22); + uint32_t xk21 = u21; + uint32_t ti22 = _t[22U]; + uint32_t + v21 = + vb22 + + + ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) + << (uint32_t)14U + | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> (uint32_t)18U); + abcd[2U] = v21; + uint32_t va22 = abcd[1U]; + uint32_t vb23 = abcd[2U]; + uint32_t vc23 = abcd[3U]; + uint32_t vd23 = abcd[0U]; + uint8_t *b23 = x + (uint32_t)16U; + uint32_t u22 = load32_le(b23); + uint32_t xk22 = u22; + uint32_t ti23 = _t[23U]; + uint32_t + v22 = + vb23 + + + ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) + << (uint32_t)20U + | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> (uint32_t)12U); + abcd[1U] = v22; + uint32_t va23 = abcd[0U]; + uint32_t vb24 = abcd[1U]; + uint32_t vc24 = abcd[2U]; + uint32_t vd24 = abcd[3U]; + uint8_t *b24 = x + (uint32_t)36U; + uint32_t u23 = load32_le(b24); + uint32_t xk23 = u23; + uint32_t ti24 = _t[24U]; + uint32_t + v23 = + vb24 + + + ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) + << (uint32_t)5U + | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> (uint32_t)27U); + abcd[0U] = v23; + uint32_t va24 = abcd[3U]; + uint32_t vb25 = abcd[0U]; + uint32_t vc25 = abcd[1U]; + uint32_t vd25 = abcd[2U]; + uint8_t *b25 = x + (uint32_t)56U; + uint32_t u24 = load32_le(b25); + uint32_t xk24 = u24; + uint32_t ti25 = _t[25U]; + uint32_t + v24 = + vb25 + + + ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) + << (uint32_t)9U + | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> (uint32_t)23U); + abcd[3U] = v24; + uint32_t va25 = abcd[2U]; + uint32_t vb26 = abcd[3U]; + uint32_t vc26 = abcd[0U]; + uint32_t vd26 = abcd[1U]; + uint8_t *b26 = x + (uint32_t)12U; + uint32_t u25 = load32_le(b26); + uint32_t xk25 = u25; + uint32_t ti26 = _t[26U]; + uint32_t + v25 = + vb26 + + + ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) + << (uint32_t)14U + | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> (uint32_t)18U); + abcd[2U] = v25; + uint32_t va26 = abcd[1U]; + uint32_t vb27 = abcd[2U]; + uint32_t vc27 = abcd[3U]; + uint32_t vd27 = abcd[0U]; + uint8_t *b27 = x + (uint32_t)32U; + uint32_t u26 = load32_le(b27); + uint32_t xk26 = u26; + uint32_t ti27 = _t[27U]; + uint32_t + v26 = + vb27 + + + ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) + << (uint32_t)20U + | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> (uint32_t)12U); + abcd[1U] = v26; + uint32_t va27 = abcd[0U]; + uint32_t vb28 = abcd[1U]; + uint32_t vc28 = abcd[2U]; + uint32_t vd28 = abcd[3U]; + uint8_t *b28 = x + (uint32_t)52U; + uint32_t u27 = load32_le(b28); + uint32_t xk27 = u27; + uint32_t ti28 = _t[28U]; + uint32_t + v27 = + vb28 + + + ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) + << (uint32_t)5U + | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> (uint32_t)27U); + abcd[0U] = v27; + uint32_t va28 = abcd[3U]; + uint32_t vb29 = abcd[0U]; + uint32_t vc29 = abcd[1U]; + uint32_t vd29 = abcd[2U]; + uint8_t *b29 = x + (uint32_t)8U; + uint32_t u28 = load32_le(b29); + uint32_t xk28 = u28; + uint32_t ti29 = _t[29U]; + uint32_t + v28 = + vb29 + + + ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) + << (uint32_t)9U + | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> (uint32_t)23U); + abcd[3U] = v28; + uint32_t va29 = abcd[2U]; + uint32_t vb30 = abcd[3U]; + uint32_t vc30 = abcd[0U]; + uint32_t vd30 = abcd[1U]; + uint8_t *b30 = x + (uint32_t)28U; + uint32_t u29 = load32_le(b30); + uint32_t xk29 = u29; + uint32_t ti30 = _t[30U]; + uint32_t + v29 = + vb30 + + + ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) + << (uint32_t)14U + | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> (uint32_t)18U); + abcd[2U] = v29; + uint32_t va30 = abcd[1U]; + uint32_t vb31 = abcd[2U]; + uint32_t vc31 = abcd[3U]; + uint32_t vd31 = abcd[0U]; + uint8_t *b31 = x + (uint32_t)48U; + uint32_t u30 = load32_le(b31); + uint32_t xk30 = u30; + uint32_t ti31 = _t[31U]; + uint32_t + v30 = + vb31 + + + ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) + << (uint32_t)20U + | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> (uint32_t)12U); + abcd[1U] = v30; + uint32_t va31 = abcd[0U]; + uint32_t vb32 = abcd[1U]; + uint32_t vc32 = abcd[2U]; + uint32_t vd32 = abcd[3U]; + uint8_t *b32 = x + (uint32_t)20U; + uint32_t u31 = load32_le(b32); + uint32_t xk31 = u31; + uint32_t ti32 = _t[32U]; + uint32_t + v31 = + vb32 + + + ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) + << (uint32_t)4U + | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> (uint32_t)28U); + abcd[0U] = v31; + uint32_t va32 = abcd[3U]; + uint32_t vb33 = abcd[0U]; + uint32_t vc33 = abcd[1U]; + uint32_t vd33 = abcd[2U]; + uint8_t *b33 = x + (uint32_t)32U; + uint32_t u32 = load32_le(b33); + uint32_t xk32 = u32; + uint32_t ti33 = _t[33U]; + uint32_t + v32 = + vb33 + + + ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) + << (uint32_t)11U + | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> (uint32_t)21U); + abcd[3U] = v32; + uint32_t va33 = abcd[2U]; + uint32_t vb34 = abcd[3U]; + uint32_t vc34 = abcd[0U]; + uint32_t vd34 = abcd[1U]; + uint8_t *b34 = x + (uint32_t)44U; + uint32_t u33 = load32_le(b34); + uint32_t xk33 = u33; + uint32_t ti34 = _t[34U]; + uint32_t + v33 = + vb34 + + + ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) + << (uint32_t)16U + | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> (uint32_t)16U); + abcd[2U] = v33; + uint32_t va34 = abcd[1U]; + uint32_t vb35 = abcd[2U]; + uint32_t vc35 = abcd[3U]; + uint32_t vd35 = abcd[0U]; + uint8_t *b35 = x + (uint32_t)56U; + uint32_t u34 = load32_le(b35); + uint32_t xk34 = u34; + uint32_t ti35 = _t[35U]; + uint32_t + v34 = + vb35 + + + ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) + << (uint32_t)23U + | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> (uint32_t)9U); + abcd[1U] = v34; + uint32_t va35 = abcd[0U]; + uint32_t vb36 = abcd[1U]; + uint32_t vc36 = abcd[2U]; + uint32_t vd36 = abcd[3U]; + uint8_t *b36 = x + (uint32_t)4U; + uint32_t u35 = load32_le(b36); + uint32_t xk35 = u35; + uint32_t ti36 = _t[36U]; + uint32_t + v35 = + vb36 + + + ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) + << (uint32_t)4U + | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> (uint32_t)28U); + abcd[0U] = v35; + uint32_t va36 = abcd[3U]; + uint32_t vb37 = abcd[0U]; + uint32_t vc37 = abcd[1U]; + uint32_t vd37 = abcd[2U]; + uint8_t *b37 = x + (uint32_t)16U; + uint32_t u36 = load32_le(b37); + uint32_t xk36 = u36; + uint32_t ti37 = _t[37U]; + uint32_t + v36 = + vb37 + + + ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) + << (uint32_t)11U + | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> (uint32_t)21U); + abcd[3U] = v36; + uint32_t va37 = abcd[2U]; + uint32_t vb38 = abcd[3U]; + uint32_t vc38 = abcd[0U]; + uint32_t vd38 = abcd[1U]; + uint8_t *b38 = x + (uint32_t)28U; + uint32_t u37 = load32_le(b38); + uint32_t xk37 = u37; + uint32_t ti38 = _t[38U]; + uint32_t + v37 = + vb38 + + + ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) + << (uint32_t)16U + | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> (uint32_t)16U); + abcd[2U] = v37; + uint32_t va38 = abcd[1U]; + uint32_t vb39 = abcd[2U]; + uint32_t vc39 = abcd[3U]; + uint32_t vd39 = abcd[0U]; + uint8_t *b39 = x + (uint32_t)40U; + uint32_t u38 = load32_le(b39); + uint32_t xk38 = u38; + uint32_t ti39 = _t[39U]; + uint32_t + v38 = + vb39 + + + ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) + << (uint32_t)23U + | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> (uint32_t)9U); + abcd[1U] = v38; + uint32_t va39 = abcd[0U]; + uint32_t vb40 = abcd[1U]; + uint32_t vc40 = abcd[2U]; + uint32_t vd40 = abcd[3U]; + uint8_t *b40 = x + (uint32_t)52U; + uint32_t u39 = load32_le(b40); + uint32_t xk39 = u39; + uint32_t ti40 = _t[40U]; + uint32_t + v39 = + vb40 + + + ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) + << (uint32_t)4U + | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> (uint32_t)28U); + abcd[0U] = v39; + uint32_t va40 = abcd[3U]; + uint32_t vb41 = abcd[0U]; + uint32_t vc41 = abcd[1U]; + uint32_t vd41 = abcd[2U]; + uint8_t *b41 = x; + uint32_t u40 = load32_le(b41); + uint32_t xk40 = u40; + uint32_t ti41 = _t[41U]; + uint32_t + v40 = + vb41 + + + ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) + << (uint32_t)11U + | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> (uint32_t)21U); + abcd[3U] = v40; + uint32_t va41 = abcd[2U]; + uint32_t vb42 = abcd[3U]; + uint32_t vc42 = abcd[0U]; + uint32_t vd42 = abcd[1U]; + uint8_t *b42 = x + (uint32_t)12U; + uint32_t u41 = load32_le(b42); + uint32_t xk41 = u41; + uint32_t ti42 = _t[42U]; + uint32_t + v41 = + vb42 + + + ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) + << (uint32_t)16U + | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> (uint32_t)16U); + abcd[2U] = v41; + uint32_t va42 = abcd[1U]; + uint32_t vb43 = abcd[2U]; + uint32_t vc43 = abcd[3U]; + uint32_t vd43 = abcd[0U]; + uint8_t *b43 = x + (uint32_t)24U; + uint32_t u42 = load32_le(b43); + uint32_t xk42 = u42; + uint32_t ti43 = _t[43U]; + uint32_t + v42 = + vb43 + + + ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) + << (uint32_t)23U + | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> (uint32_t)9U); + abcd[1U] = v42; + uint32_t va43 = abcd[0U]; + uint32_t vb44 = abcd[1U]; + uint32_t vc44 = abcd[2U]; + uint32_t vd44 = abcd[3U]; + uint8_t *b44 = x + (uint32_t)36U; + uint32_t u43 = load32_le(b44); + uint32_t xk43 = u43; + uint32_t ti44 = _t[44U]; + uint32_t + v43 = + vb44 + + + ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) + << (uint32_t)4U + | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> (uint32_t)28U); + abcd[0U] = v43; + uint32_t va44 = abcd[3U]; + uint32_t vb45 = abcd[0U]; + uint32_t vc45 = abcd[1U]; + uint32_t vd45 = abcd[2U]; + uint8_t *b45 = x + (uint32_t)48U; + uint32_t u44 = load32_le(b45); + uint32_t xk44 = u44; + uint32_t ti45 = _t[45U]; + uint32_t + v44 = + vb45 + + + ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) + << (uint32_t)11U + | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> (uint32_t)21U); + abcd[3U] = v44; + uint32_t va45 = abcd[2U]; + uint32_t vb46 = abcd[3U]; + uint32_t vc46 = abcd[0U]; + uint32_t vd46 = abcd[1U]; + uint8_t *b46 = x + (uint32_t)60U; + uint32_t u45 = load32_le(b46); + uint32_t xk45 = u45; + uint32_t ti46 = _t[46U]; + uint32_t + v45 = + vb46 + + + ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) + << (uint32_t)16U + | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> (uint32_t)16U); + abcd[2U] = v45; + uint32_t va46 = abcd[1U]; + uint32_t vb47 = abcd[2U]; + uint32_t vc47 = abcd[3U]; + uint32_t vd47 = abcd[0U]; + uint8_t *b47 = x + (uint32_t)8U; + uint32_t u46 = load32_le(b47); + uint32_t xk46 = u46; + uint32_t ti47 = _t[47U]; + uint32_t + v46 = + vb47 + + + ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) + << (uint32_t)23U + | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> (uint32_t)9U); + abcd[1U] = v46; + uint32_t va47 = abcd[0U]; + uint32_t vb48 = abcd[1U]; + uint32_t vc48 = abcd[2U]; + uint32_t vd48 = abcd[3U]; + uint8_t *b48 = x; + uint32_t u47 = load32_le(b48); + uint32_t xk47 = u47; + uint32_t ti48 = _t[48U]; + uint32_t + v47 = + vb48 + + + ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) + << (uint32_t)6U + | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> (uint32_t)26U); + abcd[0U] = v47; + uint32_t va48 = abcd[3U]; + uint32_t vb49 = abcd[0U]; + uint32_t vc49 = abcd[1U]; + uint32_t vd49 = abcd[2U]; + uint8_t *b49 = x + (uint32_t)28U; + uint32_t u48 = load32_le(b49); + uint32_t xk48 = u48; + uint32_t ti49 = _t[49U]; + uint32_t + v48 = + vb49 + + + ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) + << (uint32_t)10U + | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> (uint32_t)22U); + abcd[3U] = v48; + uint32_t va49 = abcd[2U]; + uint32_t vb50 = abcd[3U]; + uint32_t vc50 = abcd[0U]; + uint32_t vd50 = abcd[1U]; + uint8_t *b50 = x + (uint32_t)56U; + uint32_t u49 = load32_le(b50); + uint32_t xk49 = u49; + uint32_t ti50 = _t[50U]; + uint32_t + v49 = + vb50 + + + ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) + << (uint32_t)15U + | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> (uint32_t)17U); + abcd[2U] = v49; + uint32_t va50 = abcd[1U]; + uint32_t vb51 = abcd[2U]; + uint32_t vc51 = abcd[3U]; + uint32_t vd51 = abcd[0U]; + uint8_t *b51 = x + (uint32_t)20U; + uint32_t u50 = load32_le(b51); + uint32_t xk50 = u50; + uint32_t ti51 = _t[51U]; + uint32_t + v50 = + vb51 + + + ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) + << (uint32_t)21U + | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> (uint32_t)11U); + abcd[1U] = v50; + uint32_t va51 = abcd[0U]; + uint32_t vb52 = abcd[1U]; + uint32_t vc52 = abcd[2U]; + uint32_t vd52 = abcd[3U]; + uint8_t *b52 = x + (uint32_t)48U; + uint32_t u51 = load32_le(b52); + uint32_t xk51 = u51; + uint32_t ti52 = _t[52U]; + uint32_t + v51 = + vb52 + + + ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) + << (uint32_t)6U + | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> (uint32_t)26U); + abcd[0U] = v51; + uint32_t va52 = abcd[3U]; + uint32_t vb53 = abcd[0U]; + uint32_t vc53 = abcd[1U]; + uint32_t vd53 = abcd[2U]; + uint8_t *b53 = x + (uint32_t)12U; + uint32_t u52 = load32_le(b53); + uint32_t xk52 = u52; + uint32_t ti53 = _t[53U]; + uint32_t + v52 = + vb53 + + + ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) + << (uint32_t)10U + | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> (uint32_t)22U); + abcd[3U] = v52; + uint32_t va53 = abcd[2U]; + uint32_t vb54 = abcd[3U]; + uint32_t vc54 = abcd[0U]; + uint32_t vd54 = abcd[1U]; + uint8_t *b54 = x + (uint32_t)40U; + uint32_t u53 = load32_le(b54); + uint32_t xk53 = u53; + uint32_t ti54 = _t[54U]; + uint32_t + v53 = + vb54 + + + ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) + << (uint32_t)15U + | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> (uint32_t)17U); + abcd[2U] = v53; + uint32_t va54 = abcd[1U]; + uint32_t vb55 = abcd[2U]; + uint32_t vc55 = abcd[3U]; + uint32_t vd55 = abcd[0U]; + uint8_t *b55 = x + (uint32_t)4U; + uint32_t u54 = load32_le(b55); + uint32_t xk54 = u54; + uint32_t ti55 = _t[55U]; + uint32_t + v54 = + vb55 + + + ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) + << (uint32_t)21U + | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> (uint32_t)11U); + abcd[1U] = v54; + uint32_t va55 = abcd[0U]; + uint32_t vb56 = abcd[1U]; + uint32_t vc56 = abcd[2U]; + uint32_t vd56 = abcd[3U]; + uint8_t *b56 = x + (uint32_t)32U; + uint32_t u55 = load32_le(b56); + uint32_t xk55 = u55; + uint32_t ti56 = _t[56U]; + uint32_t + v55 = + vb56 + + + ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) + << (uint32_t)6U + | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> (uint32_t)26U); + abcd[0U] = v55; + uint32_t va56 = abcd[3U]; + uint32_t vb57 = abcd[0U]; + uint32_t vc57 = abcd[1U]; + uint32_t vd57 = abcd[2U]; + uint8_t *b57 = x + (uint32_t)60U; + uint32_t u56 = load32_le(b57); + uint32_t xk56 = u56; + uint32_t ti57 = _t[57U]; + uint32_t + v56 = + vb57 + + + ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) + << (uint32_t)10U + | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> (uint32_t)22U); + abcd[3U] = v56; + uint32_t va57 = abcd[2U]; + uint32_t vb58 = abcd[3U]; + uint32_t vc58 = abcd[0U]; + uint32_t vd58 = abcd[1U]; + uint8_t *b58 = x + (uint32_t)24U; + uint32_t u57 = load32_le(b58); + uint32_t xk57 = u57; + uint32_t ti58 = _t[58U]; + uint32_t + v57 = + vb58 + + + ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) + << (uint32_t)15U + | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> (uint32_t)17U); + abcd[2U] = v57; + uint32_t va58 = abcd[1U]; + uint32_t vb59 = abcd[2U]; + uint32_t vc59 = abcd[3U]; + uint32_t vd59 = abcd[0U]; + uint8_t *b59 = x + (uint32_t)52U; + uint32_t u58 = load32_le(b59); + uint32_t xk58 = u58; + uint32_t ti59 = _t[59U]; + uint32_t + v58 = + vb59 + + + ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) + << (uint32_t)21U + | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> (uint32_t)11U); + abcd[1U] = v58; + uint32_t va59 = abcd[0U]; + uint32_t vb60 = abcd[1U]; + uint32_t vc60 = abcd[2U]; + uint32_t vd60 = abcd[3U]; + uint8_t *b60 = x + (uint32_t)16U; + uint32_t u59 = load32_le(b60); + uint32_t xk59 = u59; + uint32_t ti60 = _t[60U]; + uint32_t + v59 = + vb60 + + + ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) + << (uint32_t)6U + | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> (uint32_t)26U); + abcd[0U] = v59; + uint32_t va60 = abcd[3U]; + uint32_t vb61 = abcd[0U]; + uint32_t vc61 = abcd[1U]; + uint32_t vd61 = abcd[2U]; + uint8_t *b61 = x + (uint32_t)44U; + uint32_t u60 = load32_le(b61); + uint32_t xk60 = u60; + uint32_t ti61 = _t[61U]; + uint32_t + v60 = + vb61 + + + ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) + << (uint32_t)10U + | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> (uint32_t)22U); + abcd[3U] = v60; + uint32_t va61 = abcd[2U]; + uint32_t vb62 = abcd[3U]; + uint32_t vc62 = abcd[0U]; + uint32_t vd62 = abcd[1U]; + uint8_t *b62 = x + (uint32_t)8U; + uint32_t u61 = load32_le(b62); + uint32_t xk61 = u61; + uint32_t ti62 = _t[62U]; + uint32_t + v61 = + vb62 + + + ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) + << (uint32_t)15U + | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> (uint32_t)17U); + abcd[2U] = v61; + uint32_t va62 = abcd[1U]; + uint32_t vb = abcd[2U]; + uint32_t vc = abcd[3U]; + uint32_t vd = abcd[0U]; + uint8_t *b63 = x + (uint32_t)36U; + uint32_t u62 = load32_le(b63); + uint32_t xk62 = u62; + uint32_t ti = _t[63U]; + uint32_t + v62 = + vb + + + ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) + << (uint32_t)21U + | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> (uint32_t)11U); + abcd[1U] = v62; + uint32_t a = abcd[0U]; + uint32_t b = abcd[1U]; + uint32_t c = abcd[2U]; + uint32_t d = abcd[3U]; + abcd[0U] = a + aa; + abcd[1U] = b + bb; + abcd[2U] = c + cc; + abcd[3U] = d + dd; +} + +static void legacy_pad(uint64_t len, uint8_t *dst) +{ + uint8_t *dst1 = dst; + dst1[0U] = (uint8_t)0x80U; + uint8_t *dst2 = dst + (uint32_t)1U; + for + (uint32_t + i = (uint32_t)0U; + i + < ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) % (uint32_t)64U; + i++) + { + dst2[i] = (uint8_t)0U; + } + uint8_t + *dst3 = + dst + + + (uint32_t)1U + + + ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) + % (uint32_t)64U; + store64_le(dst3, len << (uint32_t)3U); +} + +void Hacl_Hash_Core_MD5_legacy_finish(uint32_t *s, uint8_t *dst) +{ + KRML_MAYBE_FOR4(i, + (uint32_t)0U, + (uint32_t)4U, + (uint32_t)1U, + store32_le(dst + i * (uint32_t)4U, s[i]);); +} + +void Hacl_Hash_MD5_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) +{ + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint32_t sz = (uint32_t)64U; + uint8_t *block = blocks + sz * i; + legacy_update(s, block); + } +} + +void +Hacl_Hash_MD5_legacy_update_last( + uint32_t *s, + uint64_t prev_len, + uint8_t *input, + uint32_t input_len +) +{ + uint32_t blocks_n = input_len / (uint32_t)64U; + uint32_t blocks_len = blocks_n * (uint32_t)64U; + uint8_t *blocks = input; + uint32_t rest_len = input_len - blocks_len; + uint8_t *rest = input + blocks_len; + Hacl_Hash_MD5_legacy_update_multi(s, blocks, blocks_n); + uint64_t total_input_len = prev_len + (uint64_t)input_len; + uint32_t + pad_len = + (uint32_t)1U + + + ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(total_input_len % (uint64_t)(uint32_t)64U))) + % (uint32_t)64U + + (uint32_t)8U; + uint32_t tmp_len = rest_len + pad_len; + uint8_t tmp_twoblocks[128U] = { 0U }; + uint8_t *tmp = tmp_twoblocks; + uint8_t *tmp_rest = tmp; + uint8_t *tmp_pad = tmp + rest_len; + memcpy(tmp_rest, rest, rest_len * sizeof (uint8_t)); + legacy_pad(total_input_len, tmp_pad); + Hacl_Hash_MD5_legacy_update_multi(s, tmp, tmp_len / (uint32_t)64U); +} + +void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + uint32_t + s[4U] = + { (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U }; + uint32_t blocks_n0 = input_len / (uint32_t)64U; + uint32_t blocks_n1; + if (input_len % (uint32_t)64U == (uint32_t)0U && blocks_n0 > (uint32_t)0U) + { + blocks_n1 = blocks_n0 - (uint32_t)1U; + } + else + { + blocks_n1 = blocks_n0; + } + uint32_t blocks_len0 = blocks_n1 * (uint32_t)64U; + uint8_t *blocks0 = input; + uint32_t rest_len0 = input_len - blocks_len0; + uint8_t *rest0 = input + blocks_len0; + uint32_t blocks_n = blocks_n1; + uint32_t blocks_len = blocks_len0; + uint8_t *blocks = blocks0; + uint32_t rest_len = rest_len0; + uint8_t *rest = rest0; + Hacl_Hash_MD5_legacy_update_multi(s, blocks, blocks_n); + Hacl_Hash_MD5_legacy_update_last(s, (uint64_t)blocks_len, rest, rest_len); + Hacl_Hash_Core_MD5_legacy_finish(s, dst); +} + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_create_in(void) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)4U, sizeof (uint32_t)); + Hacl_Streaming_MD_state_32 + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); + p[0U] = s; + Hacl_Hash_Core_MD5_legacy_init(block_state); + return p; +} + +void Hacl_Streaming_MD5_legacy_init(Hacl_Streaming_MD_state_32 *s) +{ + Hacl_Streaming_MD_state_32 scrut = *s; + uint8_t *buf = scrut.buf; + uint32_t *block_state = scrut.block_state; + Hacl_Hash_Core_MD5_legacy_init(block_state); + Hacl_Streaming_MD_state_32 + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +/** +0 = success, 1 = max length exceeded +*/ +uint32_t +Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +{ + Hacl_Streaming_MD_state_32 s = *p; + uint64_t total_len = s.total_len; + if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + { + return (uint32_t)1U; + } + uint32_t sz; + if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + { + sz = (uint32_t)64U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + } + if (len <= (uint32_t)64U - sz) + { + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, data, len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)len; + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == (uint32_t)0U) + { + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + if (!(sz1 == (uint32_t)0U)) + { + Hacl_Hash_MD5_legacy_update_multi(block_state1, buf, (uint32_t)1U); + } + uint32_t ite; + if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + { + ite = (uint32_t)64U; + } + else + { + ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + } + uint32_t n_blocks = (len - ite) / (uint32_t)64U; + uint32_t data1_len = n_blocks * (uint32_t)64U; + uint32_t data2_len = len - data1_len; + uint8_t *data1 = data; + uint8_t *data2 = data + data1_len; + Hacl_Hash_MD5_legacy_update_multi(block_state1, data1, data1_len / (uint32_t)64U); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)len + } + ); + } + else + { + uint32_t diff = (uint32_t)64U - sz; + uint8_t *data1 = data; + uint8_t *data2 = data + diff; + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + { + sz10 = (uint32_t)64U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, data1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Streaming_MD_state_32 s10 = *p; + uint32_t *block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + if (!(sz1 == (uint32_t)0U)) + { + Hacl_Hash_MD5_legacy_update_multi(block_state1, buf, (uint32_t)1U); + } + uint32_t ite; + if + ( + (uint64_t)(len - diff) + % (uint64_t)(uint32_t)64U + == (uint64_t)0U + && (uint64_t)(len - diff) > (uint64_t)0U + ) + { + ite = (uint32_t)64U; + } + else + { + ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + } + uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; + uint32_t data1_len = n_blocks * (uint32_t)64U; + uint32_t data2_len = len - diff - data1_len; + uint8_t *data11 = data2; + uint8_t *data21 = data2 + data1_len; + Hacl_Hash_MD5_legacy_update_multi(block_state1, data11, data1_len / (uint32_t)64U); + uint8_t *dst = buf; + memcpy(dst, data21, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(len - diff) + } + ); + } + return (uint32_t)0U; +} + +void Hacl_Streaming_MD5_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +{ + Hacl_Streaming_MD_state_32 scrut = *p; + uint32_t *block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = (uint32_t)64U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + } + uint8_t *buf_1 = buf_; + uint32_t tmp_block_state[4U] = { 0U }; + memcpy(tmp_block_state, block_state, (uint32_t)4U * sizeof (uint32_t)); + uint32_t ite; + if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + { + ite = (uint32_t)64U; + } + else + { + ite = r % (uint32_t)64U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + Hacl_Hash_MD5_legacy_update_multi(tmp_block_state, buf_multi, (uint32_t)0U); + uint64_t prev_len_last = total_len - (uint64_t)r; + Hacl_Hash_MD5_legacy_update_last(tmp_block_state, prev_len_last, buf_last, r); + Hacl_Hash_Core_MD5_legacy_finish(tmp_block_state, dst); +} + +void Hacl_Streaming_MD5_legacy_free(Hacl_Streaming_MD_state_32 *s) +{ + Hacl_Streaming_MD_state_32 scrut = *s; + uint8_t *buf = scrut.buf; + uint32_t *block_state = scrut.block_state; + KRML_HOST_FREE(block_state); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(s); +} + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_state_32 *s0) +{ + Hacl_Streaming_MD_state_32 scrut = *s0; + uint32_t *block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); + memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)4U, sizeof (uint32_t)); + memcpy(block_state, block_state0, (uint32_t)4U * sizeof (uint32_t)); + Hacl_Streaming_MD_state_32 + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); + p[0U] = s; + return p; +} + +void Hacl_Streaming_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + Hacl_Hash_MD5_legacy_hash(input, input_len, dst); +} + diff --git a/Modules/_hacl/Hacl_Hash_MD5.h b/Modules/_hacl/Hacl_Hash_MD5.h new file mode 100644 index 00000000000000..015e3668751b75 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_MD5.h @@ -0,0 +1,65 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_MD5_H +#define __Hacl_Hash_MD5_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + +typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_MD5_state; + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_create_in(void); + +void Hacl_Streaming_MD5_legacy_init(Hacl_Streaming_MD_state_32 *s); + +/** +0 = success, 1 = max length exceeded +*/ +uint32_t +Hacl_Streaming_MD5_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len); + +void Hacl_Streaming_MD5_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); + +void Hacl_Streaming_MD5_legacy_free(Hacl_Streaming_MD_state_32 *s); + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_MD5_legacy_copy(Hacl_Streaming_MD_state_32 *s0); + +void Hacl_Streaming_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_MD5_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Hash_SHA1.c b/Modules/_hacl/Hacl_Hash_SHA1.c new file mode 100644 index 00000000000000..e155e338271c56 --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA1.c @@ -0,0 +1,508 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#include "internal/Hacl_Hash_SHA1.h" + +static uint32_t +_h0[5U] = + { + (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U, + (uint32_t)0xc3d2e1f0U + }; + +void Hacl_Hash_Core_SHA1_legacy_init(uint32_t *s) +{ + KRML_MAYBE_FOR5(i, (uint32_t)0U, (uint32_t)5U, (uint32_t)1U, s[i] = _h0[i];); +} + +static void legacy_update(uint32_t *h, uint8_t *l) +{ + uint32_t ha = h[0U]; + uint32_t hb = h[1U]; + uint32_t hc = h[2U]; + uint32_t hd = h[3U]; + uint32_t he = h[4U]; + uint32_t _w[80U] = { 0U }; + for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + { + uint32_t v; + if (i < (uint32_t)16U) + { + uint8_t *b = l + i * (uint32_t)4U; + uint32_t u = load32_be(b); + v = u; + } + else + { + uint32_t wmit3 = _w[i - (uint32_t)3U]; + uint32_t wmit8 = _w[i - (uint32_t)8U]; + uint32_t wmit14 = _w[i - (uint32_t)14U]; + uint32_t wmit16 = _w[i - (uint32_t)16U]; + v = + (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) + << (uint32_t)1U + | (wmit3 ^ (wmit8 ^ (wmit14 ^ wmit16))) >> (uint32_t)31U; + } + _w[i] = v; + } + for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + { + uint32_t _a = h[0U]; + uint32_t _b = h[1U]; + uint32_t _c = h[2U]; + uint32_t _d = h[3U]; + uint32_t _e = h[4U]; + uint32_t wmit = _w[i]; + uint32_t ite0; + if (i < (uint32_t)20U) + { + ite0 = (_b & _c) ^ (~_b & _d); + } + else if ((uint32_t)39U < i && i < (uint32_t)60U) + { + ite0 = (_b & _c) ^ ((_b & _d) ^ (_c & _d)); + } + else + { + ite0 = _b ^ (_c ^ _d); + } + uint32_t ite; + if (i < (uint32_t)20U) + { + ite = (uint32_t)0x5a827999U; + } + else if (i < (uint32_t)40U) + { + ite = (uint32_t)0x6ed9eba1U; + } + else if (i < (uint32_t)60U) + { + ite = (uint32_t)0x8f1bbcdcU; + } + else + { + ite = (uint32_t)0xca62c1d6U; + } + uint32_t _T = (_a << (uint32_t)5U | _a >> (uint32_t)27U) + ite0 + _e + ite + wmit; + h[0U] = _T; + h[1U] = _a; + h[2U] = _b << (uint32_t)30U | _b >> (uint32_t)2U; + h[3U] = _c; + h[4U] = _d; + } + for (uint32_t i = (uint32_t)0U; i < (uint32_t)80U; i++) + { + _w[i] = (uint32_t)0U; + } + uint32_t sta = h[0U]; + uint32_t stb = h[1U]; + uint32_t stc = h[2U]; + uint32_t std = h[3U]; + uint32_t ste = h[4U]; + h[0U] = sta + ha; + h[1U] = stb + hb; + h[2U] = stc + hc; + h[3U] = std + hd; + h[4U] = ste + he; +} + +static void legacy_pad(uint64_t len, uint8_t *dst) +{ + uint8_t *dst1 = dst; + dst1[0U] = (uint8_t)0x80U; + uint8_t *dst2 = dst + (uint32_t)1U; + for + (uint32_t + i = (uint32_t)0U; + i + < ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) % (uint32_t)64U; + i++) + { + dst2[i] = (uint8_t)0U; + } + uint8_t + *dst3 = + dst + + + (uint32_t)1U + + + ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(len % (uint64_t)(uint32_t)64U))) + % (uint32_t)64U; + store64_be(dst3, len << (uint32_t)3U); +} + +void Hacl_Hash_Core_SHA1_legacy_finish(uint32_t *s, uint8_t *dst) +{ + KRML_MAYBE_FOR5(i, + (uint32_t)0U, + (uint32_t)5U, + (uint32_t)1U, + store32_be(dst + i * (uint32_t)4U, s[i]);); +} + +void Hacl_Hash_SHA1_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks) +{ + for (uint32_t i = (uint32_t)0U; i < n_blocks; i++) + { + uint32_t sz = (uint32_t)64U; + uint8_t *block = blocks + sz * i; + legacy_update(s, block); + } +} + +void +Hacl_Hash_SHA1_legacy_update_last( + uint32_t *s, + uint64_t prev_len, + uint8_t *input, + uint32_t input_len +) +{ + uint32_t blocks_n = input_len / (uint32_t)64U; + uint32_t blocks_len = blocks_n * (uint32_t)64U; + uint8_t *blocks = input; + uint32_t rest_len = input_len - blocks_len; + uint8_t *rest = input + blocks_len; + Hacl_Hash_SHA1_legacy_update_multi(s, blocks, blocks_n); + uint64_t total_input_len = prev_len + (uint64_t)input_len; + uint32_t + pad_len = + (uint32_t)1U + + + ((uint32_t)128U - ((uint32_t)9U + (uint32_t)(total_input_len % (uint64_t)(uint32_t)64U))) + % (uint32_t)64U + + (uint32_t)8U; + uint32_t tmp_len = rest_len + pad_len; + uint8_t tmp_twoblocks[128U] = { 0U }; + uint8_t *tmp = tmp_twoblocks; + uint8_t *tmp_rest = tmp; + uint8_t *tmp_pad = tmp + rest_len; + memcpy(tmp_rest, rest, rest_len * sizeof (uint8_t)); + legacy_pad(total_input_len, tmp_pad); + Hacl_Hash_SHA1_legacy_update_multi(s, tmp, tmp_len / (uint32_t)64U); +} + +void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + uint32_t + s[5U] = + { + (uint32_t)0x67452301U, (uint32_t)0xefcdab89U, (uint32_t)0x98badcfeU, (uint32_t)0x10325476U, + (uint32_t)0xc3d2e1f0U + }; + uint32_t blocks_n0 = input_len / (uint32_t)64U; + uint32_t blocks_n1; + if (input_len % (uint32_t)64U == (uint32_t)0U && blocks_n0 > (uint32_t)0U) + { + blocks_n1 = blocks_n0 - (uint32_t)1U; + } + else + { + blocks_n1 = blocks_n0; + } + uint32_t blocks_len0 = blocks_n1 * (uint32_t)64U; + uint8_t *blocks0 = input; + uint32_t rest_len0 = input_len - blocks_len0; + uint8_t *rest0 = input + blocks_len0; + uint32_t blocks_n = blocks_n1; + uint32_t blocks_len = blocks_len0; + uint8_t *blocks = blocks0; + uint32_t rest_len = rest_len0; + uint8_t *rest = rest0; + Hacl_Hash_SHA1_legacy_update_multi(s, blocks, blocks_n); + Hacl_Hash_SHA1_legacy_update_last(s, (uint64_t)blocks_len, rest, rest_len); + Hacl_Hash_Core_SHA1_legacy_finish(s, dst); +} + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_create_in(void) +{ + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)5U, sizeof (uint32_t)); + Hacl_Streaming_MD_state_32 + s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); + p[0U] = s; + Hacl_Hash_Core_SHA1_legacy_init(block_state); + return p; +} + +void Hacl_Streaming_SHA1_legacy_init(Hacl_Streaming_MD_state_32 *s) +{ + Hacl_Streaming_MD_state_32 scrut = *s; + uint8_t *buf = scrut.buf; + uint32_t *block_state = scrut.block_state; + Hacl_Hash_Core_SHA1_legacy_init(block_state); + Hacl_Streaming_MD_state_32 + tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; + s[0U] = tmp; +} + +/** +0 = success, 1 = max length exceeded +*/ +uint32_t +Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) +{ + Hacl_Streaming_MD_state_32 s = *p; + uint64_t total_len = s.total_len; + if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) + { + return (uint32_t)1U; + } + uint32_t sz; + if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + { + sz = (uint32_t)64U; + } + else + { + sz = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + } + if (len <= (uint32_t)64U - sz) + { + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + uint8_t *buf2 = buf + sz1; + memcpy(buf2, data, len * sizeof (uint8_t)); + uint64_t total_len2 = total_len1 + (uint64_t)len; + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len2 + } + ); + } + else if (sz == (uint32_t)0U) + { + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state1 = s1.block_state; + uint8_t *buf = s1.buf; + uint64_t total_len1 = s1.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + if (!(sz1 == (uint32_t)0U)) + { + Hacl_Hash_SHA1_legacy_update_multi(block_state1, buf, (uint32_t)1U); + } + uint32_t ite; + if ((uint64_t)len % (uint64_t)(uint32_t)64U == (uint64_t)0U && (uint64_t)len > (uint64_t)0U) + { + ite = (uint32_t)64U; + } + else + { + ite = (uint32_t)((uint64_t)len % (uint64_t)(uint32_t)64U); + } + uint32_t n_blocks = (len - ite) / (uint32_t)64U; + uint32_t data1_len = n_blocks * (uint32_t)64U; + uint32_t data2_len = len - data1_len; + uint8_t *data1 = data; + uint8_t *data2 = data + data1_len; + Hacl_Hash_SHA1_legacy_update_multi(block_state1, data1, data1_len / (uint32_t)64U); + uint8_t *dst = buf; + memcpy(dst, data2, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)len + } + ); + } + else + { + uint32_t diff = (uint32_t)64U - sz; + uint8_t *data1 = data; + uint8_t *data2 = data + diff; + Hacl_Streaming_MD_state_32 s1 = *p; + uint32_t *block_state10 = s1.block_state; + uint8_t *buf0 = s1.buf; + uint64_t total_len10 = s1.total_len; + uint32_t sz10; + if (total_len10 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len10 > (uint64_t)0U) + { + sz10 = (uint32_t)64U; + } + else + { + sz10 = (uint32_t)(total_len10 % (uint64_t)(uint32_t)64U); + } + uint8_t *buf2 = buf0 + sz10; + memcpy(buf2, data1, diff * sizeof (uint8_t)); + uint64_t total_len2 = total_len10 + (uint64_t)diff; + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state10, + .buf = buf0, + .total_len = total_len2 + } + ); + Hacl_Streaming_MD_state_32 s10 = *p; + uint32_t *block_state1 = s10.block_state; + uint8_t *buf = s10.buf; + uint64_t total_len1 = s10.total_len; + uint32_t sz1; + if (total_len1 % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len1 > (uint64_t)0U) + { + sz1 = (uint32_t)64U; + } + else + { + sz1 = (uint32_t)(total_len1 % (uint64_t)(uint32_t)64U); + } + if (!(sz1 == (uint32_t)0U)) + { + Hacl_Hash_SHA1_legacy_update_multi(block_state1, buf, (uint32_t)1U); + } + uint32_t ite; + if + ( + (uint64_t)(len - diff) + % (uint64_t)(uint32_t)64U + == (uint64_t)0U + && (uint64_t)(len - diff) > (uint64_t)0U + ) + { + ite = (uint32_t)64U; + } + else + { + ite = (uint32_t)((uint64_t)(len - diff) % (uint64_t)(uint32_t)64U); + } + uint32_t n_blocks = (len - diff - ite) / (uint32_t)64U; + uint32_t data1_len = n_blocks * (uint32_t)64U; + uint32_t data2_len = len - diff - data1_len; + uint8_t *data11 = data2; + uint8_t *data21 = data2 + data1_len; + Hacl_Hash_SHA1_legacy_update_multi(block_state1, data11, data1_len / (uint32_t)64U); + uint8_t *dst = buf; + memcpy(dst, data21, data2_len * sizeof (uint8_t)); + *p + = + ( + (Hacl_Streaming_MD_state_32){ + .block_state = block_state1, + .buf = buf, + .total_len = total_len1 + (uint64_t)(len - diff) + } + ); + } + return (uint32_t)0U; +} + +void Hacl_Streaming_SHA1_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) +{ + Hacl_Streaming_MD_state_32 scrut = *p; + uint32_t *block_state = scrut.block_state; + uint8_t *buf_ = scrut.buf; + uint64_t total_len = scrut.total_len; + uint32_t r; + if (total_len % (uint64_t)(uint32_t)64U == (uint64_t)0U && total_len > (uint64_t)0U) + { + r = (uint32_t)64U; + } + else + { + r = (uint32_t)(total_len % (uint64_t)(uint32_t)64U); + } + uint8_t *buf_1 = buf_; + uint32_t tmp_block_state[5U] = { 0U }; + memcpy(tmp_block_state, block_state, (uint32_t)5U * sizeof (uint32_t)); + uint32_t ite; + if (r % (uint32_t)64U == (uint32_t)0U && r > (uint32_t)0U) + { + ite = (uint32_t)64U; + } + else + { + ite = r % (uint32_t)64U; + } + uint8_t *buf_last = buf_1 + r - ite; + uint8_t *buf_multi = buf_1; + Hacl_Hash_SHA1_legacy_update_multi(tmp_block_state, buf_multi, (uint32_t)0U); + uint64_t prev_len_last = total_len - (uint64_t)r; + Hacl_Hash_SHA1_legacy_update_last(tmp_block_state, prev_len_last, buf_last, r); + Hacl_Hash_Core_SHA1_legacy_finish(tmp_block_state, dst); +} + +void Hacl_Streaming_SHA1_legacy_free(Hacl_Streaming_MD_state_32 *s) +{ + Hacl_Streaming_MD_state_32 scrut = *s; + uint8_t *buf = scrut.buf; + uint32_t *block_state = scrut.block_state; + KRML_HOST_FREE(block_state); + KRML_HOST_FREE(buf); + KRML_HOST_FREE(s); +} + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_state_32 *s0) +{ + Hacl_Streaming_MD_state_32 scrut = *s0; + uint32_t *block_state0 = scrut.block_state; + uint8_t *buf0 = scrut.buf; + uint64_t total_len0 = scrut.total_len; + uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); + memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); + uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)5U, sizeof (uint32_t)); + memcpy(block_state, block_state0, (uint32_t)5U * sizeof (uint32_t)); + Hacl_Streaming_MD_state_32 + s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); + p[0U] = s; + return p; +} + +void Hacl_Streaming_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst) +{ + Hacl_Hash_SHA1_legacy_hash(input, input_len, dst); +} + diff --git a/Modules/_hacl/Hacl_Hash_SHA1.h b/Modules/_hacl/Hacl_Hash_SHA1.h new file mode 100644 index 00000000000000..5e2ae8e713292d --- /dev/null +++ b/Modules/_hacl/Hacl_Hash_SHA1.h @@ -0,0 +1,65 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Hash_SHA1_H +#define __Hacl_Hash_SHA1_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "Hacl_Streaming_Types.h" + +typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA1_state; + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_create_in(void); + +void Hacl_Streaming_SHA1_legacy_init(Hacl_Streaming_MD_state_32 *s); + +/** +0 = success, 1 = max length exceeded +*/ +uint32_t +Hacl_Streaming_SHA1_legacy_update(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len); + +void Hacl_Streaming_SHA1_legacy_finish(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); + +void Hacl_Streaming_SHA1_legacy_free(Hacl_Streaming_MD_state_32 *s); + +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA1_legacy_copy(Hacl_Streaming_MD_state_32 *s0); + +void Hacl_Streaming_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Hash_SHA1_H_DEFINED +#endif diff --git a/Modules/_hacl/Hacl_Streaming_SHA2.c b/Modules/_hacl/Hacl_Streaming_SHA2.c index 8169c7a356731e..69c3be8cdf7fd1 100644 --- a/Modules/_hacl/Hacl_Streaming_SHA2.c +++ b/Modules/_hacl/Hacl_Streaming_SHA2.c @@ -477,17 +477,14 @@ static inline void sha384_finish(uint64_t *st, uint8_t *h) Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. */ -Hacl_Streaming_SHA2_state_sha2_224 *Hacl_Streaming_SHA2_create_in_256(void) +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_256(void) { uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); - Hacl_Streaming_SHA2_state_sha2_224 + Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_SHA2_state_sha2_224 - *p = - (Hacl_Streaming_SHA2_state_sha2_224 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_224 - )); + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; sha256_init(block_state); return p; @@ -499,10 +496,9 @@ The state is to be freed by calling `free_256`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_SHA2_state_sha2_224 -*Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_SHA2_state_sha2_224 *s0) +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state_32 *s0) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *s0; + Hacl_Streaming_MD_state_32 scrut = *s0; uint32_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; @@ -510,13 +506,10 @@ Hacl_Streaming_SHA2_state_sha2_224 memcpy(buf, buf0, (uint32_t)64U * sizeof (uint8_t)); uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint32_t)); - Hacl_Streaming_SHA2_state_sha2_224 + Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; - Hacl_Streaming_SHA2_state_sha2_224 - *p = - (Hacl_Streaming_SHA2_state_sha2_224 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_224 - )); + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; return p; } @@ -524,21 +517,21 @@ Hacl_Streaming_SHA2_state_sha2_224 /** Reset an existing state to the initial hash state with empty data. */ -void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_SHA2_state_sha2_224 *s) +void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_MD_state_32 *s) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *s; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; sha256_init(block_state); - Hacl_Streaming_SHA2_state_sha2_224 + Hacl_Streaming_MD_state_32 tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; s[0U] = tmp; } static inline uint32_t -update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t len) +update_224_256(Hacl_Streaming_MD_state_32 *p, uint8_t *data, uint32_t len) { - Hacl_Streaming_SHA2_state_sha2_224 s = *p; + Hacl_Streaming_MD_state_32 s = *p; uint64_t total_len = s.total_len; if ((uint64_t)len > (uint64_t)2305843009213693951U - total_len) { @@ -555,7 +548,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le } if (len <= (uint32_t)64U - sz) { - Hacl_Streaming_SHA2_state_sha2_224 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *p; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; @@ -574,7 +567,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_224){ + (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, .total_len = total_len2 @@ -583,7 +576,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le } else if (sz == (uint32_t)0U) { - Hacl_Streaming_SHA2_state_sha2_224 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *p; uint32_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; @@ -620,7 +613,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_224){ + (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, .total_len = total_len1 + (uint64_t)len @@ -632,7 +625,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le uint32_t diff = (uint32_t)64U - sz; uint8_t *data1 = data; uint8_t *data2 = data + diff; - Hacl_Streaming_SHA2_state_sha2_224 s1 = *p; + Hacl_Streaming_MD_state_32 s1 = *p; uint32_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; @@ -651,13 +644,13 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_224){ + (Hacl_Streaming_MD_state_32){ .block_state = block_state10, .buf = buf0, .total_len = total_len2 } ); - Hacl_Streaming_SHA2_state_sha2_224 s10 = *p; + Hacl_Streaming_MD_state_32 s10 = *p; uint32_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; @@ -700,7 +693,7 @@ update_224_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_224){ + (Hacl_Streaming_MD_state_32){ .block_state = block_state1, .buf = buf, .total_len = total_len1 + (uint64_t)(len - diff) @@ -719,7 +712,7 @@ This function is identical to the update function for SHA2_224. */ uint32_t Hacl_Streaming_SHA2_update_256( - Hacl_Streaming_SHA2_state_sha2_224 *p, + Hacl_Streaming_MD_state_32 *p, uint8_t *input, uint32_t input_len ) @@ -733,9 +726,9 @@ valid after a call to `finish_256`, meaning the user may feed more data into the hash via `update_256`. (The finish_256 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *dst) +void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *p; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; @@ -773,9 +766,9 @@ Free a state allocated with `create_in_256`. This function is identical to the free function for SHA2_224. */ -void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_SHA2_state_sha2_224 *s) +void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_MD_state_32 *s) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *s; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); @@ -802,36 +795,33 @@ void Hacl_Streaming_SHA2_sha256(uint8_t *input, uint32_t input_len, uint8_t *dst sha256_finish(st, rb); } -Hacl_Streaming_SHA2_state_sha2_224 *Hacl_Streaming_SHA2_create_in_224(void) +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_224(void) { uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)64U, sizeof (uint8_t)); uint32_t *block_state = (uint32_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint32_t)); - Hacl_Streaming_SHA2_state_sha2_224 + Hacl_Streaming_MD_state_32 s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_SHA2_state_sha2_224 - *p = - (Hacl_Streaming_SHA2_state_sha2_224 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_224 - )); + Hacl_Streaming_MD_state_32 + *p = (Hacl_Streaming_MD_state_32 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_32)); p[0U] = s; sha224_init(block_state); return p; } -void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_SHA2_state_sha2_224 *s) +void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_MD_state_32 *s) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *s; + Hacl_Streaming_MD_state_32 scrut = *s; uint8_t *buf = scrut.buf; uint32_t *block_state = scrut.block_state; sha224_init(block_state); - Hacl_Streaming_SHA2_state_sha2_224 + Hacl_Streaming_MD_state_32 tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; s[0U] = tmp; } uint32_t Hacl_Streaming_SHA2_update_224( - Hacl_Streaming_SHA2_state_sha2_224 *p, + Hacl_Streaming_MD_state_32 *p, uint8_t *input, uint32_t input_len ) @@ -844,9 +834,9 @@ Write the resulting hash into `dst`, an array of 28 bytes. The state remains valid after a call to `finish_224`, meaning the user may feed more data into the hash via `update_224`. */ -void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *dst) +void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_MD_state_32 *p, uint8_t *dst) { - Hacl_Streaming_SHA2_state_sha2_224 scrut = *p; + Hacl_Streaming_MD_state_32 scrut = *p; uint32_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; @@ -879,7 +869,7 @@ void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8 sha224_finish(tmp_block_state, dst); } -void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_SHA2_state_sha2_224 *p) +void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_MD_state_32 *p) { Hacl_Streaming_SHA2_free_256(p); } @@ -903,17 +893,14 @@ void Hacl_Streaming_SHA2_sha224(uint8_t *input, uint32_t input_len, uint8_t *dst sha224_finish(st, rb); } -Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_512(void) +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_512(void) { uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); - Hacl_Streaming_SHA2_state_sha2_384 + Hacl_Streaming_MD_state_64 s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_SHA2_state_sha2_384 - *p = - (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_384 - )); + Hacl_Streaming_MD_state_64 + *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; Hacl_SHA2_Scalar32_sha512_init(block_state); return p; @@ -925,10 +912,9 @@ The state is to be freed by calling `free_512`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_SHA2_state_sha2_384 -*Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_SHA2_state_sha2_384 *s0) +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state_64 *s0) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *s0; + Hacl_Streaming_MD_state_64 scrut = *s0; uint64_t *block_state0 = scrut.block_state; uint8_t *buf0 = scrut.buf; uint64_t total_len0 = scrut.total_len; @@ -936,32 +922,29 @@ Hacl_Streaming_SHA2_state_sha2_384 memcpy(buf, buf0, (uint32_t)128U * sizeof (uint8_t)); uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); memcpy(block_state, block_state0, (uint32_t)8U * sizeof (uint64_t)); - Hacl_Streaming_SHA2_state_sha2_384 + Hacl_Streaming_MD_state_64 s = { .block_state = block_state, .buf = buf, .total_len = total_len0 }; - Hacl_Streaming_SHA2_state_sha2_384 - *p = - (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_384 - )); + Hacl_Streaming_MD_state_64 + *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; return p; } -void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_SHA2_state_sha2_384 *s) +void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_MD_state_64 *s) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *s; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; Hacl_SHA2_Scalar32_sha512_init(block_state); - Hacl_Streaming_SHA2_state_sha2_384 + Hacl_Streaming_MD_state_64 tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; s[0U] = tmp; } static inline uint32_t -update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t len) +update_384_512(Hacl_Streaming_MD_state_64 *p, uint8_t *data, uint32_t len) { - Hacl_Streaming_SHA2_state_sha2_384 s = *p; + Hacl_Streaming_MD_state_64 s = *p; uint64_t total_len = s.total_len; if ((uint64_t)len > (uint64_t)18446744073709551615U - total_len) { @@ -978,7 +961,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le } if (len <= (uint32_t)128U - sz) { - Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *p; uint64_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; @@ -997,7 +980,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_384){ + (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, .total_len = total_len2 @@ -1006,7 +989,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le } else if (sz == (uint32_t)0U) { - Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *p; uint64_t *block_state1 = s1.block_state; uint8_t *buf = s1.buf; uint64_t total_len1 = s1.total_len; @@ -1043,7 +1026,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_384){ + (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, .total_len = total_len1 + (uint64_t)len @@ -1055,7 +1038,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le uint32_t diff = (uint32_t)128U - sz; uint8_t *data1 = data; uint8_t *data2 = data + diff; - Hacl_Streaming_SHA2_state_sha2_384 s1 = *p; + Hacl_Streaming_MD_state_64 s1 = *p; uint64_t *block_state10 = s1.block_state; uint8_t *buf0 = s1.buf; uint64_t total_len10 = s1.total_len; @@ -1074,13 +1057,13 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_384){ + (Hacl_Streaming_MD_state_64){ .block_state = block_state10, .buf = buf0, .total_len = total_len2 } ); - Hacl_Streaming_SHA2_state_sha2_384 s10 = *p; + Hacl_Streaming_MD_state_64 s10 = *p; uint64_t *block_state1 = s10.block_state; uint8_t *buf = s10.buf; uint64_t total_len1 = s10.total_len; @@ -1123,7 +1106,7 @@ update_384_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *data, uint32_t le *p = ( - (Hacl_Streaming_SHA2_state_sha2_384){ + (Hacl_Streaming_MD_state_64){ .block_state = block_state1, .buf = buf, .total_len = total_len1 + (uint64_t)(len - diff) @@ -1142,7 +1125,7 @@ This function is identical to the update function for SHA2_384. */ uint32_t Hacl_Streaming_SHA2_update_512( - Hacl_Streaming_SHA2_state_sha2_384 *p, + Hacl_Streaming_MD_state_64 *p, uint8_t *input, uint32_t input_len ) @@ -1156,9 +1139,9 @@ valid after a call to `finish_512`, meaning the user may feed more data into the hash via `update_512`. (The finish_512 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst) +void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_MD_state_64 *p, uint8_t *dst) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *p; + Hacl_Streaming_MD_state_64 scrut = *p; uint64_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; @@ -1200,9 +1183,9 @@ Free a state allocated with `create_in_512`. This function is identical to the free function for SHA2_384. */ -void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_SHA2_state_sha2_384 *s) +void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_MD_state_64 *s) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *s; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; KRML_HOST_FREE(block_state); @@ -1229,36 +1212,33 @@ void Hacl_Streaming_SHA2_sha512(uint8_t *input, uint32_t input_len, uint8_t *dst sha512_finish(st, rb); } -Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_384(void) +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_384(void) { uint8_t *buf = (uint8_t *)KRML_HOST_CALLOC((uint32_t)128U, sizeof (uint8_t)); uint64_t *block_state = (uint64_t *)KRML_HOST_CALLOC((uint32_t)8U, sizeof (uint64_t)); - Hacl_Streaming_SHA2_state_sha2_384 + Hacl_Streaming_MD_state_64 s = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; - Hacl_Streaming_SHA2_state_sha2_384 - *p = - (Hacl_Streaming_SHA2_state_sha2_384 *)KRML_HOST_MALLOC(sizeof ( - Hacl_Streaming_SHA2_state_sha2_384 - )); + Hacl_Streaming_MD_state_64 + *p = (Hacl_Streaming_MD_state_64 *)KRML_HOST_MALLOC(sizeof (Hacl_Streaming_MD_state_64)); p[0U] = s; sha384_init(block_state); return p; } -void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_SHA2_state_sha2_384 *s) +void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_MD_state_64 *s) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *s; + Hacl_Streaming_MD_state_64 scrut = *s; uint8_t *buf = scrut.buf; uint64_t *block_state = scrut.block_state; sha384_init(block_state); - Hacl_Streaming_SHA2_state_sha2_384 + Hacl_Streaming_MD_state_64 tmp = { .block_state = block_state, .buf = buf, .total_len = (uint64_t)(uint32_t)0U }; s[0U] = tmp; } uint32_t Hacl_Streaming_SHA2_update_384( - Hacl_Streaming_SHA2_state_sha2_384 *p, + Hacl_Streaming_MD_state_64 *p, uint8_t *input, uint32_t input_len ) @@ -1271,9 +1251,9 @@ Write the resulting hash into `dst`, an array of 48 bytes. The state remains valid after a call to `finish_384`, meaning the user may feed more data into the hash via `update_384`. */ -void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst) +void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_MD_state_64 *p, uint8_t *dst) { - Hacl_Streaming_SHA2_state_sha2_384 scrut = *p; + Hacl_Streaming_MD_state_64 scrut = *p; uint64_t *block_state = scrut.block_state; uint8_t *buf_ = scrut.buf; uint64_t total_len = scrut.total_len; @@ -1310,7 +1290,7 @@ void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8 sha384_finish(tmp_block_state, dst); } -void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_SHA2_state_sha2_384 *p) +void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_MD_state_64 *p) { Hacl_Streaming_SHA2_free_512(p); } diff --git a/Modules/_hacl/Hacl_Streaming_SHA2.h b/Modules/_hacl/Hacl_Streaming_SHA2.h index 2c905854f336fd..b58df4c4d121c9 100644 --- a/Modules/_hacl/Hacl_Streaming_SHA2.h +++ b/Modules/_hacl/Hacl_Streaming_SHA2.h @@ -36,33 +36,22 @@ extern "C" { #include "krml/lowstar_endianness.h" #include "krml/internal/target.h" +#include "Hacl_Streaming_Types.h" -typedef struct Hacl_Streaming_SHA2_state_sha2_224_s -{ - uint32_t *block_state; - uint8_t *buf; - uint64_t total_len; -} -Hacl_Streaming_SHA2_state_sha2_224; +typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA2_state_sha2_224; -typedef Hacl_Streaming_SHA2_state_sha2_224 Hacl_Streaming_SHA2_state_sha2_256; +typedef Hacl_Streaming_MD_state_32 Hacl_Streaming_SHA2_state_sha2_256; -typedef struct Hacl_Streaming_SHA2_state_sha2_384_s -{ - uint64_t *block_state; - uint8_t *buf; - uint64_t total_len; -} -Hacl_Streaming_SHA2_state_sha2_384; +typedef Hacl_Streaming_MD_state_64 Hacl_Streaming_SHA2_state_sha2_384; -typedef Hacl_Streaming_SHA2_state_sha2_384 Hacl_Streaming_SHA2_state_sha2_512; +typedef Hacl_Streaming_MD_state_64 Hacl_Streaming_SHA2_state_sha2_512; /** Allocate initial state for the SHA2_256 hash. The state is to be freed by calling `free_256`. */ -Hacl_Streaming_SHA2_state_sha2_224 *Hacl_Streaming_SHA2_create_in_256(void); +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_256(void); /** Copies the state passed as argument into a newly allocated state (deep copy). @@ -70,13 +59,12 @@ The state is to be freed by calling `free_256`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_SHA2_state_sha2_224 -*Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_SHA2_state_sha2_224 *s0); +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_copy_256(Hacl_Streaming_MD_state_32 *s0); /** Reset an existing state to the initial hash state with empty data. */ -void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_SHA2_state_sha2_224 *s); +void Hacl_Streaming_SHA2_init_256(Hacl_Streaming_MD_state_32 *s); /** Feed an arbitrary amount of data into the hash. This function returns 0 for @@ -87,7 +75,7 @@ This function is identical to the update function for SHA2_224. */ uint32_t Hacl_Streaming_SHA2_update_256( - Hacl_Streaming_SHA2_state_sha2_224 *p, + Hacl_Streaming_MD_state_32 *p, uint8_t *input, uint32_t input_len ); @@ -98,27 +86,27 @@ valid after a call to `finish_256`, meaning the user may feed more data into the hash via `update_256`. (The finish_256 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *dst); +void Hacl_Streaming_SHA2_finish_256(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); /** Free a state allocated with `create_in_256`. This function is identical to the free function for SHA2_224. */ -void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_SHA2_state_sha2_224 *s); +void Hacl_Streaming_SHA2_free_256(Hacl_Streaming_MD_state_32 *s); /** Hash `input`, of len `input_len`, into `dst`, an array of 32 bytes. */ void Hacl_Streaming_SHA2_sha256(uint8_t *input, uint32_t input_len, uint8_t *dst); -Hacl_Streaming_SHA2_state_sha2_224 *Hacl_Streaming_SHA2_create_in_224(void); +Hacl_Streaming_MD_state_32 *Hacl_Streaming_SHA2_create_in_224(void); -void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_SHA2_state_sha2_224 *s); +void Hacl_Streaming_SHA2_init_224(Hacl_Streaming_MD_state_32 *s); uint32_t Hacl_Streaming_SHA2_update_224( - Hacl_Streaming_SHA2_state_sha2_224 *p, + Hacl_Streaming_MD_state_32 *p, uint8_t *input, uint32_t input_len ); @@ -128,16 +116,16 @@ Write the resulting hash into `dst`, an array of 28 bytes. The state remains valid after a call to `finish_224`, meaning the user may feed more data into the hash via `update_224`. */ -void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_SHA2_state_sha2_224 *p, uint8_t *dst); +void Hacl_Streaming_SHA2_finish_224(Hacl_Streaming_MD_state_32 *p, uint8_t *dst); -void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_SHA2_state_sha2_224 *p); +void Hacl_Streaming_SHA2_free_224(Hacl_Streaming_MD_state_32 *p); /** Hash `input`, of len `input_len`, into `dst`, an array of 28 bytes. */ void Hacl_Streaming_SHA2_sha224(uint8_t *input, uint32_t input_len, uint8_t *dst); -Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_512(void); +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_512(void); /** Copies the state passed as argument into a newly allocated state (deep copy). @@ -145,10 +133,9 @@ The state is to be freed by calling `free_512`. Cloning the state this way is useful, for instance, if your control-flow diverges and you need to feed more (different) data into the hash in each branch. */ -Hacl_Streaming_SHA2_state_sha2_384 -*Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_SHA2_state_sha2_384 *s0); +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_copy_512(Hacl_Streaming_MD_state_64 *s0); -void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_SHA2_state_sha2_384 *s); +void Hacl_Streaming_SHA2_init_512(Hacl_Streaming_MD_state_64 *s); /** Feed an arbitrary amount of data into the hash. This function returns 0 for @@ -159,7 +146,7 @@ This function is identical to the update function for SHA2_384. */ uint32_t Hacl_Streaming_SHA2_update_512( - Hacl_Streaming_SHA2_state_sha2_384 *p, + Hacl_Streaming_MD_state_64 *p, uint8_t *input, uint32_t input_len ); @@ -170,27 +157,27 @@ valid after a call to `finish_512`, meaning the user may feed more data into the hash via `update_512`. (The finish_512 function operates on an internal copy of the state and therefore does not invalidate the client-held state `p`.) */ -void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst); +void Hacl_Streaming_SHA2_finish_512(Hacl_Streaming_MD_state_64 *p, uint8_t *dst); /** Free a state allocated with `create_in_512`. This function is identical to the free function for SHA2_384. */ -void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_SHA2_state_sha2_384 *s); +void Hacl_Streaming_SHA2_free_512(Hacl_Streaming_MD_state_64 *s); /** Hash `input`, of len `input_len`, into `dst`, an array of 64 bytes. */ void Hacl_Streaming_SHA2_sha512(uint8_t *input, uint32_t input_len, uint8_t *dst); -Hacl_Streaming_SHA2_state_sha2_384 *Hacl_Streaming_SHA2_create_in_384(void); +Hacl_Streaming_MD_state_64 *Hacl_Streaming_SHA2_create_in_384(void); -void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_SHA2_state_sha2_384 *s); +void Hacl_Streaming_SHA2_init_384(Hacl_Streaming_MD_state_64 *s); uint32_t Hacl_Streaming_SHA2_update_384( - Hacl_Streaming_SHA2_state_sha2_384 *p, + Hacl_Streaming_MD_state_64 *p, uint8_t *input, uint32_t input_len ); @@ -200,9 +187,9 @@ Write the resulting hash into `dst`, an array of 48 bytes. The state remains valid after a call to `finish_384`, meaning the user may feed more data into the hash via `update_384`. */ -void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_SHA2_state_sha2_384 *p, uint8_t *dst); +void Hacl_Streaming_SHA2_finish_384(Hacl_Streaming_MD_state_64 *p, uint8_t *dst); -void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_SHA2_state_sha2_384 *p); +void Hacl_Streaming_SHA2_free_384(Hacl_Streaming_MD_state_64 *p); /** Hash `input`, of len `input_len`, into `dst`, an array of 48 bytes. diff --git a/Modules/_hacl/Hacl_Streaming_Types.h b/Modules/_hacl/Hacl_Streaming_Types.h new file mode 100644 index 00000000000000..51057611ca978d --- /dev/null +++ b/Modules/_hacl/Hacl_Streaming_Types.h @@ -0,0 +1,59 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __Hacl_Streaming_Types_H +#define __Hacl_Streaming_Types_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +typedef struct Hacl_Streaming_MD_state_32_s +{ + uint32_t *block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Streaming_MD_state_32; + +typedef struct Hacl_Streaming_MD_state_64_s +{ + uint64_t *block_state; + uint8_t *buf; + uint64_t total_len; +} +Hacl_Streaming_MD_state_64; + +#if defined(__cplusplus) +} +#endif + +#define __Hacl_Streaming_Types_H_DEFINED +#endif diff --git a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h index ee160193539e28..3d36d440735530 100644 --- a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h +++ b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h @@ -7,13 +7,12 @@ #ifndef __FStar_UInt128_Verified_H #define __FStar_UInt128_Verified_H - - #include "FStar_UInt_8_16_32_64.h" #include #include #include "krml/types.h" #include "krml/internal/target.h" + static inline uint64_t FStar_UInt128_constant_time_carry(uint64_t a, uint64_t b) { return (a ^ ((a ^ b) | ((a - b) ^ b))) >> (uint32_t)63U; diff --git a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h index 965afc836fd12b..a56c7d613498b7 100644 --- a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h +++ b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h @@ -7,15 +7,13 @@ #ifndef __FStar_UInt_8_16_32_64_H #define __FStar_UInt_8_16_32_64_H - - - #include #include #include "krml/lowstar_endianness.h" #include "krml/types.h" #include "krml/internal/target.h" + static inline uint64_t FStar_UInt64_eq_mask(uint64_t a, uint64_t b) { uint64_t x = a ^ b; diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index 9ef59859a554b5..dcbe7007b17be8 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -31,6 +31,10 @@ # define KRML_HOST_FREE free #endif +#ifndef KRML_HOST_IGNORE +# define KRML_HOST_IGNORE(x) (void)(x) +#endif + /* Macros for prettier unrolling of loops */ #define KRML_LOOP1(i, n, x) { \ x \ diff --git a/Modules/_hacl/internal/Hacl_Hash_MD5.h b/Modules/_hacl/internal/Hacl_Hash_MD5.h new file mode 100644 index 00000000000000..87ad4cf228d91b --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_MD5.h @@ -0,0 +1,61 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_MD5_H +#define __internal_Hacl_Hash_MD5_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "../Hacl_Hash_MD5.h" + +void Hacl_Hash_Core_MD5_legacy_init(uint32_t *s); + +void Hacl_Hash_Core_MD5_legacy_finish(uint32_t *s, uint8_t *dst); + +void Hacl_Hash_MD5_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); + +void +Hacl_Hash_MD5_legacy_update_last( + uint32_t *s, + uint64_t prev_len, + uint8_t *input, + uint32_t input_len +); + +void Hacl_Hash_MD5_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_MD5_H_DEFINED +#endif diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA1.h b/Modules/_hacl/internal/Hacl_Hash_SHA1.h new file mode 100644 index 00000000000000..d2d9df44c6c14c --- /dev/null +++ b/Modules/_hacl/internal/Hacl_Hash_SHA1.h @@ -0,0 +1,61 @@ +/* MIT License + * + * Copyright (c) 2016-2022 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2022-2023 HACL* Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + +#ifndef __internal_Hacl_Hash_SHA1_H +#define __internal_Hacl_Hash_SHA1_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "krml/types.h" +#include "krml/lowstar_endianness.h" +#include "krml/internal/target.h" + +#include "../Hacl_Hash_SHA1.h" + +void Hacl_Hash_Core_SHA1_legacy_init(uint32_t *s); + +void Hacl_Hash_Core_SHA1_legacy_finish(uint32_t *s, uint8_t *dst); + +void Hacl_Hash_SHA1_legacy_update_multi(uint32_t *s, uint8_t *blocks, uint32_t n_blocks); + +void +Hacl_Hash_SHA1_legacy_update_last( + uint32_t *s, + uint64_t prev_len, + uint8_t *input, + uint32_t input_len +); + +void Hacl_Hash_SHA1_legacy_hash(uint8_t *input, uint32_t input_len, uint8_t *dst); + +#if defined(__cplusplus) +} +#endif + +#define __internal_Hacl_Hash_SHA1_H_DEFINED +#endif diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index ac12f386257b19..ee28f244266b85 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -43,4 +43,21 @@ #define Hacl_Streaming_SHA2_sha512 python_hashlib_Hacl_Streaming_SHA2_sha512 #define Hacl_Streaming_SHA2_sha384 python_hashlib_Hacl_Streaming_SHA2_sha384 +#define Hacl_Streaming_MD5_legacy_create_in python_hashlib_Hacl_Streaming_MD5_legacy_create_in +#define Hacl_Streaming_MD5_legacy_init python_hashlib_Hacl_Streaming_MD5_legacy_init +#define Hacl_Streaming_MD5_legacy_update python_hashlib_Hacl_Streaming_MD5_legacy_update +#define Hacl_Streaming_MD5_legacy_finish python_hashlib_Hacl_Streaming_MD5_legacy_finish +#define Hacl_Streaming_MD5_legacy_free python_hashlib_Hacl_Streaming_MD5_legacy_free +#define Hacl_Streaming_MD5_legacy_copy python_hashlib_Hacl_Streaming_MD5_legacy_copy +#define Hacl_Streaming_MD5_legacy_hash python_hashlib_Hacl_Streaming_MD5_legacy_hash + +#define Hacl_Streaming_SHA1_legacy_create_in python_hashlib_Hacl_Streaming_SHA1_legacy_create_in +#define Hacl_Streaming_SHA1_legacy_init python_hashlib_Hacl_Streaming_SHA1_legacy_init +#define Hacl_Streaming_SHA1_legacy_update python_hashlib_Hacl_Streaming_SHA1_legacy_update +#define Hacl_Streaming_SHA1_legacy_finish python_hashlib_Hacl_Streaming_SHA1_legacy_finish +#define Hacl_Streaming_SHA1_legacy_free python_hashlib_Hacl_Streaming_SHA1_legacy_free +#define Hacl_Streaming_SHA1_legacy_copy python_hashlib_Hacl_Streaming_SHA1_legacy_copy +#define Hacl_Streaming_SHA1_legacy_hash python_hashlib_Hacl_Streaming_SHA1_legacy_hash + + #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index dba8cb3972ea17..76b92ec4599102 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=4751fc2b11639f651718abf8522fcc36902ca67c +expected_hacl_star_rev=13e0c6721ac9206c4249ecc1dc04ed617ad1e262 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" @@ -41,8 +41,15 @@ fi declare -a dist_files dist_files=( Hacl_Streaming_SHA2.h + Hacl_Streaming_Types.h + Hacl_Hash_SHA1.h + internal/Hacl_Hash_SHA1.h + Hacl_Hash_MD5.h + internal/Hacl_Hash_MD5.h internal/Hacl_SHA2_Generic.h Hacl_Streaming_SHA2.c + Hacl_Hash_SHA1.c + Hacl_Hash_MD5.c ) declare -a include_files diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 55b6535eb34b66..d8d836b8382eb1 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -437,10 +437,9 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, error: if (result != NULL) { - PyObject *exc, *val, *tb, *close_result; - PyErr_Fetch(&exc, &val, &tb); - close_result = PyObject_CallMethodNoArgs(result, &_Py_ID(close)); - _PyErr_ChainExceptions(exc, val, tb); + PyObject *exc = PyErr_GetRaisedException(); + PyObject *close_result = PyObject_CallMethodNoArgs(result, &_Py_ID(close)); + _PyErr_ChainExceptions1(exc); Py_XDECREF(close_result); Py_DECREF(result); } diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 56491f097100c0..960026707fc5ed 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -472,12 +472,13 @@ buffered_closed_get(buffered *self, void *context) static PyObject * buffered_close(buffered *self, PyObject *args) { - PyObject *res = NULL, *exc = NULL, *val, *tb; + PyObject *res = NULL; int r; CHECK_INITIALIZED(self) - if (!ENTER_BUFFERED(self)) + if (!ENTER_BUFFERED(self)) { return NULL; + } r = buffered_closed(self); if (r < 0) @@ -497,12 +498,16 @@ buffered_close(buffered *self, PyObject *args) /* flush() will most probably re-take the lock, so drop it first */ LEAVE_BUFFERED(self) res = PyObject_CallMethodNoArgs((PyObject *)self, &_Py_ID(flush)); - if (!ENTER_BUFFERED(self)) + if (!ENTER_BUFFERED(self)) { return NULL; - if (res == NULL) - PyErr_Fetch(&exc, &val, &tb); - else + } + PyObject *exc = NULL; + if (res == NULL) { + exc = PyErr_GetRaisedException(); + } + else { Py_DECREF(res); + } res = PyObject_CallMethodNoArgs(self->raw, &_Py_ID(close)); @@ -512,7 +517,7 @@ buffered_close(buffered *self, PyObject *args) } if (exc != NULL) { - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); Py_CLEAR(res); } @@ -637,17 +642,14 @@ _set_BlockingIOError(const char *msg, Py_ssize_t written) static Py_ssize_t * _buffered_check_blocking_error(void) { - PyObject *t, *v, *tb; - PyOSErrorObject *err; - - PyErr_Fetch(&t, &v, &tb); - if (v == NULL || !PyErr_GivenExceptionMatches(v, PyExc_BlockingIOError)) { - PyErr_Restore(t, v, tb); + PyObject *exc = PyErr_GetRaisedException(); + if (exc == NULL || !PyErr_GivenExceptionMatches(exc, PyExc_BlockingIOError)) { + PyErr_SetRaisedException(exc); return NULL; } - err = (PyOSErrorObject *) v; + PyOSErrorObject *err = (PyOSErrorObject *)exc; /* TODO: sanity check (err->written >= 0) */ - PyErr_Restore(t, v, tb); + PyErr_SetRaisedException(exc); return &err->written; } @@ -749,13 +751,11 @@ _buffered_init(buffered *self) int _PyIO_trap_eintr(void) { - PyObject *typ, *val, *tb; - PyOSErrorObject *env_err; - if (!PyErr_ExceptionMatches(PyExc_OSError)) + if (!PyErr_ExceptionMatches(PyExc_OSError)) { return 0; - PyErr_Fetch(&typ, &val, &tb); - PyErr_NormalizeException(&typ, &val, &tb); - env_err = (PyOSErrorObject *) val; + } + PyObject *exc = PyErr_GetRaisedException(); + PyOSErrorObject *env_err = (PyOSErrorObject *)exc; assert(env_err != NULL); if (env_err->myerrno != NULL) { assert(EINTR > 0 && EINTR < INT_MAX); @@ -764,14 +764,12 @@ _PyIO_trap_eintr(void) int myerrno = PyLong_AsLongAndOverflow(env_err->myerrno, &overflow); PyErr_Clear(); if (myerrno == EINTR) { - Py_DECREF(typ); - Py_DECREF(val); - Py_XDECREF(tb); + Py_DECREF(exc); return 1; } } /* This silences any error set by PyObject_RichCompareBool() */ - PyErr_Restore(typ, val, tb); + PyErr_SetRaisedException(exc); return 0; } @@ -2228,15 +2226,17 @@ bufferedrwpair_writable(rwpair *self, PyObject *Py_UNUSED(ignored)) static PyObject * bufferedrwpair_close(rwpair *self, PyObject *Py_UNUSED(ignored)) { - PyObject *exc = NULL, *val, *tb; + PyObject *exc = NULL; PyObject *ret = _forward_call(self->writer, &_Py_ID(close), NULL); - if (ret == NULL) - PyErr_Fetch(&exc, &val, &tb); - else + if (ret == NULL) { + exc = PyErr_GetRaisedException(); + } + else { Py_DECREF(ret); + } ret = _forward_call(self->reader, &_Py_ID(close), NULL); if (exc != NULL) { - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); Py_CLEAR(ret); } return ret; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index f424fb8439d7a8..35a498ce5a8354 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -88,14 +88,13 @@ static PyObject * fileio_dealloc_warn(fileio *self, PyObject *source) { if (self->fd >= 0 && self->closefd) { - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) self); } - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); } Py_RETURN_NONE; } @@ -140,7 +139,7 @@ _io_FileIO_close_impl(fileio *self) /*[clinic end generated code: output=7737a319ef3bad0b input=f35231760d54a522]*/ { PyObject *res; - PyObject *exc, *val, *tb; + PyObject *exc; int rc; res = PyObject_CallMethodOneArg((PyObject*)&PyRawIOBase_Type, &_Py_ID(close), (PyObject *)self); @@ -148,20 +147,25 @@ _io_FileIO_close_impl(fileio *self) self->fd = -1; return res; } - if (res == NULL) - PyErr_Fetch(&exc, &val, &tb); + if (res == NULL) { + exc = PyErr_GetRaisedException(); + } if (self->finalizing) { PyObject *r = fileio_dealloc_warn(self, (PyObject *) self); - if (r) + if (r) { Py_DECREF(r); - else + } + else { PyErr_Clear(); + } } rc = internal_close(self); - if (res == NULL) - _PyErr_ChainExceptions(exc, val, tb); - if (rc < 0) + if (res == NULL) { + _PyErr_ChainExceptions1(exc); + } + if (rc < 0) { Py_CLEAR(res); + } return res; } @@ -487,10 +491,9 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, if (!fd_is_own) self->fd = -1; if (self->fd >= 0) { - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); internal_close(self); - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); } done: diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 7b9391ec54d732..682ed000eb1fd9 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -220,7 +220,6 @@ static PyObject * _io__IOBase_close_impl(PyObject *self) /*[clinic end generated code: output=63c6a6f57d783d6d input=f4494d5c31dbc6b7]*/ { - PyObject *res, *exc, *val, *tb; int rc, closed = iobase_is_closed(self); if (closed < 0) { @@ -230,11 +229,11 @@ _io__IOBase_close_impl(PyObject *self) Py_RETURN_NONE; } - res = PyObject_CallMethodNoArgs(self, &_Py_ID(flush)); + PyObject *res = PyObject_CallMethodNoArgs(self, &_Py_ID(flush)); - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); rc = PyObject_SetAttr(self, &_Py_ID(__IOBase_closed), Py_True); - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); if (rc < 0) { Py_CLEAR(res); } @@ -252,11 +251,10 @@ static void iobase_finalize(PyObject *self) { PyObject *res; - PyObject *error_type, *error_value, *error_traceback; int closed; /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); /* If `closed` doesn't exist or can't be evaluated as bool, then the object is probably in an unusable state, so ignore. */ @@ -297,7 +295,7 @@ iobase_finalize(PyObject *self) } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } int diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index fbf0bf46840374..3ff84cb623af74 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2827,11 +2827,10 @@ _io_TextIOWrapper_tell_impl(textio *self) fail: if (saved_state) { - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); + PyObject *exc = PyErr_GetRaisedException(); res = PyObject_CallMethodOneArg( self->decoder, &_Py_ID(setstate), saved_state); - _PyErr_ChainExceptions(type, value, traceback); + _PyErr_ChainExceptions1(exc); Py_DECREF(saved_state); Py_XDECREF(res); } @@ -3028,24 +3027,28 @@ _io_TextIOWrapper_close_impl(textio *self) Py_RETURN_NONE; /* stream already closed */ } else { - PyObject *exc = NULL, *val, *tb; + PyObject *exc = NULL; if (self->finalizing) { res = PyObject_CallMethodOneArg(self->buffer, &_Py_ID(_dealloc_warn), (PyObject *)self); - if (res) + if (res) { Py_DECREF(res); - else + } + else { PyErr_Clear(); + } } res = PyObject_CallMethodNoArgs((PyObject *)self, &_Py_ID(flush)); - if (res == NULL) - PyErr_Fetch(&exc, &val, &tb); - else + if (res == NULL) { + exc = PyErr_GetRaisedException(); + } + else { Py_DECREF(res); + } res = PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(close)); if (exc != NULL) { - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); Py_CLEAR(res); } return res; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index e913d831874617..de07b50f5ce4cb 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -192,7 +192,7 @@ _io__WindowsConsoleIO_close_impl(winconsoleio *self) /*[clinic end generated code: output=27ef95b66c29057b input=68c4e5754f8136c2]*/ { PyObject *res; - PyObject *exc, *val, *tb; + PyObject *exc; int rc; res = PyObject_CallMethodOneArg((PyObject*)&PyRawIOBase_Type, &_Py_ID(close), (PyObject*)self); @@ -200,13 +200,16 @@ _io__WindowsConsoleIO_close_impl(winconsoleio *self) self->fd = -1; return res; } - if (res == NULL) - PyErr_Fetch(&exc, &val, &tb); + if (res == NULL) { + exc = PyErr_GetRaisedException(); + } rc = internal_close(self); - if (res == NULL) - _PyErr_ChainExceptions(exc, val, tb); - if (rc < 0) + if (res == NULL) { + _PyErr_ChainExceptions1(exc); + } + if (rc < 0) { Py_CLEAR(res); + } return res; } diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 37170bbea56ad3..83d034ae7eed78 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -348,8 +348,7 @@ ptrace_enter_call(PyObject *self, void *key, PyObject *userObj) * exception, and some of the code under here assumes that * PyErr_* is its own to mess around with, so we have to * save and restore any current exception. */ - PyObject *last_type, *last_value, *last_tb; - PyErr_Fetch(&last_type, &last_value, &last_tb); + PyObject *exc = PyErr_GetRaisedException(); profEntry = getEntry(pObj, key); if (profEntry == NULL) { @@ -374,7 +373,7 @@ ptrace_enter_call(PyObject *self, void *key, PyObject *userObj) initContext(pObj, pContext, profEntry); restorePyerr: - PyErr_Restore(last_type, last_value, last_tb); + PyErr_SetRaisedException(exc); } static void @@ -607,7 +606,7 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls) /*[clinic end generated code: output=1806ef720019ee03 input=445e193ef4522902]*/ { statscollector_t collect; - collect.state = PyType_GetModuleState(cls); + collect.state = _PyType_GetModuleState(cls); if (pending_exception(self)) { return NULL; } diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index b572d8cd909fd1..bccab8639159e7 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -734,7 +734,8 @@ Compressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspecs) } /*[-clinic input] -_lzma.LZMACompressor.__init__ +@classmethod +_lzma.LZMACompressor.__new__ format: int(c_default="FORMAT_XZ") = FORMAT_XZ The container format to use for the output. This can @@ -765,8 +766,8 @@ the raw compressor does not support preset compression levels. For one-shot compression, use the compress() function instead. [-clinic start generated code]*/ -static int -Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) +static PyObject * +Compressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { static char *arg_names[] = {"format", "check", "preset", "filters", NULL}; int format = FORMAT_XZ; @@ -774,31 +775,37 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) uint32_t preset = LZMA_PRESET_DEFAULT; PyObject *preset_obj = Py_None; PyObject *filterspecs = Py_None; - _lzma_state *state = PyType_GetModuleState(Py_TYPE(self)); + Compressor *self; + + _lzma_state *state = PyType_GetModuleState(type); assert(state != NULL); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iiOO:LZMACompressor", arg_names, &format, &check, &preset_obj, &filterspecs)) { - return -1; + return NULL; } if (format != FORMAT_XZ && check != -1 && check != LZMA_CHECK_NONE) { PyErr_SetString(PyExc_ValueError, "Integrity checks are only supported by FORMAT_XZ"); - return -1; + return NULL; } if (preset_obj != Py_None && filterspecs != Py_None) { PyErr_SetString(PyExc_ValueError, "Cannot specify both preset and filter chain"); - return -1; + return NULL; } - if (preset_obj != Py_None) { - if (!uint32_converter(preset_obj, &preset)) { - return -1; - } + if (preset_obj != Py_None && !uint32_converter(preset_obj, &preset)) { + return NULL; + } + + assert(type != NULL && type->tp_alloc != NULL); + self = (Compressor *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; } self->alloc.opaque = NULL; @@ -808,8 +815,9 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) self->lock = PyThread_allocate_lock(); if (self->lock == NULL) { + Py_DECREF(self); PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return -1; + return NULL; } self->flushed = 0; @@ -819,31 +827,33 @@ Compressor_init(Compressor *self, PyObject *args, PyObject *kwargs) check = LZMA_CHECK_CRC64; } if (Compressor_init_xz(state, &self->lzs, check, preset, filterspecs) != 0) { - break; + goto error; } - return 0; + break; case FORMAT_ALONE: if (Compressor_init_alone(state, &self->lzs, preset, filterspecs) != 0) { - break; + goto error; } - return 0; + break; case FORMAT_RAW: if (Compressor_init_raw(state, &self->lzs, filterspecs) != 0) { - break; + goto error; } - return 0; + break; default: PyErr_Format(PyExc_ValueError, "Invalid container format: %d", format); - break; + goto error; } - PyThread_free_lock(self->lock); - self->lock = NULL; - return -1; + return (PyObject *)self; + +error: + Py_DECREF(self); + return NULL; } static void @@ -902,8 +912,7 @@ PyDoc_STRVAR(Compressor_doc, static PyType_Slot lzma_compressor_type_slots[] = { {Py_tp_dealloc, Compressor_dealloc}, {Py_tp_methods, Compressor_methods}, - {Py_tp_init, Compressor_init}, - {Py_tp_new, PyType_GenericNew}, + {Py_tp_new, Compressor_new}, {Py_tp_doc, (char *)Compressor_doc}, {Py_tp_traverse, Compressor_traverse}, {0, 0} @@ -1165,7 +1174,8 @@ Decompressor_init_raw(_lzma_state *state, lzma_stream *lzs, PyObject *filterspec } /*[clinic input] -_lzma.LZMADecompressor.__init__ +@classmethod +_lzma.LZMADecompressor.__new__ format: int(c_default="FORMAT_AUTO") = FORMAT_AUTO Specifies the container format of the input stream. If this is @@ -1189,54 +1199,57 @@ Create a decompressor object for decompressing data incrementally. For one-shot decompression, use the decompress() function instead. [clinic start generated code]*/ -static int -_lzma_LZMADecompressor___init___impl(Decompressor *self, int format, - PyObject *memlimit, PyObject *filters) -/*[clinic end generated code: output=3e1821f8aa36564c input=81fe684a6c2f8a27]*/ +static PyObject * +_lzma_LZMADecompressor_impl(PyTypeObject *type, int format, + PyObject *memlimit, PyObject *filters) +/*[clinic end generated code: output=2d46d5e70f10bc7f input=ca40cd1cb1202b0d]*/ { + Decompressor *self; const uint32_t decoder_flags = LZMA_TELL_ANY_CHECK | LZMA_TELL_NO_CHECK; uint64_t memlimit_ = UINT64_MAX; lzma_ret lzret; - _lzma_state *state = PyType_GetModuleState(Py_TYPE(self)); + _lzma_state *state = PyType_GetModuleState(type); assert(state != NULL); if (memlimit != Py_None) { if (format == FORMAT_RAW) { PyErr_SetString(PyExc_ValueError, "Cannot specify memory limit with FORMAT_RAW"); - return -1; + return NULL; } memlimit_ = PyLong_AsUnsignedLongLong(memlimit); if (PyErr_Occurred()) { - return -1; + return NULL; } } if (format == FORMAT_RAW && filters == Py_None) { PyErr_SetString(PyExc_ValueError, "Must specify filters for FORMAT_RAW"); - return -1; + return NULL; } else if (format != FORMAT_RAW && filters != Py_None) { PyErr_SetString(PyExc_ValueError, "Cannot specify filters except with FORMAT_RAW"); - return -1; + return NULL; } + assert(type != NULL && type->tp_alloc != NULL); + self = (Decompressor *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } self->alloc.opaque = NULL; self->alloc.alloc = PyLzma_Malloc; self->alloc.free = PyLzma_Free; self->lzs.allocator = &self->alloc; self->lzs.next_in = NULL; - PyThread_type_lock lock = PyThread_allocate_lock(); - if (lock == NULL) { + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + Py_DECREF(self); PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return -1; - } - if (self->lock != NULL) { - PyThread_free_lock(self->lock); + return NULL; } - self->lock = lock; self->check = LZMA_CHECK_UNKNOWN; self->needs_input = 1; @@ -1251,43 +1264,43 @@ _lzma_LZMADecompressor___init___impl(Decompressor *self, int format, case FORMAT_AUTO: lzret = lzma_auto_decoder(&self->lzs, memlimit_, decoder_flags); if (catch_lzma_error(state, lzret)) { - break; + goto error; } - return 0; + break; case FORMAT_XZ: lzret = lzma_stream_decoder(&self->lzs, memlimit_, decoder_flags); if (catch_lzma_error(state, lzret)) { - break; + goto error; } - return 0; + break; case FORMAT_ALONE: self->check = LZMA_CHECK_NONE; lzret = lzma_alone_decoder(&self->lzs, memlimit_); if (catch_lzma_error(state, lzret)) { - break; + goto error; } - return 0; + break; case FORMAT_RAW: self->check = LZMA_CHECK_NONE; if (Decompressor_init_raw(state, &self->lzs, filters) == -1) { - break; + goto error; } - return 0; + break; default: PyErr_Format(PyExc_ValueError, "Invalid container format: %d", format); - break; + goto error; } + return (PyObject *)self; + error: - Py_CLEAR(self->unused_data); - PyThread_free_lock(self->lock); - self->lock = NULL; - return -1; + Py_DECREF(self); + return NULL; } static void @@ -1345,9 +1358,8 @@ static PyMemberDef Decompressor_members[] = { static PyType_Slot lzma_decompressor_type_slots[] = { {Py_tp_dealloc, Decompressor_dealloc}, {Py_tp_methods, Decompressor_methods}, - {Py_tp_init, _lzma_LZMADecompressor___init__}, - {Py_tp_new, PyType_GenericNew}, - {Py_tp_doc, (char *)_lzma_LZMADecompressor___init____doc__}, + {Py_tp_new, _lzma_LZMADecompressor}, + {Py_tp_doc, (char *)_lzma_LZMADecompressor__doc__}, {Py_tp_traverse, Decompressor_traverse}, {Py_tp_members, Decompressor_members}, {0, 0} diff --git a/Modules/_operator.c b/Modules/_operator.c index 4f2367150eefc4..38335b6995016c 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1002,7 +1002,7 @@ itemgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } else { item = args; } - _operator_state *state = PyType_GetModuleState(type); + _operator_state *state = _PyType_GetModuleState(type); /* create itemgetterobject structure */ ig = PyObject_GC_New(itemgetterobject, (PyTypeObject *) state->itemgetter_type); if (ig == NULL) { @@ -1298,7 +1298,7 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } } - _operator_state *state = PyType_GetModuleState(type); + _operator_state *state = _PyType_GetModuleState(type); /* create attrgetterobject structure */ ag = PyObject_GC_New(attrgetterobject, (PyTypeObject *)state->attrgetter_type); if (ag == NULL) { @@ -1578,7 +1578,7 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } - _operator_state *state = PyType_GetModuleState(type); + _operator_state *state = _PyType_GetModuleState(type); /* create methodcallerobject structure */ mc = PyObject_GC_New(methodcallerobject, (PyTypeObject *)state->methodcaller_type); if (mc == NULL) { diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 95f1e505dd1873..68060c07033d34 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -259,7 +259,9 @@ random_seed_time_pid(RandomObject *self) key[0] = (uint32_t)(now & 0xffffffffU); key[1] = (uint32_t)(now >> 32); -#ifdef HAVE_GETPID +#ifdef MS_WINDOWS_NON_DESKTOP + key[2] = (uint32_t)GetCurrentProcessId(); +#elif defined(HAVE_GETPID) key[2] = (uint32_t)getpid(); #else key[2] = 0; diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c index bd1dd596bdda68..633a0c0ea08d2a 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -21,6 +21,7 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "../hashlib.h" #include "sha3.c" @@ -106,7 +107,7 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) { HashReturn res; Py_buffer buf = {NULL, NULL}; - SHA3State *state = PyType_GetModuleState(type); + SHA3State *state = _PyType_GetModuleState(type); SHA3object *self = newSHA3object(type); if (self == NULL) { goto error; @@ -337,7 +338,7 @@ SHA3_get_name(SHA3object *self, void *closure) { PyTypeObject *type = Py_TYPE(self); - SHA3State *state = PyType_GetModuleState(type); + SHA3State *state = _PyType_GetModuleState(type); assert(state != NULL); if (type == state->sha3_224_type) { @@ -408,7 +409,7 @@ static PyGetSetDef SHA3_getseters[] = { {0,0} \ } -// Using PyType_GetModuleState() on these types is safe since they +// Using _PyType_GetModuleState() on these types is safe since they // cannot be subclassed: it does not have the Py_TPFLAGS_BASETYPE flag. #define SHA3_TYPE_SPEC(type_spec_obj, type_name, type_slots) \ static PyType_Spec type_spec_obj = { \ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 4c07d5e0b61f8c..fb61ef82ef869b 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -908,7 +908,6 @@ final_callback(sqlite3_context *context) PyObject* function_result; PyObject** aggregate_instance; int ok; - PyObject *exception, *value, *tb; aggregate_instance = (PyObject**)sqlite3_aggregate_context(context, 0); if (aggregate_instance == NULL) { @@ -923,7 +922,7 @@ final_callback(sqlite3_context *context) } // Keep the exception (if any) of the last call to step, value, or inverse - PyErr_Fetch(&exception, &value, &tb); + PyObject *exc = PyErr_GetRaisedException(); callback_context *ctx = (callback_context *)sqlite3_user_data(context); assert(ctx != NULL); @@ -938,7 +937,7 @@ final_callback(sqlite3_context *context) } if (!ok) { int attr_err = PyErr_ExceptionMatches(PyExc_AttributeError); - _PyErr_ChainExceptions(exception, value, tb); + _PyErr_ChainExceptions1(exc); /* Note: contrary to the step, value, and inverse callbacks, SQLite * does _not_, as of SQLite 3.38.0, propagate errors to sqlite3_step() @@ -949,7 +948,7 @@ final_callback(sqlite3_context *context) : "user-defined aggregate's 'finalize' method raised error"); } else { - PyErr_Restore(exception, value, tb); + PyErr_SetRaisedException(exc); } error: @@ -2274,15 +2273,14 @@ pysqlite_connection_exit_impl(pysqlite_Connection *self, PyObject *exc_type, if (commit) { /* Commit failed; try to rollback in order to unlock the database. * If rollback also fails, chain the exceptions. */ - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); result = pysqlite_connection_rollback_impl(self); if (result == NULL) { - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); } else { Py_DECREF(result); - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); } } return NULL; diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 6f7970cf8197a2..caeedbddb8d88b 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -705,11 +705,10 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, Py_DECREF(adapted); if (rc != SQLITE_OK) { - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); sqlite3 *db = sqlite3_db_handle(self->st); _pysqlite_seterror(state, db); - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); return; } } @@ -765,11 +764,10 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, Py_DECREF(adapted); if (rc != SQLITE_OK) { - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); sqlite3 *db = sqlite3_db_handle(self->st); _pysqlite_seterror(state, db); - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); return; } } diff --git a/Modules/_testcapi/code.c b/Modules/_testcapi/code.c new file mode 100644 index 00000000000000..588dc67971ea5a --- /dev/null +++ b/Modules/_testcapi/code.c @@ -0,0 +1,115 @@ +#include "parts.h" + +static Py_ssize_t +get_code_extra_index(PyInterpreterState* interp) { + Py_ssize_t result = -1; + + static const char *key = "_testcapi.frame_evaluation.code_index"; + + PyObject *interp_dict = PyInterpreterState_GetDict(interp); // borrowed + assert(interp_dict); // real users would handle missing dict... somehow + + PyObject *index_obj = PyDict_GetItemString(interp_dict, key); // borrowed + Py_ssize_t index = 0; + if (!index_obj) { + if (PyErr_Occurred()) { + goto finally; + } + index = PyUnstable_Eval_RequestCodeExtraIndex(NULL); + if (index < 0 || PyErr_Occurred()) { + goto finally; + } + index_obj = PyLong_FromSsize_t(index); // strong ref + if (!index_obj) { + goto finally; + } + int res = PyDict_SetItemString(interp_dict, key, index_obj); + Py_DECREF(index_obj); + if (res < 0) { + goto finally; + } + } + else { + index = PyLong_AsSsize_t(index_obj); + if (index == -1 && PyErr_Occurred()) { + goto finally; + } + } + + result = index; +finally: + return result; +} + +static PyObject * +test_code_extra(PyObject* self, PyObject *Py_UNUSED(callable)) +{ + PyObject *result = NULL; + PyObject *test_module = NULL; + PyObject *test_func = NULL; + + // Get or initialize interpreter-specific code object storage index + PyInterpreterState *interp = PyInterpreterState_Get(); + if (!interp) { + return NULL; + } + Py_ssize_t code_extra_index = get_code_extra_index(interp); + if (PyErr_Occurred()) { + goto finally; + } + + // Get a function to test with + // This can be any Python function. Use `test.test_misc.testfunction`. + test_module = PyImport_ImportModule("test.test_capi.test_misc"); + if (!test_module) { + goto finally; + } + test_func = PyObject_GetAttrString(test_module, "testfunction"); + if (!test_func) { + goto finally; + } + PyObject *test_func_code = PyFunction_GetCode(test_func); // borrowed + if (!test_func_code) { + goto finally; + } + + // Check the value is initially NULL + void *extra; + int res = PyUnstable_Code_GetExtra(test_func_code, code_extra_index, &extra); + if (res < 0) { + goto finally; + } + assert (extra == NULL); + + // Set another code extra value + res = PyUnstable_Code_SetExtra(test_func_code, code_extra_index, (void*)(uintptr_t)77); + if (res < 0) { + goto finally; + } + // Assert it was set correctly + res = PyUnstable_Code_GetExtra(test_func_code, code_extra_index, &extra); + if (res < 0) { + goto finally; + } + assert ((uintptr_t)extra == 77); + + result = Py_NewRef(Py_None); +finally: + Py_XDECREF(test_module); + Py_XDECREF(test_func); + return result; +} + +static PyMethodDef TestMethods[] = { + {"test_code_extra", test_code_extra, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Code(PyObject *m) { + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c new file mode 100644 index 00000000000000..43b88ccf261d98 --- /dev/null +++ b/Modules/_testcapi/exceptions.c @@ -0,0 +1,277 @@ +#include "parts.h" + +static PyObject * +err_set_raised(PyObject *self, PyObject *exc) +{ + Py_INCREF(exc); + PyErr_SetRaisedException(exc); + assert(PyErr_Occurred()); + return NULL; +} + +static PyObject * +err_restore(PyObject *self, PyObject *args) { + PyObject *type = NULL, *value = NULL, *traceback = NULL; + switch(PyTuple_Size(args)) { + case 3: + traceback = PyTuple_GetItem(args, 2); + Py_INCREF(traceback); + /* fall through */ + case 2: + value = PyTuple_GetItem(args, 1); + Py_INCREF(value); + /* fall through */ + case 1: + type = PyTuple_GetItem(args, 0); + Py_INCREF(type); + break; + default: + PyErr_SetString(PyExc_TypeError, + "wrong number of arguments"); + return NULL; + } + PyErr_Restore(type, value, traceback); + assert(PyErr_Occurred()); + return NULL; +} + +/* To test the format of exceptions as printed out. */ +static PyObject * +exception_print(PyObject *self, PyObject *args) +{ + PyObject *value; + PyObject *tb = NULL; + + if (!PyArg_ParseTuple(args, "O:exception_print", &value)) { + return NULL; + } + + if (PyExceptionInstance_Check(value)) { + tb = PyException_GetTraceback(value); + } + + PyErr_Display((PyObject *) Py_TYPE(value), value, tb); + Py_XDECREF(tb); + + Py_RETURN_NONE; +} + +/* Test PyErr_NewExceptionWithDoc (also exercise PyErr_NewException). + Run via Lib/test/test_exceptions.py */ +static PyObject * +make_exception_with_doc(PyObject *self, PyObject *args, PyObject *kwargs) +{ + const char *name; + const char *doc = NULL; + PyObject *base = NULL; + PyObject *dict = NULL; + + static char *kwlist[] = {"name", "doc", "base", "dict", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "s|sOO:make_exception_with_doc", kwlist, + &name, &doc, &base, &dict)) + { + return NULL; + } + + return PyErr_NewExceptionWithDoc(name, doc, base, dict); +} + +static PyObject * +raise_exception(PyObject *self, PyObject *args) +{ + PyObject *exc; + int num_args; + + if (!PyArg_ParseTuple(args, "Oi:raise_exception", &exc, &num_args)) { + return NULL; + } + + PyObject *exc_args = PyTuple_New(num_args); + if (exc_args == NULL) { + return NULL; + } + for (int i = 0; i < num_args; ++i) { + PyObject *v = PyLong_FromLong(i); + if (v == NULL) { + Py_DECREF(exc_args); + return NULL; + } + PyTuple_SET_ITEM(exc_args, i, v); + } + PyErr_SetObject(exc, exc_args); + Py_DECREF(exc_args); + return NULL; +} + +/* reliably raise a MemoryError */ +static PyObject * +raise_memoryerror(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyErr_NoMemory(); +} + +static PyObject * +test_fatal_error(PyObject *self, PyObject *args) +{ + char *message; + int release_gil = 0; + if (!PyArg_ParseTuple(args, "y|i:fatal_error", &message, &release_gil)) { + return NULL; + } + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + Py_FatalError(message); + Py_END_ALLOW_THREADS + } + else { + Py_FatalError(message); + } + // Py_FatalError() does not return, but exits the process. + Py_RETURN_NONE; +} + +static PyObject * +test_set_exc_info(PyObject *self, PyObject *args) +{ + PyObject *new_type, *new_value, *new_tb; + PyObject *type, *value, *tb; + if (!PyArg_ParseTuple(args, "OOO:test_set_exc_info", + &new_type, &new_value, &new_tb)) + { + return NULL; + } + + PyErr_GetExcInfo(&type, &value, &tb); + + Py_INCREF(new_type); + Py_INCREF(new_value); + Py_INCREF(new_tb); + PyErr_SetExcInfo(new_type, new_value, new_tb); + + PyObject *orig_exc = PyTuple_Pack(3, + type ? type : Py_None, + value ? value : Py_None, + tb ? tb : Py_None); + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(tb); + return orig_exc; +} + +static PyObject * +test_set_exception(PyObject *self, PyObject *new_exc) +{ + PyObject *exc = PyErr_GetHandledException(); + assert(PyExceptionInstance_Check(exc) || exc == NULL); + + PyErr_SetHandledException(new_exc); + return exc; +} + +static PyObject * +test_write_unraisable_exc(PyObject *self, PyObject *args) +{ + PyObject *exc, *err_msg, *obj; + if (!PyArg_ParseTuple(args, "OOO", &exc, &err_msg, &obj)) { + return NULL; + } + + const char *err_msg_utf8; + if (err_msg != Py_None) { + err_msg_utf8 = PyUnicode_AsUTF8(err_msg); + if (err_msg_utf8 == NULL) { + return NULL; + } + } + else { + err_msg_utf8 = NULL; + } + + PyErr_SetObject((PyObject *)Py_TYPE(exc), exc); + _PyErr_WriteUnraisableMsg(err_msg_utf8, obj); + Py_RETURN_NONE; +} + +/* To test the format of tracebacks as printed out. */ +static PyObject * +traceback_print(PyObject *self, PyObject *args) +{ + PyObject *file; + PyObject *traceback; + + if (!PyArg_ParseTuple(args, "OO:traceback_print", + &traceback, &file)) + { + return NULL; + } + + if (PyTraceBack_Print(traceback, file) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +/* + * Define the PyRecurdingInfinitelyError_Type + */ + +static PyTypeObject PyRecursingInfinitelyError_Type; + +static int +recurse_infinitely_error_init(PyObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *type = (PyObject *)&PyRecursingInfinitelyError_Type; + + /* Instantiating this exception starts infinite recursion. */ + Py_INCREF(type); + PyErr_SetObject(type, NULL); + return -1; +} + +static PyTypeObject PyRecursingInfinitelyError_Type = { + .tp_name = "RecursingInfinitelyError", + .tp_basicsize = sizeof(PyBaseExceptionObject), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_doc = PyDoc_STR("Instantiating this exception starts infinite recursion."), + .tp_init = (initproc)recurse_infinitely_error_init, +}; + +static PyMethodDef test_methods[] = { + {"err_restore", err_restore, METH_VARARGS}, + {"err_set_raised", err_set_raised, METH_O}, + {"exception_print", exception_print, METH_VARARGS}, + {"fatal_error", test_fatal_error, METH_VARARGS, + PyDoc_STR("fatal_error(message, release_gil=False): call Py_FatalError(message)")}, + {"make_exception_with_doc", _PyCFunction_CAST(make_exception_with_doc), + METH_VARARGS | METH_KEYWORDS}, + {"raise_exception", raise_exception, METH_VARARGS}, + {"raise_memoryerror", raise_memoryerror, METH_NOARGS}, + {"set_exc_info", test_set_exc_info, METH_VARARGS}, + {"set_exception", test_set_exception, METH_O}, + {"traceback_print", traceback_print, METH_VARARGS}, + {"write_unraisable_exc", test_write_unraisable_exc, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Exceptions(PyObject *mod) +{ + PyRecursingInfinitelyError_Type.tp_base = (PyTypeObject *)PyExc_Exception; + if (PyType_Ready(&PyRecursingInfinitelyError_Type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(mod, "RecursingInfinitelyError", + (PyObject *)&PyRecursingInfinitelyError_Type) < 0) + { + return -1; + } + + if (PyModule_AddFunctions(mod, test_methods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 39639f7ed048f2..df2a061ed82b06 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -623,16 +623,15 @@ heapctypesubclasswithfinalizer_init(PyObject *self, PyObject *args, PyObject *kw static void heapctypesubclasswithfinalizer_finalize(PyObject *self) { - PyObject *error_type, *error_value, *error_traceback, *m; PyObject *oldtype = NULL, *newtype = NULL, *refcnt = NULL; /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); if (_testcapimodule == NULL) { goto cleanup_finalize; } - m = PyState_FindModule(_testcapimodule); + PyObject *m = PyState_FindModule(_testcapimodule); if (m == NULL) { goto cleanup_finalize; } @@ -667,7 +666,7 @@ heapctypesubclasswithfinalizer_finalize(PyObject *self) Py_XDECREF(refcnt); /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } static PyType_Slot HeapCTypeSubclassWithFinalizer_slots[] = { diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 7ba3c4ebff8cde..c8f31dc8e39fae 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -36,6 +36,8 @@ int _PyTestCapi_Init_Watchers(PyObject *module); int _PyTestCapi_Init_Long(PyObject *module); int _PyTestCapi_Init_Float(PyObject *module); int _PyTestCapi_Init_Structmember(PyObject *module); +int _PyTestCapi_Init_Exceptions(PyObject *module); +int _PyTestCapi_Init_Code(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 2e8fe1dbf78651..d9ace632768ae8 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -389,16 +389,15 @@ allocate_too_many_code_watchers(PyObject *self, PyObject *args) watcher_ids[i] = watcher_id; num_watchers++; } - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); + PyObject *exc = PyErr_GetRaisedException(); for (int i = 0; i < num_watchers; i++) { if (PyCode_ClearWatcher(watcher_ids[i]) < 0) { PyErr_WriteUnraisable(Py_None); break; } } - if (type) { - PyErr_Restore(type, value, traceback); + if (exc) { + PyErr_SetRaisedException(exc); return NULL; } else if (PyErr_Occurred()) { @@ -578,16 +577,15 @@ allocate_too_many_func_watchers(PyObject *self, PyObject *args) watcher_ids[i] = watcher_id; num_watchers++; } - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); + PyObject *exc = PyErr_GetRaisedException(); for (int i = 0; i < num_watchers; i++) { if (PyFunction_ClearWatcher(watcher_ids[i]) < 0) { PyErr_WriteUnraisable(Py_None); break; } } - if (type) { - PyErr_Restore(type, value, traceback); + if (exc) { + PyErr_SetRaisedException(exc); return NULL; } else if (PyErr_Occurred()) { diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0d8d1d73fb2390..10e507d6b481de 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -720,33 +720,6 @@ pyobject_bytes_from_null(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyObject_Bytes(NULL); } -static PyObject * -raise_exception(PyObject *self, PyObject *args) -{ - PyObject *exc; - PyObject *exc_args, *v; - int num_args, i; - - if (!PyArg_ParseTuple(args, "Oi:raise_exception", - &exc, &num_args)) - return NULL; - - exc_args = PyTuple_New(num_args); - if (exc_args == NULL) - return NULL; - for (i = 0; i < num_args; ++i) { - v = PyLong_FromLong(i); - if (v == NULL) { - Py_DECREF(exc_args); - return NULL; - } - PyTuple_SET_ITEM(exc_args, i, v); - } - PyErr_SetObject(exc, exc_args); - Py_DECREF(exc_args); - return NULL; -} - static PyObject * set_errno(PyObject *self, PyObject *args) { @@ -759,40 +732,6 @@ set_errno(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject * -test_set_exception(PyObject *self, PyObject *new_exc) -{ - PyObject *exc = PyErr_GetHandledException(); - assert(PyExceptionInstance_Check(exc) || exc == NULL); - - PyErr_SetHandledException(new_exc); - return exc; -} - -static PyObject * -test_set_exc_info(PyObject *self, PyObject *args) -{ - PyObject *orig_exc; - PyObject *new_type, *new_value, *new_tb; - PyObject *type, *value, *tb; - if (!PyArg_ParseTuple(args, "OOO:test_set_exc_info", - &new_type, &new_value, &new_tb)) - return NULL; - - PyErr_GetExcInfo(&type, &value, &tb); - - Py_INCREF(new_type); - Py_INCREF(new_value); - Py_INCREF(new_tb); - PyErr_SetExcInfo(new_type, new_value, new_tb); - - orig_exc = PyTuple_Pack(3, type ? type : Py_None, value ? value : Py_None, tb ? tb : Py_None); - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(tb); - return orig_exc; -} - /* test_thread_state spawns a thread of its own, and that thread releases * `thread_done` when it's finished. The driver code has to know when the * thread finishes, because the thread uses a PyObject (the callable) that @@ -1272,57 +1211,6 @@ profile_int(PyObject *self, PyObject* args) } #endif -/* To test the format of tracebacks as printed out. */ -static PyObject * -traceback_print(PyObject *self, PyObject *args) -{ - PyObject *file; - PyObject *traceback; - int result; - - if (!PyArg_ParseTuple(args, "OO:traceback_print", - &traceback, &file)) - return NULL; - - result = PyTraceBack_Print(traceback, file); - if (result < 0) - return NULL; - Py_RETURN_NONE; -} - -/* To test the format of exceptions as printed out. */ -static PyObject * -exception_print(PyObject *self, PyObject *args) -{ - PyObject *value; - PyObject *tb = NULL; - - if (!PyArg_ParseTuple(args, "O:exception_print", - &value)) { - return NULL; - } - - if (PyExceptionInstance_Check(value)) { - tb = PyException_GetTraceback(value); - } - - PyErr_Display((PyObject *) Py_TYPE(value), value, tb); - Py_XDECREF(tb); - - Py_RETURN_NONE; -} - - - - -/* reliably raise a MemoryError */ -static PyObject * -raise_memoryerror(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyErr_NoMemory(); - return NULL; -} - /* Issue 6012 */ static PyObject *str1, *str2; static int @@ -1368,26 +1256,6 @@ code_newempty(PyObject *self, PyObject *args) return (PyObject *)PyCode_NewEmpty(filename, funcname, firstlineno); } -/* Test PyErr_NewExceptionWithDoc (also exercise PyErr_NewException). - Run via Lib/test/test_exceptions.py */ -static PyObject * -make_exception_with_doc(PyObject *self, PyObject *args, PyObject *kwargs) -{ - const char *name; - const char *doc = NULL; - PyObject *base = NULL; - PyObject *dict = NULL; - - static char *kwlist[] = {"name", "doc", "base", "dict", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "s|sOO:make_exception_with_doc", kwlist, - &name, &doc, &base, &dict)) - return NULL; - - return PyErr_NewExceptionWithDoc(name, doc, base, dict); -} - static PyObject * make_memoryview_from_NULL_pointer(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -1743,19 +1611,18 @@ static void slot_tp_del(PyObject *self) { PyObject *del, *res; - PyObject *error_type, *error_value, *error_traceback; /* Temporarily resurrect the object. */ assert(Py_REFCNT(self) == 0); Py_SET_REFCNT(self, 1); /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); PyObject *tp_del = PyUnicode_InternFromString("__tp_del__"); if (tp_del == NULL) { PyErr_WriteUnraisable(NULL); - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); return; } /* Execute __del__ method, if any. */ @@ -1770,7 +1637,7 @@ slot_tp_del(PyObject *self) } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); /* Undo the temporary resurrection; can't use DECREF here, it would * cause a recursive call. @@ -2491,31 +2358,6 @@ negative_refcount(PyObject *self, PyObject *Py_UNUSED(args)) #endif -static PyObject* -test_write_unraisable_exc(PyObject *self, PyObject *args) -{ - PyObject *exc, *err_msg, *obj; - if (!PyArg_ParseTuple(args, "OOO", &exc, &err_msg, &obj)) { - return NULL; - } - - const char *err_msg_utf8; - if (err_msg != Py_None) { - err_msg_utf8 = PyUnicode_AsUTF8(err_msg); - if (err_msg_utf8 == NULL) { - return NULL; - } - } - else { - err_msg_utf8 = NULL; - } - - PyErr_SetObject((PyObject *)Py_TYPE(exc), exc); - _PyErr_WriteUnraisableMsg(err_msg_utf8, obj); - Py_RETURN_NONE; -} - - static PyObject * sequence_getitem(PyObject *self, PyObject *args) { @@ -2874,25 +2716,6 @@ test_py_is_funcs(PyObject *self, PyObject *Py_UNUSED(ignored)) } -static PyObject * -test_fatal_error(PyObject *self, PyObject *args) -{ - char *message; - int release_gil = 0; - if (!PyArg_ParseTuple(args, "y|i:fatal_error", &message, &release_gil)) - return NULL; - if (release_gil) { - Py_BEGIN_ALLOW_THREADS - Py_FatalError(message); - Py_END_ALLOW_THREADS - } - else { - Py_FatalError(message); - } - // Py_FatalError() does not return, but exits the process. - Py_RETURN_NONE; -} - // type->tp_version_tag static PyObject * type_get_version(PyObject *self, PyObject *type) @@ -3076,6 +2899,16 @@ eval_get_func_desc(PyObject *self, PyObject *func) return PyUnicode_FromString(PyEval_GetFuncDesc(func)); } +static PyObject * +gen_get_code(PyObject *self, PyObject *gen) +{ + if (!PyGen_Check(gen)) { + PyErr_SetString(PyExc_TypeError, "argument must be a generator object"); + return NULL; + } + return (PyObject *)PyGen_GetCode((PyGenObject *)gen); +} + static PyObject * eval_eval_code_ex(PyObject *mod, PyObject *pos_args) { @@ -3482,46 +3315,9 @@ function_set_kw_defaults(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject * -err_set_raised(PyObject *self, PyObject *exc) -{ - Py_INCREF(exc); - PyErr_SetRaisedException(exc); - assert(PyErr_Occurred()); - return NULL; -} - -static PyObject * -err_restore(PyObject *self, PyObject *args) { - PyObject *type = NULL, *value = NULL, *traceback = NULL; - switch(PyTuple_Size(args)) { - case 3: - traceback = PyTuple_GetItem(args, 2); - Py_INCREF(traceback); - /* fall through */ - case 2: - value = PyTuple_GetItem(args, 1); - Py_INCREF(value); - /* fall through */ - case 1: - type = PyTuple_GetItem(args, 0); - Py_INCREF(type); - break; - default: - PyErr_SetString(PyExc_TypeError, - "wrong number of arguments"); - return NULL; - } - PyErr_Restore(type, value, traceback); - assert(PyErr_Occurred()); - return NULL; -} - static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyMethodDef TestMethods[] = { - {"raise_exception", raise_exception, METH_VARARGS}, - {"raise_memoryerror", raise_memoryerror, METH_NOARGS}, {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, {"test_sizeof_c_types", test_sizeof_c_types, METH_NOARGS}, @@ -3564,15 +3360,9 @@ static PyMethodDef TestMethods[] = { #ifdef HAVE_GETTIMEOFDAY {"profile_int", profile_int, METH_NOARGS}, #endif - {"traceback_print", traceback_print, METH_VARARGS}, - {"exception_print", exception_print, METH_VARARGS}, - {"set_exception", test_set_exception, METH_O}, - {"set_exc_info", test_set_exc_info, METH_VARARGS}, {"argparsing", argparsing, METH_VARARGS}, {"code_newempty", code_newempty, METH_VARARGS}, {"eval_code_ex", eval_eval_code_ex, METH_VARARGS}, - {"make_exception_with_doc", _PyCFunction_CAST(make_exception_with_doc), - METH_VARARGS | METH_KEYWORDS}, {"make_memoryview_from_NULL_pointer", make_memoryview_from_NULL_pointer, METH_NOARGS}, {"crash_no_current_thread", crash_no_current_thread, METH_NOARGS}, @@ -3623,7 +3413,6 @@ static PyMethodDef TestMethods[] = { #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, #endif - {"write_unraisable_exc", test_write_unraisable_exc, METH_VARARGS}, {"sequence_getitem", sequence_getitem, METH_VARARGS}, {"sequence_setitem", sequence_setitem, METH_VARARGS}, {"sequence_delitem", sequence_delitem, METH_VARARGS}, @@ -3643,8 +3432,6 @@ static PyMethodDef TestMethods[] = { {"test_refcount_funcs", test_refcount_funcs, METH_NOARGS}, {"test_py_is_macros", test_py_is_macros, METH_NOARGS}, {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, - {"fatal_error", test_fatal_error, METH_VARARGS, - PyDoc_STR("fatal_error(message, release_gil=False): call Py_FatalError(message)")}, {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL}, {"frame_getlocals", frame_getlocals, METH_O, NULL}, @@ -3657,6 +3444,7 @@ static PyMethodDef TestMethods[] = { {"frame_getvarstring", test_frame_getvarstring, METH_VARARGS, NULL}, {"eval_get_func_name", eval_get_func_name, METH_O, NULL}, {"eval_get_func_desc", eval_get_func_desc, METH_O, NULL}, + {"gen_get_code", gen_get_code, METH_O, NULL}, {"get_feature_macros", get_feature_macros, METH_NOARGS, NULL}, {"test_code_api", test_code_api, METH_NOARGS, NULL}, {"settrace_to_record", settrace_to_record, METH_O, NULL}, @@ -3669,8 +3457,6 @@ static PyMethodDef TestMethods[] = { {"function_set_defaults", function_set_defaults, METH_VARARGS, NULL}, {"function_get_kw_defaults", function_get_kw_defaults, METH_O, NULL}, {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, - {"err_set_raised", err_set_raised, METH_O, NULL}, - {"err_restore", err_restore, METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ }; @@ -3891,61 +3677,6 @@ static PyTypeObject awaitType = { }; -static int recurse_infinitely_error_init(PyObject *, PyObject *, PyObject *); - -static PyTypeObject PyRecursingInfinitelyError_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "RecursingInfinitelyError", /* tp_name */ - sizeof(PyBaseExceptionObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR("Instantiating this exception starts infinite recursion."), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)recurse_infinitely_error_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - -static int -recurse_infinitely_error_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - PyObject *type = (PyObject *)&PyRecursingInfinitelyError_Type; - - /* Instantiating this exception starts infinite recursion. */ - Py_INCREF(type); - PyErr_SetObject(type, NULL); - return -1; -} - - /* Test bpo-35983: create a subclass of "list" which checks that instances * are not deallocated twice */ @@ -4272,14 +4003,6 @@ PyInit__testcapi(void) Py_INCREF(&MethStatic_Type); PyModule_AddObject(m, "MethStatic", (PyObject *)&MethStatic_Type); - PyRecursingInfinitelyError_Type.tp_base = (PyTypeObject *)PyExc_Exception; - if (PyType_Ready(&PyRecursingInfinitelyError_Type) < 0) { - return NULL; - } - Py_INCREF(&PyRecursingInfinitelyError_Type); - PyModule_AddObject(m, "RecursingInfinitelyError", - (PyObject *)&PyRecursingInfinitelyError_Type); - PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX)); PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN)); PyModule_AddObject(m, "UCHAR_MAX", PyLong_FromLong(UCHAR_MAX)); @@ -4357,6 +4080,12 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Structmember(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Exceptions(m) < 0) { + return NULL; + } + if (_PyTestCapi_Init_Code(m) < 0) { + return NULL; + } #ifndef LIMITED_API_AVAILABLE PyModule_AddObjectRef(m, "LIMITED_API_AVAILABLE", Py_False); diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 565221c887e5ae..a16157702ae789 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -140,7 +140,7 @@ init_module(PyObject *module, module_state *state) if (initialized == NULL) { return -1; } - if (PyModule_AddObjectRef(module, "_initialized", initialized) != 0) { + if (PyModule_AddObjectRef(module, "_module_initialized", initialized) != 0) { return -1; } @@ -148,13 +148,13 @@ init_module(PyObject *module, module_state *state) } -PyDoc_STRVAR(common_initialized_doc, -"initialized()\n\ +PyDoc_STRVAR(common_state_initialized_doc, +"state_initialized()\n\ \n\ -Return the seconds-since-epoch when the module was initialized."); +Return the seconds-since-epoch when the module state was initialized."); static PyObject * -common_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) +common_state_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) { module_state *state = get_module_state(self); if (state == NULL) { @@ -164,9 +164,9 @@ common_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyFloat_FromDouble(d); } -#define INITIALIZED_METHODDEF \ - {"initialized", common_initialized, METH_NOARGS, \ - common_initialized_doc} +#define STATE_INITIALIZED_METHODDEF \ + {"state_initialized", common_state_initialized, METH_NOARGS, \ + common_state_initialized_doc} PyDoc_STRVAR(common_look_up_self_doc, @@ -265,7 +265,7 @@ basic__clear_globals(PyObject *self, PyObject *Py_UNUSED(ignored)) static PyMethodDef TestMethods_Basic[] = { LOOK_UP_SELF_METHODDEF, SUM_METHODDEF, - INITIALIZED_METHODDEF, + STATE_INITIALIZED_METHODDEF, INITIALIZED_COUNT_METHODDEF, _CLEAR_GLOBALS_METHODDEF, {NULL, NULL} /* sentinel */ @@ -360,7 +360,7 @@ PyInit__testsinglephase_basic_copy(void) static PyMethodDef TestMethods_Reinit[] = { LOOK_UP_SELF_METHODDEF, SUM_METHODDEF, - INITIALIZED_METHODDEF, + STATE_INITIALIZED_METHODDEF, {NULL, NULL} /* sentinel */ }; @@ -421,7 +421,7 @@ PyInit__testsinglephase_with_reinit(void) static PyMethodDef TestMethods_WithState[] = { LOOK_UP_SELF_METHODDEF, SUM_METHODDEF, - INITIALIZED_METHODDEF, + STATE_INITIALIZED_METHODDEF, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 60538c31874864..a0cd4a2363fb53 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -100,9 +100,9 @@ add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared) static int _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) { - PyObject *exctype, *excval, *exctb; + PyObject *exc; if (ignoreexc) { - PyErr_Fetch(&exctype, &excval, &exctb); + exc = PyErr_GetRaisedException(); } int res = _PyCrossInterpreterData_Release(data); if (res < 0) { @@ -125,7 +125,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) } } if (ignoreexc) { - PyErr_Restore(exctype, excval, exctb); + PyErr_SetRaisedException(exc); } return res; } diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 9f423559f51a43..c215a75b804fdb 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -204,7 +204,7 @@ zoneinfo_get_state(PyObject *mod) static inline zoneinfo_state * zoneinfo_get_state_by_cls(PyTypeObject *cls) { - zoneinfo_state *state = (zoneinfo_state *)PyType_GetModuleState(cls); + zoneinfo_state *state = (zoneinfo_state *)_PyType_GetModuleState(cls); assert(state != NULL); return state; } @@ -270,10 +270,9 @@ zoneinfo_new_instance(zoneinfo_state *state, PyTypeObject *type, PyObject *key) Py_CLEAR(self); cleanup: if (file_obj != NULL) { - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); PyObject *tmp = PyObject_CallMethod(file_obj, "close", NULL); - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); if (tmp == NULL) { Py_CLEAR(self); } diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index 50a48b0bf2b825..d7797d639ae32e 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -71,6 +71,48 @@ _bz2_BZ2Compressor_flush(BZ2Compressor *self, PyObject *Py_UNUSED(ignored)) return _bz2_BZ2Compressor_flush_impl(self); } +PyDoc_STRVAR(_bz2_BZ2Compressor__doc__, +"BZ2Compressor(compresslevel=9, /)\n" +"--\n" +"\n" +"Create a compressor object for compressing data incrementally.\n" +"\n" +" compresslevel\n" +" Compression level, as a number between 1 and 9.\n" +"\n" +"For one-shot compression, use the compress() function instead."); + +static PyObject * +_bz2_BZ2Compressor_impl(PyTypeObject *type, int compresslevel); + +static PyObject * +_bz2_BZ2Compressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = clinic_state()->bz2_compressor_type; + int compresslevel = 9; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("BZ2Compressor", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("BZ2Compressor", PyTuple_GET_SIZE(args), 0, 1)) { + goto exit; + } + if (PyTuple_GET_SIZE(args) < 1) { + goto skip_optional; + } + compresslevel = _PyLong_AsInt(PyTuple_GET_ITEM(args, 0)); + if (compresslevel == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + return_value = _bz2_BZ2Compressor_impl(type, compresslevel); + +exit: + return return_value; +} + PyDoc_STRVAR(_bz2_BZ2Decompressor_decompress__doc__, "decompress($self, /, data, max_length=-1)\n" "--\n" @@ -168,4 +210,35 @@ _bz2_BZ2Decompressor_decompress(BZ2Decompressor *self, PyObject *const *args, Py return return_value; } -/*[clinic end generated code: output=829bed4097cf2e63 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_bz2_BZ2Decompressor__doc__, +"BZ2Decompressor()\n" +"--\n" +"\n" +"Create a decompressor object for decompressing data incrementally.\n" +"\n" +"For one-shot decompression, use the decompress() function instead."); + +static PyObject * +_bz2_BZ2Decompressor_impl(PyTypeObject *type); + +static PyObject * +_bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = clinic_state()->bz2_decompressor_type; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("BZ2Decompressor", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("BZ2Decompressor", kwargs)) { + goto exit; + } + return_value = _bz2_BZ2Decompressor_impl(type); + +exit: + return return_value; +} +/*[clinic end generated code: output=805400e4805098ec input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index 286d2b0070659f..9b396a56683921 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -169,7 +169,7 @@ _lzma_LZMADecompressor_decompress(Decompressor *self, PyObject *const *args, Py_ return return_value; } -PyDoc_STRVAR(_lzma_LZMADecompressor___init____doc__, +PyDoc_STRVAR(_lzma_LZMADecompressor__doc__, "LZMADecompressor(format=FORMAT_AUTO, memlimit=None, filters=None)\n" "--\n" "\n" @@ -192,14 +192,14 @@ PyDoc_STRVAR(_lzma_LZMADecompressor___init____doc__, "\n" "For one-shot decompression, use the decompress() function instead."); -static int -_lzma_LZMADecompressor___init___impl(Decompressor *self, int format, - PyObject *memlimit, PyObject *filters); +static PyObject * +_lzma_LZMADecompressor_impl(PyTypeObject *type, int format, + PyObject *memlimit, PyObject *filters); -static int -_lzma_LZMADecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_lzma_LZMADecompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 3 @@ -257,7 +257,7 @@ _lzma_LZMADecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs } filters = fastargs[2]; skip_optional_pos: - return_value = _lzma_LZMADecompressor___init___impl((Decompressor *)self, format, memlimit, filters); + return_value = _lzma_LZMADecompressor_impl(type, format, memlimit, filters); exit: return return_value; @@ -338,4 +338,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=da3e83ba97244044 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=96c1fbdada1ef232 input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 53e34061d53773..b4f7e5424b4ccf 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -829,7 +829,7 @@ cmath_sqrt_impl(PyObject *module, Py_complex z) ax = fabs(z.real); ay = fabs(z.imag); - if (ax < DBL_MIN && ay < DBL_MIN && (ax > 0. || ay > 0.)) { + if (ax < DBL_MIN && ay < DBL_MIN) { /* here we catch cases where hypot(ax, ay) is subnormal */ ax = ldexp(ax, CM_SCALE_UP); s = ldexp(sqrt(ax + hypot(ax, ldexp(ay, CM_SCALE_UP))), @@ -1013,7 +1013,7 @@ cmath_phase_impl(PyObject *module, Py_complex z) double phi; errno = 0; - phi = c_atan2(z); + phi = c_atan2(z); /* should not cause any exception */ if (errno != 0) return math_error(); else diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6986695e47b1ae..c986e02867ca82 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3,6 +3,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "structmember.h" // PyMemberDef @@ -48,7 +49,7 @@ get_module_state(PyObject *mod) static inline itertools_state * get_module_state_by_cls(PyTypeObject *cls) { - void *state = PyType_GetModuleState(cls); + void *state = _PyType_GetModuleState(cls); assert(state != NULL); return (itertools_state *)state; } diff --git a/Modules/md5module.c b/Modules/md5module.c index 48b11e0779f875..4f7bc77a8836a3 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -22,6 +22,7 @@ #include "Python.h" #include "hashlib.h" #include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() /*[clinic input] module _md5 @@ -43,283 +44,17 @@ typedef long long MD5_INT64; /* 64-bit integer */ #define MD5_BLOCKSIZE 64 #define MD5_DIGESTSIZE 16 -/* The structure for storing MD5 info */ +#include "_hacl/Hacl_Hash_MD5.h" -struct md5_state { - MD5_INT64 length; - MD5_INT32 state[4], curlen; - unsigned char buf[MD5_BLOCKSIZE]; -}; typedef struct { PyObject_HEAD - struct md5_state hash_state; + Hacl_Streaming_MD5_state *hash_state; } MD5object; #include "clinic/md5module.c.h" -/* ------------------------------------------------------------------------ - * - * This code for the MD5 algorithm was noted as public domain. The - * original headers are pasted below. - * - * Several changes have been made to make it more compatible with the - * Python environment and desired interface. - * - */ - -/* LibTomCrypt, modular cryptographic library -- Tom St Denis - * - * LibTomCrypt is a library that provides various cryptographic - * algorithms in a highly modular and flexible manner. - * - * The library is free for all purposes without any express - * guarantee it works. - * - * Tom St Denis, tomstdenis@gmail.com, https://www.libtom.net - */ - -/* rotate the hard way (platform optimizations could be done) */ -#define ROLc(x, y) ( (((unsigned long)(x)<<(unsigned long)((y)&31)) | (((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) - -/* Endian Neutral macros that work on all platforms */ - -#define STORE32L(x, y) \ - { (y)[3] = (unsigned char)(((x)>>24)&255); (y)[2] = (unsigned char)(((x)>>16)&255); \ - (y)[1] = (unsigned char)(((x)>>8)&255); (y)[0] = (unsigned char)((x)&255); } - -#define LOAD32L(x, y) \ - { x = ((unsigned long)((y)[3] & 255)<<24) | \ - ((unsigned long)((y)[2] & 255)<<16) | \ - ((unsigned long)((y)[1] & 255)<<8) | \ - ((unsigned long)((y)[0] & 255)); } - -#define STORE64L(x, y) \ - { (y)[7] = (unsigned char)(((x)>>56)&255); (y)[6] = (unsigned char)(((x)>>48)&255); \ - (y)[5] = (unsigned char)(((x)>>40)&255); (y)[4] = (unsigned char)(((x)>>32)&255); \ - (y)[3] = (unsigned char)(((x)>>24)&255); (y)[2] = (unsigned char)(((x)>>16)&255); \ - (y)[1] = (unsigned char)(((x)>>8)&255); (y)[0] = (unsigned char)((x)&255); } - - -/* MD5 macros */ - -#define F(x,y,z) (z ^ (x & (y ^ z))) -#define G(x,y,z) (y ^ (z & (y ^ x))) -#define H(x,y,z) (x^y^z) -#define I(x,y,z) (y^(x|(~z))) - -#define FF(a,b,c,d,M,s,t) \ - a = (a + F(b,c,d) + M + t); a = ROLc(a, s) + b; - -#define GG(a,b,c,d,M,s,t) \ - a = (a + G(b,c,d) + M + t); a = ROLc(a, s) + b; - -#define HH(a,b,c,d,M,s,t) \ - a = (a + H(b,c,d) + M + t); a = ROLc(a, s) + b; - -#define II(a,b,c,d,M,s,t) \ - a = (a + I(b,c,d) + M + t); a = ROLc(a, s) + b; - - -static void md5_compress(struct md5_state *md5, const unsigned char *buf) -{ - MD5_INT32 i, W[16], a, b, c, d; - - assert(md5 != NULL); - assert(buf != NULL); - - /* copy the state into 512-bits into W[0..15] */ - for (i = 0; i < 16; i++) { - LOAD32L(W[i], buf + (4*i)); - } - - /* copy state */ - a = md5->state[0]; - b = md5->state[1]; - c = md5->state[2]; - d = md5->state[3]; - - FF(a,b,c,d,W[0],7,0xd76aa478UL) - FF(d,a,b,c,W[1],12,0xe8c7b756UL) - FF(c,d,a,b,W[2],17,0x242070dbUL) - FF(b,c,d,a,W[3],22,0xc1bdceeeUL) - FF(a,b,c,d,W[4],7,0xf57c0fafUL) - FF(d,a,b,c,W[5],12,0x4787c62aUL) - FF(c,d,a,b,W[6],17,0xa8304613UL) - FF(b,c,d,a,W[7],22,0xfd469501UL) - FF(a,b,c,d,W[8],7,0x698098d8UL) - FF(d,a,b,c,W[9],12,0x8b44f7afUL) - FF(c,d,a,b,W[10],17,0xffff5bb1UL) - FF(b,c,d,a,W[11],22,0x895cd7beUL) - FF(a,b,c,d,W[12],7,0x6b901122UL) - FF(d,a,b,c,W[13],12,0xfd987193UL) - FF(c,d,a,b,W[14],17,0xa679438eUL) - FF(b,c,d,a,W[15],22,0x49b40821UL) - GG(a,b,c,d,W[1],5,0xf61e2562UL) - GG(d,a,b,c,W[6],9,0xc040b340UL) - GG(c,d,a,b,W[11],14,0x265e5a51UL) - GG(b,c,d,a,W[0],20,0xe9b6c7aaUL) - GG(a,b,c,d,W[5],5,0xd62f105dUL) - GG(d,a,b,c,W[10],9,0x02441453UL) - GG(c,d,a,b,W[15],14,0xd8a1e681UL) - GG(b,c,d,a,W[4],20,0xe7d3fbc8UL) - GG(a,b,c,d,W[9],5,0x21e1cde6UL) - GG(d,a,b,c,W[14],9,0xc33707d6UL) - GG(c,d,a,b,W[3],14,0xf4d50d87UL) - GG(b,c,d,a,W[8],20,0x455a14edUL) - GG(a,b,c,d,W[13],5,0xa9e3e905UL) - GG(d,a,b,c,W[2],9,0xfcefa3f8UL) - GG(c,d,a,b,W[7],14,0x676f02d9UL) - GG(b,c,d,a,W[12],20,0x8d2a4c8aUL) - HH(a,b,c,d,W[5],4,0xfffa3942UL) - HH(d,a,b,c,W[8],11,0x8771f681UL) - HH(c,d,a,b,W[11],16,0x6d9d6122UL) - HH(b,c,d,a,W[14],23,0xfde5380cUL) - HH(a,b,c,d,W[1],4,0xa4beea44UL) - HH(d,a,b,c,W[4],11,0x4bdecfa9UL) - HH(c,d,a,b,W[7],16,0xf6bb4b60UL) - HH(b,c,d,a,W[10],23,0xbebfbc70UL) - HH(a,b,c,d,W[13],4,0x289b7ec6UL) - HH(d,a,b,c,W[0],11,0xeaa127faUL) - HH(c,d,a,b,W[3],16,0xd4ef3085UL) - HH(b,c,d,a,W[6],23,0x04881d05UL) - HH(a,b,c,d,W[9],4,0xd9d4d039UL) - HH(d,a,b,c,W[12],11,0xe6db99e5UL) - HH(c,d,a,b,W[15],16,0x1fa27cf8UL) - HH(b,c,d,a,W[2],23,0xc4ac5665UL) - II(a,b,c,d,W[0],6,0xf4292244UL) - II(d,a,b,c,W[7],10,0x432aff97UL) - II(c,d,a,b,W[14],15,0xab9423a7UL) - II(b,c,d,a,W[5],21,0xfc93a039UL) - II(a,b,c,d,W[12],6,0x655b59c3UL) - II(d,a,b,c,W[3],10,0x8f0ccc92UL) - II(c,d,a,b,W[10],15,0xffeff47dUL) - II(b,c,d,a,W[1],21,0x85845dd1UL) - II(a,b,c,d,W[8],6,0x6fa87e4fUL) - II(d,a,b,c,W[15],10,0xfe2ce6e0UL) - II(c,d,a,b,W[6],15,0xa3014314UL) - II(b,c,d,a,W[13],21,0x4e0811a1UL) - II(a,b,c,d,W[4],6,0xf7537e82UL) - II(d,a,b,c,W[11],10,0xbd3af235UL) - II(c,d,a,b,W[2],15,0x2ad7d2bbUL) - II(b,c,d,a,W[9],21,0xeb86d391UL) - - md5->state[0] = md5->state[0] + a; - md5->state[1] = md5->state[1] + b; - md5->state[2] = md5->state[2] + c; - md5->state[3] = md5->state[3] + d; -} - - -/** - Initialize the hash state - @param md5 The hash state you wish to initialize -*/ -static void -md5_init(struct md5_state *md5) -{ - assert(md5 != NULL); - md5->state[0] = 0x67452301UL; - md5->state[1] = 0xefcdab89UL; - md5->state[2] = 0x98badcfeUL; - md5->state[3] = 0x10325476UL; - md5->curlen = 0; - md5->length = 0; -} - -/** - Process a block of memory though the hash - @param md5 The hash state - @param in The data to hash - @param inlen The length of the data (octets) -*/ -static void -md5_process(struct md5_state *md5, const unsigned char *in, Py_ssize_t inlen) -{ - Py_ssize_t n; - - assert(md5 != NULL); - assert(in != NULL); - assert(md5->curlen <= sizeof(md5->buf)); - - while (inlen > 0) { - if (md5->curlen == 0 && inlen >= MD5_BLOCKSIZE) { - md5_compress(md5, in); - md5->length += MD5_BLOCKSIZE * 8; - in += MD5_BLOCKSIZE; - inlen -= MD5_BLOCKSIZE; - } else { - n = Py_MIN(inlen, (Py_ssize_t)(MD5_BLOCKSIZE - md5->curlen)); - memcpy(md5->buf + md5->curlen, in, (size_t)n); - md5->curlen += (MD5_INT32)n; - in += n; - inlen -= n; - if (md5->curlen == MD5_BLOCKSIZE) { - md5_compress(md5, md5->buf); - md5->length += 8*MD5_BLOCKSIZE; - md5->curlen = 0; - } - } - } -} - -/** - Terminate the hash to get the digest - @param md5 The hash state - @param out [out] The destination of the hash (16 bytes) -*/ -static void -md5_done(struct md5_state *md5, unsigned char *out) -{ - int i; - - assert(md5 != NULL); - assert(out != NULL); - assert(md5->curlen < sizeof(md5->buf)); - - /* increase the length of the message */ - md5->length += md5->curlen * 8; - - /* append the '1' bit */ - md5->buf[md5->curlen++] = (unsigned char)0x80; - - /* if the length is currently above 56 bytes we append zeros - * then compress. Then we can fall back to padding zeros and length - * encoding like normal. - */ - if (md5->curlen > 56) { - while (md5->curlen < 64) { - md5->buf[md5->curlen++] = (unsigned char)0; - } - md5_compress(md5, md5->buf); - md5->curlen = 0; - } - - /* pad up to 56 bytes of zeroes */ - while (md5->curlen < 56) { - md5->buf[md5->curlen++] = (unsigned char)0; - } - - /* store length */ - STORE64L(md5->length, md5->buf+56); - md5_compress(md5, md5->buf); - - /* copy output */ - for (i = 0; i < 4; i++) { - STORE32L(md5->state[i], out+(4*i)); - } -} - -/* .Source: /cvs/libtom/libtomcrypt/src/hashes/md5.c,v $ */ -/* .Revision: 1.10 $ */ -/* .Date: 2007/05/12 14:25:28 $ */ - -/* - * End of copied MD5 code. - * - * ------------------------------------------------------------------------ - */ typedef struct { PyTypeObject* md5_type; @@ -350,8 +85,9 @@ MD5_traverse(PyObject *ptr, visitproc visit, void *arg) } static void -MD5_dealloc(PyObject *ptr) +MD5_dealloc(MD5object *ptr) { + Hacl_Streaming_MD5_legacy_free(ptr->hash_state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -373,13 +109,13 @@ static PyObject * MD5Type_copy_impl(MD5object *self, PyTypeObject *cls) /*[clinic end generated code: output=bf055e08244bf5ee input=d89087dcfb2a8620]*/ { - MD5State *st = PyType_GetModuleState(cls); + MD5State *st = _PyType_GetModuleState(cls); MD5object *newobj; if ((newobj = newMD5object(st))==NULL) return NULL; - newobj->hash_state = self->hash_state; + newobj->hash_state = Hacl_Streaming_MD5_legacy_copy(self->hash_state); return (PyObject *)newobj; } @@ -394,10 +130,7 @@ MD5Type_digest_impl(MD5object *self) /*[clinic end generated code: output=eb691dc4190a07ec input=bc0c4397c2994be6]*/ { unsigned char digest[MD5_DIGESTSIZE]; - struct md5_state temp; - - temp = self->hash_state; - md5_done(&temp, digest); + Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); return PyBytes_FromStringAndSize((const char *)digest, MD5_DIGESTSIZE); } @@ -412,15 +145,21 @@ MD5Type_hexdigest_impl(MD5object *self) /*[clinic end generated code: output=17badced1f3ac932 input=b60b19de644798dd]*/ { unsigned char digest[MD5_DIGESTSIZE]; - struct md5_state temp; - - /* Get the raw (binary) digest value */ - temp = self->hash_state; - md5_done(&temp, digest); - + Hacl_Streaming_MD5_legacy_finish(self->hash_state, digest); return _Py_strhex((const char*)digest, MD5_DIGESTSIZE); } +static void update(Hacl_Streaming_MD5_state *state, uint8_t *buf, Py_ssize_t len) { +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_MD5_legacy_update(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + Hacl_Streaming_MD5_legacy_update(state, buf, (uint32_t) len); +} + /*[clinic input] MD5Type.update @@ -438,7 +177,7 @@ MD5Type_update(MD5object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - md5_process(&self->hash_state, buf.buf, buf.len); + update(self->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -531,7 +270,7 @@ _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - md5_init(&new->hash_state); + new->hash_state = Hacl_Streaming_MD5_legacy_create_in(); if (PyErr_Occurred()) { Py_DECREF(new); @@ -540,7 +279,7 @@ _md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - md5_process(&new->hash_state, buf.buf, buf.len); + update(new->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); } diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 8244202376c74e..a01e798265c5a5 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1603,6 +1603,12 @@ mmap_exec(PyObject *module) // Mostly a no-op on Linux and NetBSD, but useful on OpenBSD // for stack usage (even on x86 arch) ADD_INT_MACRO(module, MAP_STACK); +#endif +#ifdef MAP_ALIGNED_SUPER + ADD_INT_MACRO(module, MAP_ALIGNED_SUPER); +#endif +#ifdef MAP_CONCEAL + ADD_INT_MACRO(module, MAP_CONCEAL); #endif if (PyModule_AddIntConstant(module, "PAGESIZE", (long)my_getpagesize()) < 0 ) { return -1; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 524dc7eb1ccc97..6ea216b3bf5e79 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7946,7 +7946,7 @@ os_getgid_impl(PyObject *module) #endif /* HAVE_GETGID */ -#ifdef HAVE_GETPID +#if defined(HAVE_GETPID) /*[clinic input] os.getpid @@ -7957,9 +7957,13 @@ static PyObject * os_getpid_impl(PyObject *module) /*[clinic end generated code: output=9ea6fdac01ed2b3c input=5a9a00f0ab68aa00]*/ { +#ifdef MS_WINDOWS_NON_DESKTOP + return PyLong_FromUnsignedLong(GetCurrentProcessId()); +#else return PyLong_FromPid(getpid()); +#endif } -#endif /* HAVE_GETPID */ +#endif /* defined(HAVE_GETPID) */ #ifdef NGROUPS_MAX #define MAX_GROUPS NGROUPS_MAX @@ -8265,12 +8269,11 @@ static PyObject* win32_getppid() { HANDLE snapshot; - pid_t mypid; PyObject* result = NULL; BOOL have_record; PROCESSENTRY32 pe; - mypid = getpid(); /* This function never fails */ + DWORD mypid = GetCurrentProcessId(); /* This function never fails */ snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); if (snapshot == INVALID_HANDLE_VALUE) @@ -8279,9 +8282,9 @@ win32_getppid() pe.dwSize = sizeof(pe); have_record = Process32First(snapshot, &pe); while (have_record) { - if (mypid == (pid_t)pe.th32ProcessID) { + if (mypid == pe.th32ProcessID) { /* We could cache the ulong value in a static variable. */ - result = PyLong_FromPid((pid_t)pe.th32ParentProcessID); + result = PyLong_FromUnsignedLong(pe.th32ParentProcessID); break; } @@ -14756,10 +14759,9 @@ ScandirIterator_exit(ScandirIterator *self, PyObject *args) static void ScandirIterator_finalize(ScandirIterator *iterator) { - PyObject *error_type, *error_value, *error_traceback; /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); if (!ScandirIterator_is_closed(iterator)) { ScandirIterator_closedir(iterator); @@ -14776,7 +14778,7 @@ ScandirIterator_finalize(ScandirIterator *iterator) path_cleanup(&iterator->path); /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } static void diff --git a/Modules/sha1module.c b/Modules/sha1module.c index 9153557fbde740..f8d4056fd34b65 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -22,6 +22,7 @@ #include "Python.h" #include "hashlib.h" #include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() /*[clinic input] module _sha1 @@ -43,260 +44,16 @@ typedef long long SHA1_INT64; /* 64-bit integer */ #define SHA1_BLOCKSIZE 64 #define SHA1_DIGESTSIZE 20 -/* The structure for storing SHA1 info */ - -struct sha1_state { - SHA1_INT64 length; - SHA1_INT32 state[5], curlen; - unsigned char buf[SHA1_BLOCKSIZE]; -}; +#include "_hacl/Hacl_Hash_SHA1.h" typedef struct { PyObject_HEAD - struct sha1_state hash_state; + Hacl_Streaming_SHA1_state *hash_state; } SHA1object; #include "clinic/sha1module.c.h" -/* ------------------------------------------------------------------------ - * - * This code for the SHA1 algorithm was noted as public domain. The - * original headers are pasted below. - * - * Several changes have been made to make it more compatible with the - * Python environment and desired interface. - * - */ - -/* LibTomCrypt, modular cryptographic library -- Tom St Denis - * - * LibTomCrypt is a library that provides various cryptographic - * algorithms in a highly modular and flexible manner. - * - * The library is free for all purposes without any express - * guarantee it works. - * - * Tom St Denis, tomstdenis@gmail.com, https://www.libtom.net - */ - -/* rotate the hard way (platform optimizations could be done) */ -#define ROL(x, y) ( (((unsigned long)(x)<<(unsigned long)((y)&31)) | (((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) -#define ROLc(x, y) ( (((unsigned long)(x)<<(unsigned long)((y)&31)) | (((unsigned long)(x)&0xFFFFFFFFUL)>>(unsigned long)(32-((y)&31)))) & 0xFFFFFFFFUL) - -/* Endian Neutral macros that work on all platforms */ - -#define STORE32H(x, y) \ - { (y)[0] = (unsigned char)(((x)>>24)&255); (y)[1] = (unsigned char)(((x)>>16)&255); \ - (y)[2] = (unsigned char)(((x)>>8)&255); (y)[3] = (unsigned char)((x)&255); } - -#define LOAD32H(x, y) \ - { x = ((unsigned long)((y)[0] & 255)<<24) | \ - ((unsigned long)((y)[1] & 255)<<16) | \ - ((unsigned long)((y)[2] & 255)<<8) | \ - ((unsigned long)((y)[3] & 255)); } - -#define STORE64H(x, y) \ - { (y)[0] = (unsigned char)(((x)>>56)&255); (y)[1] = (unsigned char)(((x)>>48)&255); \ - (y)[2] = (unsigned char)(((x)>>40)&255); (y)[3] = (unsigned char)(((x)>>32)&255); \ - (y)[4] = (unsigned char)(((x)>>24)&255); (y)[5] = (unsigned char)(((x)>>16)&255); \ - (y)[6] = (unsigned char)(((x)>>8)&255); (y)[7] = (unsigned char)((x)&255); } - - -/* SHA1 macros */ - -#define F0(x,y,z) (z ^ (x & (y ^ z))) -#define F1(x,y,z) (x ^ y ^ z) -#define F2(x,y,z) ((x & y) | (z & (x | y))) -#define F3(x,y,z) (x ^ y ^ z) - -static void sha1_compress(struct sha1_state *sha1, unsigned char *buf) -{ - SHA1_INT32 a,b,c,d,e,W[80],i; - - /* copy the state into 512-bits into W[0..15] */ - for (i = 0; i < 16; i++) { - LOAD32H(W[i], buf + (4*i)); - } - - /* copy state */ - a = sha1->state[0]; - b = sha1->state[1]; - c = sha1->state[2]; - d = sha1->state[3]; - e = sha1->state[4]; - - /* expand it */ - for (i = 16; i < 80; i++) { - W[i] = ROL(W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16], 1); - } - - /* compress */ - /* round one */ - #define FF_0(a,b,c,d,e,i) e = (ROLc(a, 5) + F0(b,c,d) + e + W[i] + 0x5a827999UL); b = ROLc(b, 30); - #define FF_1(a,b,c,d,e,i) e = (ROLc(a, 5) + F1(b,c,d) + e + W[i] + 0x6ed9eba1UL); b = ROLc(b, 30); - #define FF_2(a,b,c,d,e,i) e = (ROLc(a, 5) + F2(b,c,d) + e + W[i] + 0x8f1bbcdcUL); b = ROLc(b, 30); - #define FF_3(a,b,c,d,e,i) e = (ROLc(a, 5) + F3(b,c,d) + e + W[i] + 0xca62c1d6UL); b = ROLc(b, 30); - - for (i = 0; i < 20; ) { - FF_0(a,b,c,d,e,i++); - FF_0(e,a,b,c,d,i++); - FF_0(d,e,a,b,c,i++); - FF_0(c,d,e,a,b,i++); - FF_0(b,c,d,e,a,i++); - } - - /* round two */ - for (; i < 40; ) { - FF_1(a,b,c,d,e,i++); - FF_1(e,a,b,c,d,i++); - FF_1(d,e,a,b,c,i++); - FF_1(c,d,e,a,b,i++); - FF_1(b,c,d,e,a,i++); - } - - /* round three */ - for (; i < 60; ) { - FF_2(a,b,c,d,e,i++); - FF_2(e,a,b,c,d,i++); - FF_2(d,e,a,b,c,i++); - FF_2(c,d,e,a,b,i++); - FF_2(b,c,d,e,a,i++); - } - - /* round four */ - for (; i < 80; ) { - FF_3(a,b,c,d,e,i++); - FF_3(e,a,b,c,d,i++); - FF_3(d,e,a,b,c,i++); - FF_3(c,d,e,a,b,i++); - FF_3(b,c,d,e,a,i++); - } - - #undef FF_0 - #undef FF_1 - #undef FF_2 - #undef FF_3 - - /* store */ - sha1->state[0] = sha1->state[0] + a; - sha1->state[1] = sha1->state[1] + b; - sha1->state[2] = sha1->state[2] + c; - sha1->state[3] = sha1->state[3] + d; - sha1->state[4] = sha1->state[4] + e; -} - -/** - Initialize the hash state - @param sha1 The hash state you wish to initialize -*/ -static void -sha1_init(struct sha1_state *sha1) -{ - assert(sha1 != NULL); - sha1->state[0] = 0x67452301UL; - sha1->state[1] = 0xefcdab89UL; - sha1->state[2] = 0x98badcfeUL; - sha1->state[3] = 0x10325476UL; - sha1->state[4] = 0xc3d2e1f0UL; - sha1->curlen = 0; - sha1->length = 0; -} - -/** - Process a block of memory though the hash - @param sha1 The hash state - @param in The data to hash - @param inlen The length of the data (octets) -*/ -static void -sha1_process(struct sha1_state *sha1, - const unsigned char *in, Py_ssize_t inlen) -{ - Py_ssize_t n; - - assert(sha1 != NULL); - assert(in != NULL); - assert(sha1->curlen <= sizeof(sha1->buf)); - - while (inlen > 0) { - if (sha1->curlen == 0 && inlen >= SHA1_BLOCKSIZE) { - sha1_compress(sha1, (unsigned char *)in); - sha1->length += SHA1_BLOCKSIZE * 8; - in += SHA1_BLOCKSIZE; - inlen -= SHA1_BLOCKSIZE; - } else { - n = Py_MIN(inlen, (Py_ssize_t)(SHA1_BLOCKSIZE - sha1->curlen)); - memcpy(sha1->buf + sha1->curlen, in, (size_t)n); - sha1->curlen += (SHA1_INT32)n; - in += n; - inlen -= n; - if (sha1->curlen == SHA1_BLOCKSIZE) { - sha1_compress(sha1, sha1->buf); - sha1->length += 8*SHA1_BLOCKSIZE; - sha1->curlen = 0; - } - } - } -} - -/** - Terminate the hash to get the digest - @param sha1 The hash state - @param out [out] The destination of the hash (20 bytes) -*/ -static void -sha1_done(struct sha1_state *sha1, unsigned char *out) -{ - int i; - - assert(sha1 != NULL); - assert(out != NULL); - assert(sha1->curlen < sizeof(sha1->buf)); - - /* increase the length of the message */ - sha1->length += sha1->curlen * 8; - - /* append the '1' bit */ - sha1->buf[sha1->curlen++] = (unsigned char)0x80; - - /* if the length is currently above 56 bytes we append zeros - * then compress. Then we can fall back to padding zeros and length - * encoding like normal. - */ - if (sha1->curlen > 56) { - while (sha1->curlen < 64) { - sha1->buf[sha1->curlen++] = (unsigned char)0; - } - sha1_compress(sha1, sha1->buf); - sha1->curlen = 0; - } - - /* pad up to 56 bytes of zeroes */ - while (sha1->curlen < 56) { - sha1->buf[sha1->curlen++] = (unsigned char)0; - } - - /* store length */ - STORE64H(sha1->length, sha1->buf+56); - sha1_compress(sha1, sha1->buf); - - /* copy output */ - for (i = 0; i < 5; i++) { - STORE32H(sha1->state[i], out+(4*i)); - } -} - - -/* .Source: /cvs/libtom/libtomcrypt/src/hashes/sha1.c,v $ */ -/* .Revision: 1.10 $ */ -/* .Date: 2007/05/12 14:25:28 $ */ - -/* - * End of copied SHA1 code. - * - * ------------------------------------------------------------------------ - */ typedef struct { PyTypeObject* sha1_type; @@ -328,8 +85,9 @@ SHA1_traverse(PyObject *ptr, visitproc visit, void *arg) } static void -SHA1_dealloc(PyObject *ptr) +SHA1_dealloc(SHA1object *ptr) { + Hacl_Streaming_SHA1_legacy_free(ptr->hash_state); PyTypeObject *tp = Py_TYPE(ptr); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -351,13 +109,13 @@ static PyObject * SHA1Type_copy_impl(SHA1object *self, PyTypeObject *cls) /*[clinic end generated code: output=b32d4461ce8bc7a7 input=6c22e66fcc34c58e]*/ { - SHA1State *st = PyType_GetModuleState(cls); + SHA1State *st = _PyType_GetModuleState(cls); SHA1object *newobj; if ((newobj = newSHA1object(st)) == NULL) return NULL; - newobj->hash_state = self->hash_state; + newobj->hash_state = Hacl_Streaming_SHA1_legacy_copy(self->hash_state); return (PyObject *)newobj; } @@ -372,10 +130,7 @@ SHA1Type_digest_impl(SHA1object *self) /*[clinic end generated code: output=2f05302a7aa2b5cb input=13824b35407444bd]*/ { unsigned char digest[SHA1_DIGESTSIZE]; - struct sha1_state temp; - - temp = self->hash_state; - sha1_done(&temp, digest); + Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); return PyBytes_FromStringAndSize((const char *)digest, SHA1_DIGESTSIZE); } @@ -390,15 +145,21 @@ SHA1Type_hexdigest_impl(SHA1object *self) /*[clinic end generated code: output=4161fd71e68c6659 input=97691055c0c74ab0]*/ { unsigned char digest[SHA1_DIGESTSIZE]; - struct sha1_state temp; - - /* Get the raw (binary) digest value */ - temp = self->hash_state; - sha1_done(&temp, digest); - + Hacl_Streaming_SHA1_legacy_finish(self->hash_state, digest); return _Py_strhex((const char *)digest, SHA1_DIGESTSIZE); } +static void update(Hacl_Streaming_SHA1_state *state, uint8_t *buf, Py_ssize_t len) { +#if PY_SSIZE_T_MAX > UINT32_MAX + while (len > UINT32_MAX) { + Hacl_Streaming_SHA1_legacy_update(state, buf, UINT32_MAX); + len -= UINT32_MAX; + buf += UINT32_MAX; + } +#endif + Hacl_Streaming_SHA1_legacy_update(state, buf, (uint32_t) len); +} + /*[clinic input] SHA1Type.update @@ -416,7 +177,7 @@ SHA1Type_update(SHA1object *self, PyObject *obj) GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); - sha1_process(&self->hash_state, buf.buf, buf.len); + update(self->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); Py_RETURN_NONE; @@ -509,7 +270,7 @@ _sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } - sha1_init(&new->hash_state); + new->hash_state = Hacl_Streaming_SHA1_legacy_create_in(); if (PyErr_Occurred()) { Py_DECREF(new); @@ -518,7 +279,7 @@ _sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) return NULL; } if (string) { - sha1_process(&new->hash_state, buf.buf, buf.len); + update(new->hash_state, buf.buf, buf.len); PyBuffer_Release(&buf); } diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 9999f255cd578a..72de20b44762d7 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -23,6 +23,7 @@ #include "Python.h" #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "pycore_strhex.h" // _Py_strhex() #include "structmember.h" // PyMemberDef #include "hashlib.h" @@ -217,7 +218,7 @@ SHA256Type_copy_impl(SHA256object *self, PyTypeObject *cls) /*[clinic end generated code: output=fabd515577805cd3 input=3137146fcb88e212]*/ { SHA256object *newobj; - sha2_state *state = PyType_GetModuleState(cls); + sha2_state *state = _PyType_GetModuleState(cls); if (Py_IS_TYPE(self, state->sha256_type)) { if ((newobj = newSHA256object(state)) == NULL) { return NULL; @@ -245,7 +246,7 @@ SHA512Type_copy_impl(SHA512object *self, PyTypeObject *cls) /*[clinic end generated code: output=66d2a8ef20de8302 input=f673a18f66527c90]*/ { SHA512object *newobj; - sha2_state *state = PyType_GetModuleState(cls); + sha2_state *state = _PyType_GetModuleState(cls); if (Py_IS_TYPE((PyObject*)self, state->sha512_type)) { if ((newobj = newSHA512object(state)) == NULL) { @@ -482,7 +483,7 @@ static PyType_Slot sha512_type_slots[] = { {0,0} }; -// Using PyType_GetModuleState() on these types is safe since they +// Using _PyType_GetModuleState() on these types is safe since they // cannot be subclassed: they don't have the Py_TPFLAGS_BASETYPE flag. static PyType_Spec sha224_type_spec = { .name = "_sha2.SHA224Type", diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 44a5ecf63e9d1e..cd26eca351c0ed 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -234,14 +234,13 @@ signal_default_int_handler_impl(PyObject *module, int signalnum, static int report_wakeup_write_error(void *data) { - PyObject *exc, *val, *tb; int save_errno = errno; errno = (int) (intptr_t) data; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); PyErr_SetFromErrno(PyExc_OSError); _PyErr_WriteUnraisableMsg("when trying to write to the signal wakeup fd", NULL); - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); errno = save_errno; return 0; } @@ -252,14 +251,13 @@ report_wakeup_send_error(void* data) { int send_errno = (int) (intptr_t) data; - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); /* PyErr_SetExcFromWindowsErr() invokes FormatMessage() which recognizes the error codes used by both GetLastError() and WSAGetLastError */ PyErr_SetExcFromWindowsErr(PyExc_OSError, send_errno); _PyErr_WriteUnraisableMsg("when trying to send to the signal wakeup fd", NULL); - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); return 0; } #endif /* MS_WINDOWS */ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 2d300f19436b1a..00608be38f61bb 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -5175,10 +5175,9 @@ static void sock_finalize(PySocketSockObject *s) { SOCKET_T fd; - PyObject *error_type, *error_value, *error_traceback; /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); if (s->sock_fd != INVALID_SOCKET) { if (PyErr_ResourceWarning((PyObject *)s, 1, "unclosed %R", s)) { @@ -5202,7 +5201,7 @@ sock_finalize(PySocketSockObject *s) } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } static void diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 072089e39aa207..49d4dd524696a5 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2391,11 +2391,16 @@ PyDoc_STRVAR(length_hint_doc, static PyObject * bytearrayiter_reduce(bytesiterobject *it, PyObject *Py_UNUSED(ignored)) { + PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); + + /* _PyEval_GetBuiltin can invoke arbitrary code, + * call must be before access of iterator pointers. + * see issue #101765 */ + if (it->it_seq != NULL) { - return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)), - it->it_seq, it->it_index); + return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } else { - return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter))); + return Py_BuildValue("N(())", iter); } } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index ba2c2e978c6e42..657443f31fa709 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3169,11 +3169,16 @@ PyDoc_STRVAR(length_hint_doc, static PyObject * striter_reduce(striterobject *it, PyObject *Py_UNUSED(ignored)) { + PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); + + /* _PyEval_GetBuiltin can invoke arbitrary code, + * call must be before access of iterator pointers. + * see issue #101765 */ + if (it->it_seq != NULL) { - return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)), - it->it_seq, it->it_index); + return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } else { - return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter))); + return Py_BuildValue("N(())", iter); } } diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index 6bc25a0a409f97..a99fd74e4b621b 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -173,12 +173,10 @@ PyDoc_STRVAR(float_as_integer_ratio__doc__, "as_integer_ratio($self, /)\n" "--\n" "\n" -"Return integer ratio.\n" +"Return a pair of integers, whose ratio is exactly equal to the original float.\n" "\n" -"Return a pair of integers, whose ratio is exactly equal to the original float\n" -"and with a positive denominator.\n" -"\n" -"Raise OverflowError on infinities and a ValueError on NaNs.\n" +"The ratio is in lowest terms and has a positive denominator. Raise\n" +"OverflowError on infinities and a ValueError on NaNs.\n" "\n" ">>> (10.0).as_integer_ratio()\n" "(10, 1)\n" @@ -327,4 +325,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=74bc91bb44014df9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ea329577074911b9 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index 206bffdd086a5c..c26ceafbc2be0d 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -231,10 +231,9 @@ PyDoc_STRVAR(int_as_integer_ratio__doc__, "as_integer_ratio($self, /)\n" "--\n" "\n" -"Return integer ratio.\n" +"Return a pair of integers, whose ratio is equal to the original int.\n" "\n" -"Return a pair of integers, whose ratio is exactly equal to the original int\n" -"and with a positive denominator.\n" +"The ratio is in lowest terms and has a positive denominator.\n" "\n" ">>> (10).as_integer_ratio()\n" "(10, 1)\n" @@ -485,4 +484,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=e518fe2b5d519322 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cfdf35d916158d4f input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index a03b14edea8d4c..175bd57568f8f6 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -567,7 +567,8 @@ _PyCode_New(struct _PyCodeConstructor *con) ******************/ PyCodeObject * -PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, +PyUnstable_Code_NewWithPosOnlyArgs( + int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, @@ -691,7 +692,7 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, } PyCodeObject * -PyCode_New(int argcount, int kwonlyargcount, +PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, @@ -1371,7 +1372,7 @@ typedef struct { int -_PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) +PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) { if (!PyCode_Check(code)) { PyErr_BadInternalCall(); @@ -1392,7 +1393,7 @@ _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) int -_PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) +PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra) { PyInterpreterState *interp = _PyInterpreterState_GET(); diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 912b742f797d24..d641311f1126cd 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1546,12 +1546,10 @@ float_fromhex(PyTypeObject *type, PyObject *string) /*[clinic input] float.as_integer_ratio -Return integer ratio. +Return a pair of integers, whose ratio is exactly equal to the original float. -Return a pair of integers, whose ratio is exactly equal to the original float -and with a positive denominator. - -Raise OverflowError on infinities and a ValueError on NaNs. +The ratio is in lowest terms and has a positive denominator. Raise +OverflowError on infinities and a ValueError on NaNs. >>> (10.0).as_integer_ratio() (10, 1) @@ -1563,7 +1561,7 @@ Raise OverflowError on infinities and a ValueError on NaNs. static PyObject * float_as_integer_ratio_impl(PyObject *self) -/*[clinic end generated code: output=65f25f0d8d30a712 input=e21d08b4630c2e44]*/ +/*[clinic end generated code: output=65f25f0d8d30a712 input=d5ba7765655d75bd]*/ { double self_double; double float_part; diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 675fd496eefdaf..888cb16edd1b46 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -877,8 +877,17 @@ ga_iter_clear(PyObject *self) { static PyObject * ga_iter_reduce(PyObject *self, PyObject *Py_UNUSED(ignored)) { + PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); gaiterobject *gi = (gaiterobject *)self; - return Py_BuildValue("N(O)", _PyEval_GetBuiltin(&_Py_ID(iter)), gi->obj); + + /* _PyEval_GetBuiltin can invoke arbitrary code, + * call must be before access of iterator pointers. + * see issue #101765 */ + + if (gi->obj) + return Py_BuildValue("N(O)", iter, gi->obj); + else + return Py_BuildValue("N(())", iter); } static PyMethodDef ga_iter_methods[] = { diff --git a/Objects/genobject.c b/Objects/genobject.c index aec64ca7004e11..4ab6581e12ab3a 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -24,6 +24,21 @@ static const char *NON_INIT_CORO_MSG = "can't send non-None value to a " static const char *ASYNC_GEN_IGNORED_EXIT_MSG = "async generator ignored GeneratorExit"; +/* Returns a borrowed reference */ +static inline PyCodeObject * +_PyGen_GetCode(PyGenObject *gen) { + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); + return frame->f_code; +} + +PyCodeObject * +PyGen_GetCode(PyGenObject *gen) { + assert(PyGen_Check(gen)); + PyCodeObject *res = _PyGen_GetCode(gen); + Py_INCREF(res); + return res; +} + static inline int exc_state_traverse(_PyErr_StackItem *exc_state, visitproc visit, void *arg) { @@ -34,7 +49,6 @@ exc_state_traverse(_PyErr_StackItem *exc_state, visitproc visit, void *arg) static int gen_traverse(PyGenObject *gen, visitproc visit, void *arg) { - Py_VISIT(gen->gi_code); Py_VISIT(gen->gi_name); Py_VISIT(gen->gi_qualname); if (gen->gi_frame_state < FRAME_CLEARED) { @@ -88,8 +102,8 @@ _PyGen_Finalize(PyObject *self) /* If `gen` is a coroutine, and if it was never awaited on, issue a RuntimeWarning. */ - if (gen->gi_code != NULL && - ((PyCodeObject *)gen->gi_code)->co_flags & CO_COROUTINE && + assert(_PyGen_GetCode(gen) != NULL); + if (_PyGen_GetCode(gen)->co_flags & CO_COROUTINE && gen->gi_frame_state == FRAME_CREATED) { _PyErr_WarnUnawaitedCoroutine((PyObject *)gen); @@ -137,12 +151,12 @@ gen_dealloc(PyGenObject *gen) _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; gen->gi_frame_state = FRAME_CLEARED; frame->previous = NULL; - _PyFrame_Clear(frame); + _PyFrame_ClearExceptCode(frame); } - if (((PyCodeObject *)gen->gi_code)->co_flags & CO_COROUTINE) { + if (_PyGen_GetCode(gen)->co_flags & CO_COROUTINE) { Py_CLEAR(((PyCoroObject *)gen)->cr_origin_or_finalizer); } - Py_CLEAR(gen->gi_code); + Py_DECREF(_PyGen_GetCode(gen)); Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); _PyErr_ClearExcState(&gen->gi_exc_state); @@ -332,7 +346,7 @@ _PyGen_yf(PyGenObject *gen) /* Return immediately if the frame didn't start yet. SEND always come after LOAD_CONST: a code object should not start with SEND */ - assert(_PyCode_CODE(gen->gi_code)[0].op.code != SEND); + assert(_PyCode_CODE(_PyGen_GetCode(gen))[0].op.code != SEND); return NULL; } _Py_CODEUNIT next = frame->prev_instr[1]; @@ -767,6 +781,21 @@ gen_getframe(PyGenObject *gen, void *Py_UNUSED(ignored)) return _gen_getframe(gen, "gi_frame"); } +static PyObject * +_gen_getcode(PyGenObject *gen, const char *const name) +{ + if (PySys_Audit("object.__getattr__", "Os", gen, name) < 0) { + return NULL; + } + return Py_NewRef(_PyGen_GetCode(gen)); +} + +static PyObject * +gen_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) +{ + return _gen_getcode(gen, "gi_code"); +} + static PyGetSetDef gen_getsetlist[] = { {"__name__", (getter)gen_get_name, (setter)gen_set_name, PyDoc_STR("name of the generator")}, @@ -777,11 +806,11 @@ static PyGetSetDef gen_getsetlist[] = { {"gi_running", (getter)gen_getrunning, NULL, NULL}, {"gi_frame", (getter)gen_getframe, NULL, NULL}, {"gi_suspended", (getter)gen_getsuspended, NULL, NULL}, + {"gi_code", (getter)gen_getcode, NULL, NULL}, {NULL} /* Sentinel */ }; static PyMemberDef gen_memberlist[] = { - {"gi_code", T_OBJECT, offsetof(PyGenObject, gi_code), READONLY|PY_AUDIT_READ}, {NULL} /* Sentinel */ }; @@ -790,7 +819,7 @@ gen_sizeof(PyGenObject *gen, PyObject *Py_UNUSED(ignored)) { Py_ssize_t res; res = offsetof(PyGenObject, gi_iframe) + offsetof(_PyInterpreterFrame, localsplus); - PyCodeObject *code = gen->gi_code; + PyCodeObject *code = _PyGen_GetCode(gen); res += _PyFrame_NumSlotsForCodeObject(code) * sizeof(PyObject *); return PyLong_FromSsize_t(res); } @@ -878,7 +907,6 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) return NULL; } gen->gi_frame_state = FRAME_CLEARED; - gen->gi_code = (PyCodeObject *)Py_NewRef(func->func_code); gen->gi_weakreflist = NULL; gen->gi_exc_state.exc_value = NULL; gen->gi_exc_state.previous_item = NULL; @@ -960,8 +988,6 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, f->f_frame = frame; frame->owner = FRAME_OWNED_BY_GENERATOR; assert(PyObject_GC_IsTracked((PyObject *)f)); - gen->gi_code = PyFrame_GetCode(f); - Py_INCREF(gen->gi_code); Py_DECREF(f); gen->gi_weakreflist = NULL; gen->gi_exc_state.exc_value = NULL; @@ -969,11 +995,11 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, if (name != NULL) gen->gi_name = Py_NewRef(name); else - gen->gi_name = Py_NewRef(gen->gi_code->co_name); + gen->gi_name = Py_NewRef(_PyGen_GetCode(gen)->co_name); if (qualname != NULL) gen->gi_qualname = Py_NewRef(qualname); else - gen->gi_qualname = Py_NewRef(gen->gi_code->co_qualname); + gen->gi_qualname = Py_NewRef(_PyGen_GetCode(gen)->co_qualname); _PyObject_GC_TRACK(gen); return (PyObject *)gen; } @@ -1001,7 +1027,7 @@ static int gen_is_coroutine(PyObject *o) { if (PyGen_CheckExact(o)) { - PyCodeObject *code = (PyCodeObject *)((PyGenObject*)o)->gi_code; + PyCodeObject *code = _PyGen_GetCode((PyGenObject*)o); if (code->co_flags & CO_ITERABLE_COROUTINE) { return 1; } @@ -1110,6 +1136,12 @@ cr_getframe(PyCoroObject *coro, void *Py_UNUSED(ignored)) return _gen_getframe((PyGenObject *)coro, "cr_frame"); } +static PyObject * +cr_getcode(PyCoroObject *coro, void *Py_UNUSED(ignored)) +{ + return _gen_getcode((PyGenObject *)coro, "cr_code"); +} + static PyGetSetDef coro_getsetlist[] = { {"__name__", (getter)gen_get_name, (setter)gen_set_name, @@ -1120,12 +1152,12 @@ static PyGetSetDef coro_getsetlist[] = { PyDoc_STR("object being awaited on, or None")}, {"cr_running", (getter)cr_getrunning, NULL, NULL}, {"cr_frame", (getter)cr_getframe, NULL, NULL}, + {"cr_code", (getter)cr_getcode, NULL, NULL}, {"cr_suspended", (getter)cr_getsuspended, NULL, NULL}, {NULL} /* Sentinel */ }; static PyMemberDef coro_memberlist[] = { - {"cr_code", T_OBJECT, offsetof(PyCoroObject, cr_code), READONLY|PY_AUDIT_READ}, {"cr_origin", T_OBJECT, offsetof(PyCoroObject, cr_origin_or_finalizer), READONLY}, {NULL} /* Sentinel */ }; @@ -1514,6 +1546,12 @@ ag_getframe(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) return _gen_getframe((PyGenObject *)ag, "ag_frame"); } +static PyObject * +ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) +{ + return _gen_getcode(gen, "ag__code"); +} + static PyGetSetDef async_gen_getsetlist[] = { {"__name__", (getter)gen_get_name, (setter)gen_set_name, PyDoc_STR("name of the async generator")}, @@ -1522,13 +1560,13 @@ static PyGetSetDef async_gen_getsetlist[] = { {"ag_await", (getter)coro_get_cr_await, NULL, PyDoc_STR("object being awaited on, or None")}, {"ag_frame", (getter)ag_getframe, NULL, NULL}, + {"ag_code", (getter)ag_getcode, NULL, NULL}, {NULL} /* Sentinel */ }; static PyMemberDef async_gen_memberlist[] = { {"ag_running", T_BOOL, offsetof(PyAsyncGenObject, ag_running_async), READONLY}, - {"ag_code", T_OBJECT, offsetof(PyAsyncGenObject, ag_code), READONLY|PY_AUDIT_READ}, {NULL} /* Sentinel */ }; diff --git a/Objects/iterobject.c b/Objects/iterobject.c index cfd6d0a7c959c9..149b701b68e91a 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -102,11 +102,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list( static PyObject * iter_reduce(seqiterobject *it, PyObject *Py_UNUSED(ignored)) { + PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); + + /* _PyEval_GetBuiltin can invoke arbitrary code, + * call must be before access of iterator pointers. + * see issue #101765 */ + if (it->it_seq != NULL) - return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)), - it->it_seq, it->it_index); + return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); else - return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter))); + return Py_BuildValue("N(())", iter); } PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); @@ -239,11 +244,16 @@ calliter_iternext(calliterobject *it) static PyObject * calliter_reduce(calliterobject *it, PyObject *Py_UNUSED(ignored)) { + PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); + + /* _PyEval_GetBuiltin can invoke arbitrary code, + * call must be before access of iterator pointers. + * see issue #101765 */ + if (it->it_callable != NULL && it->it_sentinel != NULL) - return Py_BuildValue("N(OO)", _PyEval_GetBuiltin(&_Py_ID(iter)), - it->it_callable, it->it_sentinel); + return Py_BuildValue("N(OO)", iter, it->it_callable, it->it_sentinel); else - return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter))); + return Py_BuildValue("N(())", iter); } static PyMethodDef calliter_methods[] = { diff --git a/Objects/listobject.c b/Objects/listobject.c index ca6b712311340a..1a210e77d55c29 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3444,19 +3444,31 @@ listiter_reduce_general(void *_it, int forward) { PyObject *list; + /* _PyEval_GetBuiltin can invoke arbitrary code, + * call must be before access of iterator pointers. + * see issue #101765 */ + /* the objects are not the same, index is of different types! */ if (forward) { + PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); + if (!iter) { + return NULL; + } _PyListIterObject *it = (_PyListIterObject *)_it; if (it->it_seq) { - return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)), - it->it_seq, it->it_index); + return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } + Py_DECREF(iter); } else { + PyObject *reversed = _PyEval_GetBuiltin(&_Py_ID(reversed)); + if (!reversed) { + return NULL; + } listreviterobject *it = (listreviterobject *)_it; if (it->it_seq) { - return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(reversed)), - it->it_seq, it->it_index); + return Py_BuildValue("N(O)n", reversed, it->it_seq, it->it_index); } + Py_DECREF(reversed); } /* empty iterator, create an empty list */ list = PyList_New(0); diff --git a/Objects/longobject.c b/Objects/longobject.c index 8293f133bed213..51655cd0bad9ec 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6020,10 +6020,9 @@ int_bit_count_impl(PyObject *self) /*[clinic input] int.as_integer_ratio -Return integer ratio. +Return a pair of integers, whose ratio is equal to the original int. -Return a pair of integers, whose ratio is exactly equal to the original int -and with a positive denominator. +The ratio is in lowest terms and has a positive denominator. >>> (10).as_integer_ratio() (10, 1) @@ -6035,7 +6034,7 @@ and with a positive denominator. static PyObject * int_as_integer_ratio_impl(PyObject *self) -/*[clinic end generated code: output=e60803ae1cc8621a input=55ce3058e15de393]*/ +/*[clinic end generated code: output=e60803ae1cc8621a input=384ff1766634bec2]*/ { PyObject *ratio_tuple; PyObject *numerator = long_long(self); diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 7d6d0e1bea249e..6ee93ab5adc281 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -1048,11 +1048,16 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list( static PyObject * tupleiter_reduce(_PyTupleIterObject *it, PyObject *Py_UNUSED(ignored)) { + PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); + + /* _PyEval_GetBuiltin can invoke arbitrary code, + * call must be before access of iterator pointers. + * see issue #101765 */ + if (it->it_seq) - return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)), - it->it_seq, it->it_index); + return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); else - return Py_BuildValue("N(())", _PyEval_GetBuiltin(&_Py_ID(iter))); + return Py_BuildValue("N(())", iter); } static PyObject * diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2d1220a0695036..b3f1429debc58b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8564,7 +8564,7 @@ an all-zero entry. #NAME "($self, /)\n--\n\n" DOC) #define IBSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ ETSLOT(NAME, as_number.SLOT, FUNCTION, WRAPPER, \ - #NAME "($self, value, /)\n--\n\nCompute self " DOC " value.") + #NAME "($self, value, /)\n--\n\nReturn self" DOC "value.") #define BINSLOT(NAME, SLOT, FUNCTION, DOC) \ ETSLOT(NAME, as_number.SLOT, FUNCTION, wrap_binaryfunc_l, \ #NAME "($self, value, /)\n--\n\nReturn self" DOC "value.") diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index f0c7aa7707fdb5..1ba30421c66dba 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -14784,14 +14784,21 @@ PyDoc_STRVAR(length_hint_doc, "Private method returning an estimate of len(list( static PyObject * unicodeiter_reduce(unicodeiterobject *it, PyObject *Py_UNUSED(ignored)) { + PyObject *iter = _PyEval_GetBuiltin(&_Py_ID(iter)); + + /* _PyEval_GetBuiltin can invoke arbitrary code, + * call must be before access of iterator pointers. + * see issue #101765 */ + if (it->it_seq != NULL) { - return Py_BuildValue("N(O)n", _PyEval_GetBuiltin(&_Py_ID(iter)), - it->it_seq, it->it_index); + return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); } else { PyObject *u = unicode_new_empty(); - if (u == NULL) + if (u == NULL) { + Py_XDECREF(iter); return NULL; - return Py_BuildValue("N(N)", _PyEval_GetBuiltin(&_Py_ID(iter)), u); + } + return Py_BuildValue("N(N)", iter, u); } } diff --git a/PC/pyconfig.h b/PC/pyconfig.h index f5166a1506c945..a34d420ab7ecaa 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -72,6 +72,9 @@ WIN32 is still required for the locale module. #define USE_SOCKET #endif +#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP) +#define MS_WINDOWS_NON_DESKTOP +#endif /* Compiler specific defines */ diff --git a/PC/readme.txt b/PC/readme.txt index bef5111c591825..a917a72c1232cd 100644 --- a/PC/readme.txt +++ b/PC/readme.txt @@ -60,11 +60,6 @@ python_nt.rc Resource compiler input for python15.dll. dl_nt.c Additional sources used for 32-bit Windows features. -getpathp.c Default sys.path calculations (for all PC platforms). - -dllbase_nt.txt A (manually maintained) list of base addresses for - various DLLs, to avoid run-time relocation. - Note for Windows 3.x and DOS users ================================== diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 58bf4e1eacbf21..4cc184bfc1ac82 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -107,6 +107,8 @@ + + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 101c5322761634..fbdaf04ce37cb1 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -51,6 +51,12 @@ Source Files + + Source Files + + + Source Files + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 222963bc42d17c..85dc8caa458ed9 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -400,12 +400,14 @@ + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index efb96222043ac2..98e7d59ba1020c 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -848,6 +848,9 @@ Modules + + Modules + Modules @@ -863,6 +866,9 @@ Modules + + Modules + Modules diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index db0e597b7a5aa4..b44e303ac2594b 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1484,9 +1484,7 @@ def generate_ast_fini(module_state, f): for s in module_state: f.write(" Py_CLEAR(state->" + s + ');\n') f.write(textwrap.dedent(""" - if (_PyInterpreterState_Get() == _PyInterpreterState_Main()) { - Py_CLEAR(_Py_CACHED_OBJECT(str_replace_inf)); - } + Py_CLEAR(_Py_INTERP_CACHED_OBJECT(interp, str_replace_inf)); #if !defined(NDEBUG) state->initialized = -1; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index d113c47b95392e..6c878474afb192 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -263,9 +263,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->vararg); Py_CLEAR(state->withitem_type); - if (_PyInterpreterState_Get() == _PyInterpreterState_Main()) { - Py_CLEAR(_Py_CACHED_OBJECT(str_replace_inf)); - } + Py_CLEAR(_Py_INTERP_CACHED_OBJECT(interp, str_replace_inf)); #if !defined(NDEBUG) state->initialized = -1; diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index 79b2e2f15ba243..8aff045101cc72 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_ast.h" // expr_ty +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_runtime.h" // _Py_ID() #include // DBL_MAX_10_EXP #include @@ -13,7 +14,10 @@ _Py_DECLARE_STR(open_br, "{"); _Py_DECLARE_STR(dbl_open_br, "{{"); _Py_DECLARE_STR(close_br, "}"); _Py_DECLARE_STR(dbl_close_br, "}}"); -#define _str_replace_inf _Py_CACHED_OBJECT(str_replace_inf) + +/* We would statically initialize this if doing so were simple enough. */ +#define _str_replace_inf(interp) \ + _Py_INTERP_CACHED_OBJECT(interp, str_replace_inf) /* Forward declarations for recursion via helper functions. */ static PyObject * @@ -78,10 +82,11 @@ append_repr(_PyUnicodeWriter *writer, PyObject *obj) if ((PyFloat_CheckExact(obj) && Py_IS_INFINITY(PyFloat_AS_DOUBLE(obj))) || PyComplex_CheckExact(obj)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyObject *new_repr = PyUnicode_Replace( repr, &_Py_ID(inf), - _str_replace_inf, + _str_replace_inf(interp), -1 ); Py_DECREF(repr); @@ -916,9 +921,13 @@ append_ast_expr(_PyUnicodeWriter *writer, expr_ty e, int level) static int maybe_init_static_strings(void) { - if (!_str_replace_inf && - !(_str_replace_inf = PyUnicode_FromFormat("1e%d", 1 + DBL_MAX_10_EXP))) { - return -1; + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_str_replace_inf(interp) == NULL) { + PyObject *tmp = PyUnicode_FromFormat("1e%d", 1 + DBL_MAX_10_EXP); + if (tmp == NULL) { + return -1; + } + _str_replace_inf(interp) = tmp; } return 0; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f15d5ea0c77bf7..dcd0d0491b1365 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -17,7 +17,7 @@ #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES -#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -1760,9 +1760,7 @@ dummy_func( int offset = next_instr[1].op.arg; int err = PyObject_IsTrue(cond); Py_DECREF(cond); - if (err < 0) { - goto error; - } + ERROR_IF(err < 0, error); if (jump_on_true == (err != 0)) { JUMPBY(offset); } @@ -2184,35 +2182,27 @@ dummy_func( // Common case: no jump, leave it to the code generator } - // This is slightly different, when the loop isn't terminated we - // jump over the immediately following STORE_FAST instruction. - inst(FOR_ITER_RANGE, (unused/1, iter -- iter, unused)) { + inst(FOR_ITER_RANGE, (unused/1, iter -- iter, next)) { assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); - _Py_CODEUNIT next = next_instr[INLINE_CACHE_ENTRIES_FOR_ITER]; - assert(_PyOpcode_Deopt[next.op.code] == STORE_FAST); if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); // Jump over END_FOR instruction. JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); + DISPATCH(); } - else { - long value = r->start; - r->start = value + r->step; - r->len--; - if (_PyLong_AssignValue(&GETLOCAL(next.op.arg), value) < 0) { - goto error; - } - // The STORE_FAST is already done. - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + 1); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) { + goto error; } - DISPATCH(); } - // This is *not* a super-instruction, unique in the family. inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) { assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)iter; diff --git a/Python/ceval.c b/Python/ceval.c index b85231a1df8a95..b422d0ed34ede3 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -13,7 +13,7 @@ #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_opcode.h" // EXTRA_CASES -#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pyerrors.h" // _PyErr_Fetch(), _PyErr_GetRaisedException() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -105,8 +105,7 @@ static void dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) { PyObject **stack_base = _PyFrame_Stackbase(frame); - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); + PyObject *exc = PyErr_GetRaisedException(); printf(" stack=["); for (PyObject **ptr = stack_base; ptr < stack_pointer; ptr++) { if (ptr != stack_base) { @@ -120,7 +119,7 @@ dump_stack(_PyInterpreterFrame *frame, PyObject **stack_pointer) } printf("]\n"); fflush(stdout); - PyErr_Restore(type, value, traceback); + PyErr_SetRaisedException(exc); } static void @@ -157,8 +156,7 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) return; } PyFunctionObject *f = (PyFunctionObject *)fobj; - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); + PyObject *exc = PyErr_GetRaisedException(); PyObject *name = f->func_qualname; if (name == NULL) { name = f->func_name; @@ -178,7 +176,7 @@ lltrace_resume_frame(_PyInterpreterFrame *frame) } printf("\n"); fflush(stdout); - PyErr_Restore(type, value, traceback); + PyErr_SetRaisedException(exc); } #endif static int call_trace(Py_tracefunc, PyObject *, @@ -1032,7 +1030,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int PyObject *v = POP(); Py_XDECREF(v); } - PyObject *exc, *val, *tb; if (lasti) { int frame_lasti = _PyInterpreterFrame_LASTI(frame); PyObject *lasti = PyLong_FromLong(frame_lasti); @@ -1041,19 +1038,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int } PUSH(lasti); } - _PyErr_Fetch(tstate, &exc, &val, &tb); + /* Make the raw exception data available to the handler, so a program can emulate the Python main loop. */ - _PyErr_NormalizeException(tstate, &exc, &val, &tb); - if (tb != NULL) - PyException_SetTraceback(val, tb); - else - PyException_SetTraceback(val, Py_None); - Py_XDECREF(tb); - Py_XDECREF(exc); - PUSH(val); + PUSH(_PyErr_GetRaisedException(tstate)); JUMPTO(handler); /* Resume normal execution */ DISPATCH(); @@ -1604,41 +1594,6 @@ initialize_locals(PyThreadState *tstate, PyFunctionObject *func, return -1; } -/* Consumes references to func, locals and all the args */ -static _PyInterpreterFrame * -_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, PyObject* const* args, - size_t argcount, PyObject *kwnames) -{ - PyCodeObject * code = (PyCodeObject *)func->func_code; - CALL_STAT_INC(frames_pushed); - _PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize); - if (frame == NULL) { - goto fail; - } - _PyFrame_Initialize(frame, func, locals, code, 0); - PyObject **localsarray = &frame->localsplus[0]; - if (initialize_locals(tstate, func, localsarray, args, argcount, kwnames)) { - assert(frame->owner != FRAME_OWNED_BY_GENERATOR); - _PyEvalFrameClearAndPop(tstate, frame); - return NULL; - } - return frame; -fail: - /* Consume the references */ - for (size_t i = 0; i < argcount; i++) { - Py_DECREF(args[i]); - } - if (kwnames) { - Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); - for (Py_ssize_t i = 0; i < kwcount; i++) { - Py_DECREF(args[i+argcount]); - } - } - PyErr_NoMemory(); - return NULL; -} - static void clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { @@ -1649,7 +1604,8 @@ clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) tstate->datastack_top); tstate->c_recursion_remaining--; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); - _PyFrame_Clear(frame); + _PyFrame_ClearExceptCode(frame); + Py_DECREF(frame->f_code); tstate->c_recursion_remaining++; _PyThreadState_PopFrame(tstate, frame); } @@ -1665,7 +1621,7 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) gen->gi_exc_state.previous_item = NULL; tstate->c_recursion_remaining--; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); - _PyFrame_Clear(frame); + _PyFrame_ClearExceptCode(frame); tstate->c_recursion_remaining++; frame->previous = NULL; } @@ -1681,6 +1637,39 @@ _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) } } +/* Consumes references to func, locals and all the args */ +static _PyInterpreterFrame * +_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, PyObject* const* args, + size_t argcount, PyObject *kwnames) +{ + PyCodeObject * code = (PyCodeObject *)func->func_code; + CALL_STAT_INC(frames_pushed); + _PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize); + if (frame == NULL) { + goto fail; + } + _PyFrame_Initialize(frame, func, locals, code, 0); + if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) { + assert(frame->owner == FRAME_OWNED_BY_THREAD); + clear_thread_frame(tstate, frame); + return NULL; + } + return frame; +fail: + /* Consume the references */ + for (size_t i = 0; i < argcount; i++) { + Py_DECREF(args[i]); + } + if (kwnames) { + Py_ssize_t kwcount = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t i = 0; i < kwcount; i++) { + Py_DECREF(args[i+argcount]); + } + } + PyErr_NoMemory(); + return NULL; +} PyObject * _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, @@ -2076,19 +2065,15 @@ call_trace_protected(Py_tracefunc func, PyObject *obj, PyThreadState *tstate, _PyInterpreterFrame *frame, int what, PyObject *arg) { - PyObject *type, *value, *traceback; - int err; - _PyErr_Fetch(tstate, &type, &value, &traceback); - err = call_trace(func, obj, tstate, frame, what, arg); + PyObject *exc = _PyErr_GetRaisedException(tstate); + int err = call_trace(func, obj, tstate, frame, what, arg); if (err == 0) { - _PyErr_Restore(tstate, type, value, traceback); + _PyErr_SetRaisedException(tstate, exc); return 0; } else { - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(traceback); + Py_XDECREF(exc); return -1; } } @@ -2936,18 +2921,15 @@ format_exc_check_arg(PyThreadState *tstate, PyObject *exc, if (exc == PyExc_NameError) { // Include the name in the NameError exceptions to offer suggestions later. - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - PyErr_NormalizeException(&type, &value, &traceback); - if (PyErr_GivenExceptionMatches(value, PyExc_NameError)) { - PyNameErrorObject* exc = (PyNameErrorObject*) value; - if (exc->name == NULL) { + PyObject *exc = PyErr_GetRaisedException(); + if (PyErr_GivenExceptionMatches(exc, PyExc_NameError)) { + if (((PyNameErrorObject*)exc)->name == NULL) { // We do not care if this fails because we are going to restore the // NameError anyway. - (void)PyObject_SetAttr(value, &_Py_ID(name), obj); + (void)PyObject_SetAttr(exc, &_Py_ID(name), obj); } } - PyErr_Restore(type, value, traceback); + PyErr_SetRaisedException(exc); } } @@ -2989,7 +2971,7 @@ format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int oparg) Py_ssize_t -_PyEval_RequestCodeExtraIndex(freefunc free) +PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) { PyInterpreterState *interp = _PyInterpreterState_GET(); Py_ssize_t new_index; @@ -3068,15 +3050,11 @@ maybe_dtrace_line(_PyInterpreterFrame *frame, /* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions for the limited API. */ -#undef Py_EnterRecursiveCall - int Py_EnterRecursiveCall(const char *where) { return _Py_EnterRecursiveCall(where); } -#undef Py_LeaveRecursiveCall - void Py_LeaveRecursiveCall(void) { _Py_LeaveRecursiveCall(); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 1bf223348d28fa..749d8144bf7a23 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -2,7 +2,7 @@ #include "Python.h" #include "pycore_atomic.h" // _Py_atomic_int #include "pycore_ceval.h" // _PyEval_SignalReceived() -#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pylifecycle.h" // _PyErr_Print() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // _Py_RunGC() @@ -870,10 +870,9 @@ _Py_FinishPendingCalls(PyThreadState *tstate) } if (make_pending_calls(tstate->interp) < 0) { - PyObject *exc, *val, *tb; - _PyErr_Fetch(tstate, &exc, &val, &tb); + PyObject *exc = _PyErr_GetRaisedException(tstate); PyErr_BadInternalCall(); - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); _PyErr_Print(tstate); } } diff --git a/Python/compile.c b/Python/compile.c index 16ff29ff33f173..78145d41973f18 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1637,8 +1637,7 @@ static void compiler_exit_scope(struct compiler *c) { // Don't call PySequence_DelItem() with an exception raised - PyObject *exc_type, *exc_val, *exc_tb; - PyErr_Fetch(&exc_type, &exc_val, &exc_tb); + PyObject *exc = PyErr_GetRaisedException(); c->c_nestlevel--; compiler_unit_free(c->u); @@ -1659,7 +1658,7 @@ compiler_exit_scope(struct compiler *c) c->u = NULL; } - PyErr_Restore(exc_type, exc_val, exc_tb); + PyErr_SetRaisedException(exc); } /* Search if variable annotations are present statically in a block. */ @@ -9892,56 +9891,77 @@ instructions_to_cfg(PyObject *instructions, cfg_builder *g) { assert(PyList_Check(instructions)); - Py_ssize_t instr_size = PyList_GET_SIZE(instructions); - for (Py_ssize_t i = 0; i < instr_size; i++) { + Py_ssize_t num_insts = PyList_GET_SIZE(instructions); + bool *is_target = PyMem_Calloc(num_insts, sizeof(bool)); + if (is_target == NULL) { + return ERROR; + } + for (Py_ssize_t i = 0; i < num_insts; i++) { PyObject *item = PyList_GET_ITEM(instructions, i); - if (PyLong_Check(item)) { - int lbl_id = PyLong_AsLong(item); + if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { + PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); + goto error; + } + int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); + if (PyErr_Occurred()) { + goto error; + } + if (HAS_TARGET(opcode)) { + int oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); if (PyErr_Occurred()) { - return ERROR; + goto error; } - if (lbl_id <= 0 || lbl_id > instr_size) { - /* expect label in a reasonable range */ + if (oparg < 0 || oparg >= num_insts) { PyErr_SetString(PyExc_ValueError, "label out of range"); - return ERROR; + goto error; } - jump_target_label lbl = {lbl_id}; + is_target[oparg] = true; + } + } + + for (int i = 0; i < num_insts; i++) { + if (is_target[i]) { + jump_target_label lbl = {i}; RETURN_IF_ERROR(cfg_builder_use_label(g, lbl)); } - else { - if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { - PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); - return ERROR; - } - int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); - if (PyErr_Occurred()) { - return ERROR; - } - int oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); - if (PyErr_Occurred()) { - return ERROR; - } - location loc; - loc.lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 2)); - if (PyErr_Occurred()) { - return ERROR; - } - loc.end_lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 3)); - if (PyErr_Occurred()) { - return ERROR; - } - loc.col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 4)); - if (PyErr_Occurred()) { - return ERROR; - } - loc.end_col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 5)); - if (PyErr_Occurred()) { - return ERROR; - } - RETURN_IF_ERROR(cfg_builder_addop(g, opcode, oparg, loc)); + PyObject *item = PyList_GET_ITEM(instructions, i); + if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 6) { + PyErr_SetString(PyExc_ValueError, "expected a 6-tuple"); + goto error; + } + int opcode = PyLong_AsLong(PyTuple_GET_ITEM(item, 0)); + if (PyErr_Occurred()) { + goto error; + } + int oparg = PyLong_AsLong(PyTuple_GET_ITEM(item, 1)); + if (PyErr_Occurred()) { + goto error; + } + location loc; + loc.lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 2)); + if (PyErr_Occurred()) { + goto error; } + loc.end_lineno = PyLong_AsLong(PyTuple_GET_ITEM(item, 3)); + if (PyErr_Occurred()) { + goto error; + } + loc.col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 4)); + if (PyErr_Occurred()) { + goto error; + } + loc.end_col_offset = PyLong_AsLong(PyTuple_GET_ITEM(item, 5)); + if (PyErr_Occurred()) { + goto error; + } + RETURN_IF_ERROR(cfg_builder_addop(g, opcode, oparg, loc)); } + + PyMem_Free(is_target); return SUCCESS; +error: + PyMem_Free(is_target); + return ERROR; } static PyObject * @@ -9951,20 +9971,12 @@ cfg_to_instructions(cfg_builder *g) if (instructions == NULL) { return NULL; } - int lbl = 1; + int lbl = 0; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - b->b_label = lbl++; + b->b_label = lbl; + lbl += b->b_iused; } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - PyObject *lbl = PyLong_FromLong(b->b_label); - if (lbl == NULL) { - goto error; - } - if (PyList_Append(instructions, lbl) != 0) { - Py_DECREF(lbl); - goto error; - } - Py_DECREF(lbl); for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; location loc = instr->i_loc; diff --git a/Python/dtoa.c b/Python/dtoa.c index cff5f1b0658eae..6ea60ac9946e0f 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -119,7 +119,7 @@ #include "Python.h" #include "pycore_dtoa.h" // _PY_SHORT_FLOAT_REPR -#include "pycore_runtime.h" // _PyRuntime +#include "pycore_pystate.h" // _PyInterpreterState_GET() #include // exit() /* if _PY_SHORT_FLOAT_REPR == 0, then don't even try to compile @@ -339,9 +339,9 @@ typedef struct Bigint Bigint; Bfree to PyMem_Free. Investigate whether this has any significant performance on impact. */ -#define freelist _PyRuntime.dtoa.freelist -#define private_mem _PyRuntime.dtoa.preallocated -#define pmem_next _PyRuntime.dtoa.preallocated_next +#define freelist interp->dtoa.freelist +#define private_mem interp->dtoa.preallocated +#define pmem_next interp->dtoa.preallocated_next /* Allocate space for a Bigint with up to 1<next; @@ -385,6 +386,7 @@ Bfree(Bigint *v) if (v->k > Bigint_Kmax) FREE((void*)v); else { + PyInterpreterState *interp = _PyInterpreterState_GET(); v->next = freelist[v->k]; freelist[v->k] = v; } @@ -692,7 +694,8 @@ pow5mult(Bigint *b, int k) if (!(k >>= 2)) return b; - p5 = _PyRuntime.dtoa.p5s; + PyInterpreterState *interp = _PyInterpreterState_GET(); + p5 = interp->dtoa.p5s; if (!p5) { /* first time */ p5 = i2b(625); @@ -700,7 +703,7 @@ pow5mult(Bigint *b, int k) Bfree(b); return NULL; } - _PyRuntime.dtoa.p5s = p5; + interp->dtoa.p5s = p5; p5->next = 0; } for(;;) { diff --git a/Python/frame.c b/Python/frame.c index 6a287d4724051a..c2c0be30113912 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -29,17 +29,14 @@ PyFrameObject * _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame) { assert(frame->frame_obj == NULL); - PyObject *error_type, *error_value, *error_traceback; - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); PyFrameObject *f = _PyFrame_New_NoTrack(frame->f_code); if (f == NULL) { - Py_XDECREF(error_type); - Py_XDECREF(error_value); - Py_XDECREF(error_traceback); + Py_XDECREF(exc); return NULL; } - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); if (frame->frame_obj) { // GH-97002: How did we get into this horrible situation? Most likely, // allocating f triggered a GC collection, which ran some code that @@ -84,6 +81,7 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); assert(frame->owner != FRAME_CLEARED); Py_ssize_t size = ((char*)&frame->localsplus[frame->stacktop]) - (char *)frame; + Py_INCREF(frame->f_code); memcpy((_PyInterpreterFrame *)f->_f_frame_data, frame, size); frame = (_PyInterpreterFrame *)f->_f_frame_data; f->f_frame = frame; @@ -118,7 +116,7 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) } void -_PyFrame_Clear(_PyInterpreterFrame *frame) +_PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) { /* It is the responsibility of the owning generator/coroutine * to have cleared the enclosing generator, if any. */ @@ -144,7 +142,6 @@ _PyFrame_Clear(_PyInterpreterFrame *frame) Py_XDECREF(frame->frame_obj); Py_XDECREF(frame->f_locals); Py_DECREF(frame->f_funcobj); - Py_DECREF(frame->f_code); } int diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5b7ab37a9a5cc7..d155a42fa3cf51 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -22,7 +22,7 @@ if (value == NULL) goto unbound_local_error; Py_INCREF(value); STACK_GROW(1); - POKE(1, value); + stack_pointer[-1] = value; DISPATCH(); } @@ -32,7 +32,7 @@ if (value == NULL) goto unbound_local_error; Py_INCREF(value); STACK_GROW(1); - POKE(1, value); + stack_pointer[-1] = value; DISPATCH(); } @@ -42,7 +42,7 @@ assert(value != NULL); Py_INCREF(value); STACK_GROW(1); - POKE(1, value); + stack_pointer[-1] = value; DISPATCH(); } @@ -52,7 +52,7 @@ // do not use SETLOCAL here, it decrefs the old value GETLOCAL(oparg) = NULL; STACK_GROW(1); - POKE(1, value); + stack_pointer[-1] = value; DISPATCH(); } @@ -62,12 +62,12 @@ value = GETITEM(consts, oparg); Py_INCREF(value); STACK_GROW(1); - POKE(1, value); + stack_pointer[-1] = value; DISPATCH(); } TARGET(STORE_FAST) { - PyObject *value = PEEK(1); + PyObject *value = stack_pointer[-1]; SETLOCAL(oparg, value); STACK_SHRINK(1); DISPATCH(); @@ -83,8 +83,7 @@ Py_INCREF(value); _tmp_2 = value; } - NEXTOPARG(); - JUMPBY(1); + oparg = (next_instr++)->op.arg; { PyObject *value; value = GETLOCAL(oparg); @@ -93,8 +92,8 @@ _tmp_1 = value; } STACK_GROW(2); - POKE(1, _tmp_1); - POKE(2, _tmp_2); + stack_pointer[-1] = _tmp_1; + stack_pointer[-2] = _tmp_2; DISPATCH(); } @@ -108,8 +107,7 @@ Py_INCREF(value); _tmp_2 = value; } - NEXTOPARG(); - JUMPBY(1); + oparg = (next_instr++)->op.arg; { PyObject *value; value = GETITEM(consts, oparg); @@ -117,19 +115,18 @@ _tmp_1 = value; } STACK_GROW(2); - POKE(1, _tmp_1); - POKE(2, _tmp_2); + stack_pointer[-1] = _tmp_1; + stack_pointer[-2] = _tmp_2; DISPATCH(); } TARGET(STORE_FAST__LOAD_FAST) { - PyObject *_tmp_1 = PEEK(1); + PyObject *_tmp_1 = stack_pointer[-1]; { PyObject *value = _tmp_1; SETLOCAL(oparg, value); } - NEXTOPARG(); - JUMPBY(1); + oparg = (next_instr++)->op.arg; { PyObject *value; value = GETLOCAL(oparg); @@ -137,19 +134,18 @@ Py_INCREF(value); _tmp_1 = value; } - POKE(1, _tmp_1); + stack_pointer[-1] = _tmp_1; DISPATCH(); } TARGET(STORE_FAST__STORE_FAST) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); + PyObject *_tmp_1 = stack_pointer[-1]; + PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; SETLOCAL(oparg, value); } - NEXTOPARG(); - JUMPBY(1); + oparg = (next_instr++)->op.arg; { PyObject *value = _tmp_2; SETLOCAL(oparg, value); @@ -167,8 +163,7 @@ Py_INCREF(value); _tmp_2 = value; } - NEXTOPARG(); - JUMPBY(1); + oparg = (next_instr++)->op.arg; { PyObject *value; value = GETLOCAL(oparg); @@ -177,13 +172,13 @@ _tmp_1 = value; } STACK_GROW(2); - POKE(1, _tmp_1); - POKE(2, _tmp_2); + stack_pointer[-1] = _tmp_1; + stack_pointer[-2] = _tmp_2; DISPATCH(); } TARGET(POP_TOP) { - PyObject *value = PEEK(1); + PyObject *value = stack_pointer[-1]; Py_DECREF(value); STACK_SHRINK(1); DISPATCH(); @@ -193,13 +188,13 @@ PyObject *res; res = NULL; STACK_GROW(1); - POKE(1, res); + stack_pointer[-1] = res; DISPATCH(); } TARGET(END_FOR) { - PyObject *_tmp_1 = PEEK(1); - PyObject *_tmp_2 = PEEK(2); + PyObject *_tmp_1 = stack_pointer[-1]; + PyObject *_tmp_2 = stack_pointer[-2]; { PyObject *value = _tmp_1; Py_DECREF(value); @@ -213,17 +208,17 @@ } TARGET(UNARY_NEGATIVE) { - PyObject *value = PEEK(1); + PyObject *value = stack_pointer[-1]; PyObject *res; res = PyNumber_Negative(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; - POKE(1, res); + stack_pointer[-1] = res; DISPATCH(); } TARGET(UNARY_NOT) { - PyObject *value = PEEK(1); + PyObject *value = stack_pointer[-1]; PyObject *res; int err = PyObject_IsTrue(value); Py_DECREF(value); @@ -235,23 +230,23 @@ res = Py_False; } Py_INCREF(res); - POKE(1, res); + stack_pointer[-1] = res; DISPATCH(); } TARGET(UNARY_INVERT) { - PyObject *value = PEEK(1); + PyObject *value = stack_pointer[-1]; PyObject *res; res = PyNumber_Invert(value); Py_DECREF(value); if (res == NULL) goto pop_1_error; - POKE(1, res); + stack_pointer[-1] = res; DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_INT) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *prod; assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); @@ -262,14 +257,14 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (prod == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, prod); - JUMPBY(1); + stack_pointer[-1] = prod; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *prod; assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); @@ -282,14 +277,14 @@ _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); if (prod == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, prod); - JUMPBY(1); + stack_pointer[-1] = prod; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_INT) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *sub; assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); @@ -300,14 +295,14 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sub == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, sub); - JUMPBY(1); + stack_pointer[-1] = sub; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *sub; assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); @@ -319,14 +314,14 @@ _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); if (sub == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, sub); - JUMPBY(1); + stack_pointer[-1] = sub; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_UNICODE) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *res; assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); @@ -337,14 +332,14 @@ _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (res == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, res); - JUMPBY(1); + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP); @@ -377,8 +372,8 @@ } TARGET(BINARY_OP_ADD_FLOAT) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *sum; assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); @@ -391,14 +386,14 @@ _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); if (sum == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, sum); - JUMPBY(1); + stack_pointer[-1] = sum; + next_instr += 1; DISPATCH(); } TARGET(BINARY_OP_ADD_INT) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *sum; assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); @@ -409,16 +404,16 @@ _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); if (sum == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, sum); - JUMPBY(1); + stack_pointer[-1] = sum; + next_instr += 1; DISPATCH(); } TARGET(BINARY_SUBSCR) { PREDICTED(BINARY_SUBSCR); static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 4, "incorrect cache size"); - PyObject *sub = PEEK(1); - PyObject *container = PEEK(2); + PyObject *sub = stack_pointer[-1]; + PyObject *container = stack_pointer[-2]; PyObject *res; #if ENABLE_SPECIALIZATION _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; @@ -436,15 +431,15 @@ Py_DECREF(sub); if (res == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } TARGET(BINARY_SLICE) { - PyObject *stop = PEEK(1); - PyObject *start = PEEK(2); - PyObject *container = PEEK(3); + PyObject *stop = stack_pointer[-1]; + PyObject *start = stack_pointer[-2]; + PyObject *container = stack_pointer[-3]; PyObject *res; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't @@ -459,15 +454,15 @@ Py_DECREF(container); if (res == NULL) goto pop_3_error; STACK_SHRINK(2); - POKE(1, res); + stack_pointer[-1] = res; DISPATCH(); } TARGET(STORE_SLICE) { - PyObject *stop = PEEK(1); - PyObject *start = PEEK(2); - PyObject *container = PEEK(3); - PyObject *v = PEEK(4); + PyObject *stop = stack_pointer[-1]; + PyObject *start = stack_pointer[-2]; + PyObject *container = stack_pointer[-3]; + PyObject *v = stack_pointer[-4]; PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); int err; if (slice == NULL) { @@ -485,8 +480,8 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - PyObject *sub = PEEK(1); - PyObject *list = PEEK(2); + PyObject *sub = stack_pointer[-1]; + PyObject *list = stack_pointer[-2]; PyObject *res; assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -504,14 +499,14 @@ _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } TARGET(BINARY_SUBSCR_TUPLE_INT) { - PyObject *sub = PEEK(1); - PyObject *tuple = PEEK(2); + PyObject *sub = stack_pointer[-1]; + PyObject *tuple = stack_pointer[-2]; PyObject *res; assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); @@ -529,14 +524,14 @@ _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(tuple); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } TARGET(BINARY_SUBSCR_DICT) { - PyObject *sub = PEEK(1); - PyObject *dict = PEEK(2); + PyObject *sub = stack_pointer[-1]; + PyObject *dict = stack_pointer[-2]; PyObject *res; assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); @@ -554,14 +549,14 @@ Py_DECREF(dict); Py_DECREF(sub); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } TARGET(BINARY_SUBSCR_GETITEM) { - PyObject *sub = PEEK(1); - PyObject *container = PEEK(2); + PyObject *sub = stack_pointer[-1]; + PyObject *container = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t func_version = read_u16(&next_instr[3].cache); PyTypeObject *tp = Py_TYPE(container); @@ -585,8 +580,8 @@ } TARGET(LIST_APPEND) { - PyObject *v = PEEK(1); - PyObject *list = PEEK(2 + (oparg-1)); + PyObject *v = stack_pointer[-1]; + PyObject *list = stack_pointer[-(2 + (oparg-1))]; if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0) goto pop_1_error; STACK_SHRINK(1); PREDICT(JUMP_BACKWARD); @@ -594,8 +589,8 @@ } TARGET(SET_ADD) { - PyObject *v = PEEK(1); - PyObject *set = PEEK(2 + (oparg-1)); + PyObject *v = stack_pointer[-1]; + PyObject *set = stack_pointer[-(2 + (oparg-1))]; int err = PySet_Add(set, v); Py_DECREF(v); if (err) goto pop_1_error; @@ -607,9 +602,9 @@ TARGET(STORE_SUBSCR) { PREDICTED(STORE_SUBSCR); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - PyObject *sub = PEEK(1); - PyObject *container = PEEK(2); - PyObject *v = PEEK(3); + PyObject *sub = stack_pointer[-1]; + PyObject *container = stack_pointer[-2]; + PyObject *v = stack_pointer[-3]; uint16_t counter = read_u16(&next_instr[0].cache); #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -631,14 +626,14 @@ Py_DECREF(sub); if (err) goto pop_3_error; STACK_SHRINK(3); - JUMPBY(1); + next_instr += 1; DISPATCH(); } TARGET(STORE_SUBSCR_LIST_INT) { - PyObject *sub = PEEK(1); - PyObject *list = PEEK(2); - PyObject *value = PEEK(3); + PyObject *sub = stack_pointer[-1]; + PyObject *list = stack_pointer[-2]; + PyObject *value = stack_pointer[-3]; assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); @@ -657,14 +652,14 @@ _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); Py_DECREF(list); STACK_SHRINK(3); - JUMPBY(1); + next_instr += 1; DISPATCH(); } TARGET(STORE_SUBSCR_DICT) { - PyObject *sub = PEEK(1); - PyObject *dict = PEEK(2); - PyObject *value = PEEK(3); + PyObject *sub = stack_pointer[-1]; + PyObject *dict = stack_pointer[-2]; + PyObject *value = stack_pointer[-3]; assert(cframe.use_tracing == 0); DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); STAT_INC(STORE_SUBSCR, hit); @@ -672,13 +667,13 @@ Py_DECREF(dict); if (err) goto pop_3_error; STACK_SHRINK(3); - JUMPBY(1); + next_instr += 1; DISPATCH(); } TARGET(DELETE_SUBSCR) { - PyObject *sub = PEEK(1); - PyObject *container = PEEK(2); + PyObject *sub = stack_pointer[-1]; + PyObject *container = stack_pointer[-2]; /* del container[sub] */ int err = PyObject_DelItem(container, sub); Py_DECREF(container); @@ -689,19 +684,19 @@ } TARGET(CALL_INTRINSIC_1) { - PyObject *value = PEEK(1); + PyObject *value = stack_pointer[-1]; PyObject *res; assert(oparg <= MAX_INTRINSIC_1); res = _PyIntrinsics_UnaryFunctions[oparg](tstate, value); Py_DECREF(value); if (res == NULL) goto pop_1_error; - POKE(1, res); + stack_pointer[-1] = res; DISPATCH(); } TARGET(CALL_INTRINSIC_2) { - PyObject *value1 = PEEK(1); - PyObject *value2 = PEEK(2); + PyObject *value1 = stack_pointer[-1]; + PyObject *value2 = stack_pointer[-2]; PyObject *res; assert(oparg <= MAX_INTRINSIC_2); res = _PyIntrinsics_BinaryFunctions[oparg](tstate, value2, value1); @@ -709,12 +704,12 @@ Py_DECREF(value1); if (res == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, res); + stack_pointer[-1] = res; DISPATCH(); } TARGET(RAISE_VARARGS) { - PyObject **args = &PEEK(oparg); + PyObject **args = (stack_pointer - oparg); PyObject *cause = NULL, *exc = NULL; switch (oparg) { case 2: @@ -735,7 +730,7 @@ } TARGET(INTERPRETER_EXIT) { - PyObject *retval = PEEK(1); + PyObject *retval = stack_pointer[-1]; assert(frame == &entry_frame); assert(_PyFrame_IsIncomplete(frame)); STACK_SHRINK(1); // Since we're not going to DISPATCH() @@ -750,7 +745,7 @@ } TARGET(RETURN_VALUE) { - PyObject *retval = PEEK(1); + PyObject *retval = stack_pointer[-1]; STACK_SHRINK(1); assert(EMPTY()); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -784,7 +779,7 @@ } TARGET(GET_AITER) { - PyObject *obj = PEEK(1); + PyObject *obj = stack_pointer[-1]; PyObject *iter; unaryfunc getter = NULL; PyTypeObject *type = Py_TYPE(obj); @@ -816,12 +811,12 @@ Py_DECREF(iter); if (true) goto pop_1_error; } - POKE(1, iter); + stack_pointer[-1] = iter; DISPATCH(); } TARGET(GET_ANEXT) { - PyObject *aiter = PEEK(1); + PyObject *aiter = stack_pointer[-1]; PyObject *awaitable; unaryfunc getter = NULL; PyObject *next_iter = NULL; @@ -867,14 +862,14 @@ } STACK_GROW(1); - POKE(1, awaitable); + stack_pointer[-1] = awaitable; PREDICT(LOAD_CONST); DISPATCH(); } TARGET(GET_AWAITABLE) { PREDICTED(GET_AWAITABLE); - PyObject *iterable = PEEK(1); + PyObject *iterable = stack_pointer[-1]; PyObject *iter; iter = _PyCoro_GetAwaitableIter(iterable); @@ -900,15 +895,15 @@ if (iter == NULL) goto pop_1_error; - POKE(1, iter); + stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); } TARGET(SEND) { PREDICTED(SEND); - PyObject *v = PEEK(1); - PyObject *receiver = PEEK(2); + PyObject *v = stack_pointer[-1]; + PyObject *receiver = stack_pointer[-2]; PyObject *retval; #if ENABLE_SPECIALIZATION _PySendCache *cache = (_PySendCache *)next_instr; @@ -945,14 +940,14 @@ assert(retval != NULL); } Py_DECREF(v); - POKE(1, retval); - JUMPBY(1); + stack_pointer[-1] = retval; + next_instr += 1; DISPATCH(); } TARGET(SEND_GEN) { - PyObject *v = PEEK(1); - PyObject *receiver = PEEK(2); + PyObject *v = stack_pointer[-1]; + PyObject *receiver = stack_pointer[-2]; assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)receiver; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && @@ -971,7 +966,7 @@ } TARGET(YIELD_VALUE) { - PyObject *retval = PEEK(1); + PyObject *retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. @@ -993,7 +988,7 @@ } TARGET(POP_EXCEPT) { - PyObject *exc_value = PEEK(1); + PyObject *exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; Py_XSETREF(exc_info->exc_value, exc_value); STACK_SHRINK(1); @@ -1001,8 +996,8 @@ } TARGET(RERAISE) { - PyObject *exc = PEEK(1); - PyObject **values = &PEEK(1 + oparg); + PyObject *exc = stack_pointer[-1]; + PyObject **values = (stack_pointer - (1 + oparg)); assert(oparg >= 0 && oparg <= 2); if (oparg) { PyObject *lasti = values[0]; @@ -1025,8 +1020,8 @@ } TARGET(END_ASYNC_FOR) { - PyObject *exc = PEEK(1); - PyObject *awaitable = PEEK(2); + PyObject *exc = stack_pointer[-1]; + PyObject *awaitable = stack_pointer[-2]; assert(exc && PyExceptionInstance_Check(exc)); if (PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration)) { Py_DECREF(awaitable); @@ -1044,9 +1039,9 @@ } TARGET(CLEANUP_THROW) { - PyObject *exc_value = PEEK(1); - PyObject *last_sent_val = PEEK(2); - PyObject *sub_iter = PEEK(3); + PyObject *exc_value = stack_pointer[-1]; + PyObject *last_sent_val = stack_pointer[-2]; + PyObject *sub_iter = stack_pointer[-3]; PyObject *none; PyObject *value; assert(throwflag); @@ -1063,8 +1058,8 @@ goto exception_unwind; } STACK_SHRINK(1); - POKE(1, value); - POKE(2, none); + stack_pointer[-1] = value; + stack_pointer[-2] = none; DISPATCH(); } @@ -1072,7 +1067,7 @@ PyObject *value; value = Py_NewRef(PyExc_AssertionError); STACK_GROW(1); - POKE(1, value); + stack_pointer[-1] = value; DISPATCH(); } @@ -1100,12 +1095,12 @@ } } STACK_GROW(1); - POKE(1, bc); + stack_pointer[-1] = bc; DISPATCH(); } TARGET(STORE_NAME) { - PyObject *v = PEEK(1); + PyObject *v = stack_pointer[-1]; PyObject *name = GETITEM(names, oparg); PyObject *ns = LOCALS(); int err; @@ -1148,7 +1143,7 @@ TARGET(UNPACK_SEQUENCE) { PREDICTED(UNPACK_SEQUENCE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - PyObject *seq = PEEK(1); + PyObject *seq = stack_pointer[-1]; #if ENABLE_SPECIALIZATION _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -1166,12 +1161,12 @@ if (res == 0) goto pop_1_error; STACK_SHRINK(1); STACK_GROW(oparg); - JUMPBY(1); + next_instr += 1; DISPATCH(); } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - PyObject *seq = PEEK(1); + PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); @@ -1182,12 +1177,12 @@ Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); - JUMPBY(1); + next_instr += 1; DISPATCH(); } TARGET(UNPACK_SEQUENCE_TUPLE) { - PyObject *seq = PEEK(1); + PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); @@ -1199,12 +1194,12 @@ Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); - JUMPBY(1); + next_instr += 1; DISPATCH(); } TARGET(UNPACK_SEQUENCE_LIST) { - PyObject *seq = PEEK(1); + PyObject *seq = stack_pointer[-1]; PyObject **values = stack_pointer - (1); DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); @@ -1216,12 +1211,12 @@ Py_DECREF(seq); STACK_SHRINK(1); STACK_GROW(oparg); - JUMPBY(1); + next_instr += 1; DISPATCH(); } TARGET(UNPACK_EX) { - PyObject *seq = PEEK(1); + PyObject *seq = stack_pointer[-1]; int totalargs = 1 + (oparg & 0xFF) + (oparg >> 8); PyObject **top = stack_pointer + totalargs - 1; int res = unpack_iterable(tstate, seq, oparg & 0xFF, oparg >> 8, top); @@ -1234,8 +1229,8 @@ TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - PyObject *owner = PEEK(1); - PyObject *v = PEEK(2); + PyObject *owner = stack_pointer[-1]; + PyObject *v = stack_pointer[-2]; uint16_t counter = read_u16(&next_instr[0].cache); #if ENABLE_SPECIALIZATION if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { @@ -1257,12 +1252,12 @@ Py_DECREF(owner); if (err) goto pop_2_error; STACK_SHRINK(2); - JUMPBY(4); + next_instr += 4; DISPATCH(); } TARGET(DELETE_ATTR) { - PyObject *owner = PEEK(1); + PyObject *owner = stack_pointer[-1]; PyObject *name = GETITEM(names, oparg); int err = PyObject_SetAttr(owner, name, (PyObject *)NULL); Py_DECREF(owner); @@ -1272,7 +1267,7 @@ } TARGET(STORE_GLOBAL) { - PyObject *v = PEEK(1); + PyObject *v = stack_pointer[-1]; PyObject *name = GETITEM(names, oparg); int err = PyDict_SetItem(GLOBALS(), name, v); Py_DECREF(v); @@ -1357,7 +1352,7 @@ } } STACK_GROW(1); - POKE(1, v); + stack_pointer[-1] = v; DISPATCH(); } @@ -1420,9 +1415,9 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, v); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), null); } - JUMPBY(5); + stack_pointer[-1] = v; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } + next_instr += 5; DISPATCH(); } @@ -1444,9 +1439,9 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), null); } - JUMPBY(5); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } + next_instr += 5; DISPATCH(); } @@ -1472,9 +1467,9 @@ null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), null); } - JUMPBY(5); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = null; } + next_instr += 5; DISPATCH(); } @@ -1545,7 +1540,7 @@ Py_INCREF(value); } STACK_GROW(1); - POKE(1, value); + stack_pointer[-1] = value; DISPATCH(); } @@ -1559,12 +1554,12 @@ } Py_INCREF(value); STACK_GROW(1); - POKE(1, value); + stack_pointer[-1] = value; DISPATCH(); } TARGET(STORE_DEREF) { - PyObject *v = PEEK(1); + PyObject *v = stack_pointer[-1]; PyObject *cell = GETLOCAL(oparg); PyObject *oldobj = PyCell_GET(cell); PyCell_SET(cell, v); @@ -1588,7 +1583,7 @@ } TARGET(BUILD_STRING) { - PyObject **pieces = &PEEK(oparg); + PyObject **pieces = (stack_pointer - oparg); PyObject *str; str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); for (int i = 0; i < oparg; i++) { @@ -1597,35 +1592,35 @@ if (str == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); STACK_GROW(1); - POKE(1, str); + stack_pointer[-1] = str; DISPATCH(); } TARGET(BUILD_TUPLE) { - PyObject **values = &PEEK(oparg); + PyObject **values = (stack_pointer - oparg); PyObject *tup; tup = _PyTuple_FromArraySteal(values, oparg); if (tup == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); STACK_GROW(1); - POKE(1, tup); + stack_pointer[-1] = tup; DISPATCH(); } TARGET(BUILD_LIST) { - PyObject **values = &PEEK(oparg); + PyObject **values = (stack_pointer - oparg); PyObject *list; list = _PyList_FromArraySteal(values, oparg); if (list == NULL) { STACK_SHRINK(oparg); goto error; } STACK_SHRINK(oparg); STACK_GROW(1); - POKE(1, list); + stack_pointer[-1] = list; DISPATCH(); } TARGET(LIST_EXTEND) { - PyObject *iterable = PEEK(1); - PyObject *list = PEEK(2 + (oparg-1)); + PyObject *iterable = stack_pointer[-1]; + PyObject *list = stack_pointer[-(2 + (oparg-1))]; PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); if (none_val == NULL) { if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && @@ -1646,8 +1641,8 @@ } TARGET(SET_UPDATE) { - PyObject *iterable = PEEK(1); - PyObject *set = PEEK(2 + (oparg-1)); + PyObject *iterable = stack_pointer[-1]; + PyObject *set = stack_pointer[-(2 + (oparg-1))]; int err = _PySet_Update(set, iterable); Py_DECREF(iterable); if (err < 0) goto pop_1_error; @@ -1656,7 +1651,7 @@ } TARGET(BUILD_SET) { - PyObject **values = &PEEK(oparg); + PyObject **values = (stack_pointer - oparg); PyObject *set; set = PySet_New(NULL); if (set == NULL) @@ -1674,12 +1669,12 @@ } STACK_SHRINK(oparg); STACK_GROW(1); - POKE(1, set); + stack_pointer[-1] = set; DISPATCH(); } TARGET(BUILD_MAP) { - PyObject **values = &PEEK(oparg*2); + PyObject **values = (stack_pointer - oparg*2); PyObject *map; map = _PyDict_FromItems( values, 2, @@ -1695,7 +1690,7 @@ if (map == NULL) { STACK_SHRINK(oparg*2); goto error; } STACK_SHRINK(oparg*2); STACK_GROW(1); - POKE(1, map); + stack_pointer[-1] = map; DISPATCH(); } @@ -1743,8 +1738,8 @@ } TARGET(BUILD_CONST_KEY_MAP) { - PyObject *keys = PEEK(1); - PyObject **values = &PEEK(1 + oparg); + PyObject *keys = stack_pointer[-1]; + PyObject **values = (stack_pointer - (1 + oparg)); PyObject *map; if (!PyTuple_CheckExact(keys) || PyTuple_GET_SIZE(keys) != (Py_ssize_t)oparg) { @@ -1761,12 +1756,12 @@ } if (map == NULL) { STACK_SHRINK(oparg); goto pop_1_error; } STACK_SHRINK(oparg); - POKE(1, map); + stack_pointer[-1] = map; DISPATCH(); } TARGET(DICT_UPDATE) { - PyObject *update = PEEK(1); + PyObject *update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (PyDict_Update(dict, update) < 0) { if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { @@ -1783,7 +1778,7 @@ } TARGET(DICT_MERGE) { - PyObject *update = PEEK(1); + PyObject *update = stack_pointer[-1]; PyObject *dict = PEEK(oparg + 1); // update is still on the stack if (_PyDict_MergeEx(dict, update, 2) < 0) { @@ -1798,8 +1793,8 @@ } TARGET(MAP_ADD) { - PyObject *value = PEEK(1); - PyObject *key = PEEK(2); + PyObject *value = stack_pointer[-1]; + PyObject *key = stack_pointer[-2]; PyObject *dict = PEEK(oparg + 2); // key, value are still on the stack assert(PyDict_CheckExact(dict)); /* dict[key] = value */ @@ -1813,7 +1808,7 @@ TARGET(LOAD_ATTR) { PREDICTED(LOAD_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - PyObject *owner = PEEK(1); + PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; #if ENABLE_SPECIALIZATION @@ -1863,14 +1858,14 @@ if (res == NULL) goto pop_1_error; } STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } - JUMPBY(9); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - PyObject *owner = PEEK(1); + PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -1890,14 +1885,14 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } - JUMPBY(9); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_MODULE) { - PyObject *owner = PEEK(1); + PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -1917,14 +1912,14 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } - JUMPBY(9); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_WITH_HINT) { - PyObject *owner = PEEK(1); + PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -1958,14 +1953,14 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } - JUMPBY(9); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_SLOT) { - PyObject *owner = PEEK(1); + PyObject *owner = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -1982,14 +1977,14 @@ res2 = NULL; Py_DECREF(owner); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } - JUMPBY(9); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_CLASS) { - PyObject *cls = PEEK(1); + PyObject *cls = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -2008,14 +2003,14 @@ Py_INCREF(res); Py_DECREF(cls); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } - JUMPBY(9); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_PROPERTY) { - PyObject *owner = PEEK(1); + PyObject *owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *fget = read_obj(&next_instr[5].cache); @@ -2045,7 +2040,7 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - PyObject *owner = PEEK(1); + PyObject *owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); uint32_t func_version = read_u32(&next_instr[3].cache); PyObject *getattribute = read_obj(&next_instr[5].cache); @@ -2077,8 +2072,8 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { - PyObject *owner = PEEK(1); - PyObject *value = PEEK(2); + PyObject *owner = stack_pointer[-1]; + PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); @@ -2100,13 +2095,13 @@ } Py_DECREF(owner); STACK_SHRINK(2); - JUMPBY(4); + next_instr += 4; DISPATCH(); } TARGET(STORE_ATTR_WITH_HINT) { - PyObject *owner = PEEK(1); - PyObject *value = PEEK(2); + PyObject *owner = stack_pointer[-1]; + PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t hint = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); @@ -2149,13 +2144,13 @@ dict->ma_version_tag = new_version; Py_DECREF(owner); STACK_SHRINK(2); - JUMPBY(4); + next_instr += 4; DISPATCH(); } TARGET(STORE_ATTR_SLOT) { - PyObject *owner = PEEK(1); - PyObject *value = PEEK(2); + PyObject *owner = stack_pointer[-1]; + PyObject *value = stack_pointer[-2]; uint32_t type_version = read_u32(&next_instr[1].cache); uint16_t index = read_u16(&next_instr[3].cache); assert(cframe.use_tracing == 0); @@ -2169,13 +2164,13 @@ Py_XDECREF(old_value); Py_DECREF(owner); STACK_SHRINK(2); - JUMPBY(4); + next_instr += 4; DISPATCH(); } TARGET(COMPARE_OP) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *res; STAT_INC(COMPARE_OP, deferred); assert((oparg >> 4) <= Py_GE); @@ -2184,15 +2179,15 @@ Py_DECREF(right); if (res == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, res); - JUMPBY(1); + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(COMPARE_AND_BRANCH) { PREDICTED(COMPARE_AND_BRANCH); - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; #if ENABLE_SPECIALIZATION _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -2215,20 +2210,18 @@ int offset = next_instr[1].op.arg; int err = PyObject_IsTrue(cond); Py_DECREF(cond); - if (err < 0) { - goto error; - } + if (err < 0) goto pop_2_error; if (jump_on_true == (err != 0)) { JUMPBY(offset); } STACK_SHRINK(2); - JUMPBY(2); + next_instr += 2; DISPATCH(); } TARGET(COMPARE_AND_BRANCH_FLOAT) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; assert(cframe.use_tracing == 0); DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_AND_BRANCH); @@ -2244,13 +2237,13 @@ JUMPBY(offset); } STACK_SHRINK(2); - JUMPBY(2); + next_instr += 2; DISPATCH(); } TARGET(COMPARE_AND_BRANCH_INT) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; assert(cframe.use_tracing == 0); DEOPT_IF(!PyLong_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyLong_CheckExact(right), COMPARE_AND_BRANCH); @@ -2269,13 +2262,13 @@ JUMPBY(offset); } STACK_SHRINK(2); - JUMPBY(2); + next_instr += 2; DISPATCH(); } TARGET(COMPARE_AND_BRANCH_STR) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; assert(cframe.use_tracing == 0); DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_AND_BRANCH); DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_AND_BRANCH); @@ -2292,26 +2285,26 @@ JUMPBY(offset); } STACK_SHRINK(2); - JUMPBY(2); + next_instr += 2; DISPATCH(); } TARGET(IS_OP) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *b; int res = Py_Is(left, right) ^ oparg; Py_DECREF(left); Py_DECREF(right); b = Py_NewRef(res ? Py_True : Py_False); STACK_SHRINK(1); - POKE(1, b); + stack_pointer[-1] = b; DISPATCH(); } TARGET(CONTAINS_OP) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *b; int res = PySequence_Contains(right, left); Py_DECREF(left); @@ -2319,13 +2312,13 @@ if (res < 0) goto pop_2_error; b = Py_NewRef((res^oparg) ? Py_True : Py_False); STACK_SHRINK(1); - POKE(1, b); + stack_pointer[-1] = b; DISPATCH(); } TARGET(CHECK_EG_MATCH) { - PyObject *match_type = PEEK(1); - PyObject *exc_value = PEEK(2); + PyObject *match_type = stack_pointer[-1]; + PyObject *exc_value = stack_pointer[-2]; PyObject *rest; PyObject *match; if (check_except_star_type_valid(tstate, match_type) < 0) { @@ -2348,14 +2341,14 @@ if (!Py_IsNone(match)) { PyErr_SetExcInfo(NULL, Py_NewRef(match), NULL); } - POKE(1, match); - POKE(2, rest); + stack_pointer[-1] = match; + stack_pointer[-2] = rest; DISPATCH(); } TARGET(CHECK_EXC_MATCH) { - PyObject *right = PEEK(1); - PyObject *left = PEEK(2); + PyObject *right = stack_pointer[-1]; + PyObject *left = stack_pointer[-2]; PyObject *b; assert(PyExceptionInstance_Check(left)); if (check_except_type_valid(tstate, right) < 0) { @@ -2366,13 +2359,13 @@ int res = PyErr_GivenExceptionMatches(left, right); Py_DECREF(right); b = Py_NewRef(res ? Py_True : Py_False); - POKE(1, b); + stack_pointer[-1] = b; DISPATCH(); } TARGET(IMPORT_NAME) { - PyObject *fromlist = PEEK(1); - PyObject *level = PEEK(2); + PyObject *fromlist = stack_pointer[-1]; + PyObject *level = stack_pointer[-2]; PyObject *res; PyObject *name = GETITEM(names, oparg); res = import_name(tstate, frame, name, fromlist, level); @@ -2380,18 +2373,18 @@ Py_DECREF(fromlist); if (res == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, res); + stack_pointer[-1] = res; DISPATCH(); } TARGET(IMPORT_FROM) { - PyObject *from = PEEK(1); + PyObject *from = stack_pointer[-1]; PyObject *res; PyObject *name = GETITEM(names, oparg); res = import_from(tstate, from, name); if (res == NULL) goto error; STACK_GROW(1); - POKE(1, res); + stack_pointer[-1] = res; DISPATCH(); } @@ -2410,7 +2403,7 @@ TARGET(POP_JUMP_IF_FALSE) { PREDICTED(POP_JUMP_IF_FALSE); - PyObject *cond = PEEK(1); + PyObject *cond = stack_pointer[-1]; if (Py_IsTrue(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2433,7 +2426,7 @@ } TARGET(POP_JUMP_IF_TRUE) { - PyObject *cond = PEEK(1); + PyObject *cond = stack_pointer[-1]; if (Py_IsFalse(cond)) { _Py_DECREF_NO_DEALLOC(cond); } @@ -2456,7 +2449,7 @@ } TARGET(POP_JUMP_IF_NOT_NONE) { - PyObject *value = PEEK(1); + PyObject *value = stack_pointer[-1]; if (!Py_IsNone(value)) { Py_DECREF(value); JUMPBY(oparg); @@ -2469,7 +2462,7 @@ } TARGET(POP_JUMP_IF_NONE) { - PyObject *value = PEEK(1); + PyObject *value = stack_pointer[-1]; if (Py_IsNone(value)) { _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); @@ -2482,7 +2475,7 @@ } TARGET(JUMP_IF_FALSE_OR_POP) { - PyObject *cond = PEEK(1); + PyObject *cond = stack_pointer[-1]; bool jump = false; int err; if (Py_IsTrue(cond)) { @@ -2511,7 +2504,7 @@ } TARGET(JUMP_IF_TRUE_OR_POP) { - PyObject *cond = PEEK(1); + PyObject *cond = stack_pointer[-1]; bool jump = false; int err; if (Py_IsFalse(cond)) { @@ -2550,7 +2543,7 @@ } TARGET(GET_LEN) { - PyObject *obj = PEEK(1); + PyObject *obj = stack_pointer[-1]; PyObject *len_o; // PUSH(len(TOS)) Py_ssize_t len_i = PyObject_Length(obj); @@ -2558,14 +2551,14 @@ len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) goto error; STACK_GROW(1); - POKE(1, len_o); + stack_pointer[-1] = len_o; DISPATCH(); } TARGET(MATCH_CLASS) { - PyObject *names = PEEK(1); - PyObject *type = PEEK(2); - PyObject *subject = PEEK(3); + PyObject *names = stack_pointer[-1]; + PyObject *type = stack_pointer[-2]; + PyObject *subject = stack_pointer[-3]; PyObject *attrs; // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or // None on failure. @@ -2582,57 +2575,57 @@ attrs = Py_NewRef(Py_None); // Failure! } STACK_SHRINK(2); - POKE(1, attrs); + stack_pointer[-1] = attrs; DISPATCH(); } TARGET(MATCH_MAPPING) { - PyObject *subject = PEEK(1); + PyObject *subject = stack_pointer[-1]; PyObject *res; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; res = Py_NewRef(match ? Py_True : Py_False); STACK_GROW(1); - POKE(1, res); + stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); DISPATCH(); } TARGET(MATCH_SEQUENCE) { - PyObject *subject = PEEK(1); + PyObject *subject = stack_pointer[-1]; PyObject *res; int match = Py_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; res = Py_NewRef(match ? Py_True : Py_False); STACK_GROW(1); - POKE(1, res); + stack_pointer[-1] = res; PREDICT(POP_JUMP_IF_FALSE); DISPATCH(); } TARGET(MATCH_KEYS) { - PyObject *keys = PEEK(1); - PyObject *subject = PEEK(2); + PyObject *keys = stack_pointer[-1]; + PyObject *subject = stack_pointer[-2]; PyObject *values_or_none; // On successful match, PUSH(values). Otherwise, PUSH(None). values_or_none = match_keys(tstate, subject, keys); if (values_or_none == NULL) goto error; STACK_GROW(1); - POKE(1, values_or_none); + stack_pointer[-1] = values_or_none; DISPATCH(); } TARGET(GET_ITER) { - PyObject *iterable = PEEK(1); + PyObject *iterable = stack_pointer[-1]; PyObject *iter; /* before: [obj]; after [getiter(obj)] */ iter = PyObject_GetIter(iterable); Py_DECREF(iterable); if (iter == NULL) goto pop_1_error; - POKE(1, iter); + stack_pointer[-1] = iter; DISPATCH(); } TARGET(GET_YIELD_FROM_ITER) { - PyObject *iterable = PEEK(1); + PyObject *iterable = stack_pointer[-1]; PyObject *iter; /* before: [obj]; after [getiter(obj)] */ if (PyCoro_CheckExact(iterable)) { @@ -2658,7 +2651,7 @@ } Py_DECREF(iterable); } - POKE(1, iter); + stack_pointer[-1] = iter; PREDICT(LOAD_CONST); DISPATCH(); } @@ -2666,7 +2659,7 @@ TARGET(FOR_ITER) { PREDICTED(FOR_ITER); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - PyObject *iter = PEEK(1); + PyObject *iter = stack_pointer[-1]; PyObject *next; #if ENABLE_SPECIALIZATION _PyForIterCache *cache = (_PyForIterCache *)next_instr; @@ -2701,13 +2694,13 @@ } // Common case: no jump, leave it to the code generator STACK_GROW(1); - POKE(1, next); - JUMPBY(1); + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_LIST) { - PyObject *iter = PEEK(1); + PyObject *iter = stack_pointer[-1]; PyObject *next; assert(cframe.use_tracing == 0); DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); @@ -2730,13 +2723,13 @@ end_for_iter_list: // Common case: no jump, leave it to the code generator STACK_GROW(1); - POKE(1, next); - JUMPBY(1); + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_TUPLE) { - PyObject *iter = PEEK(1); + PyObject *iter = stack_pointer[-1]; PyObject *next; assert(cframe.use_tracing == 0); _PyTupleIterObject *it = (_PyTupleIterObject *)iter; @@ -2759,40 +2752,40 @@ end_for_iter_tuple: // Common case: no jump, leave it to the code generator STACK_GROW(1); - POKE(1, next); - JUMPBY(1); + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_RANGE) { - PyObject *iter = PEEK(1); + PyObject *iter = stack_pointer[-1]; + PyObject *next; assert(cframe.use_tracing == 0); _PyRangeIterObject *r = (_PyRangeIterObject *)iter; DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); STAT_INC(FOR_ITER, hit); - _Py_CODEUNIT next = next_instr[INLINE_CACHE_ENTRIES_FOR_ITER]; - assert(_PyOpcode_Deopt[next.op.code] == STORE_FAST); if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); // Jump over END_FOR instruction. JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); + DISPATCH(); } - else { - long value = r->start; - r->start = value + r->step; - r->len--; - if (_PyLong_AssignValue(&GETLOCAL(next.op.arg), value) < 0) { - goto error; - } - // The STORE_FAST is already done. - JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + 1); + long value = r->start; + r->start = value + r->step; + r->len--; + next = PyLong_FromLong(value); + if (next == NULL) { + goto error; } + STACK_GROW(1); + stack_pointer[-1] = next; + next_instr += 1; DISPATCH(); } TARGET(FOR_ITER_GEN) { - PyObject *iter = PEEK(1); + PyObject *iter = stack_pointer[-1]; assert(cframe.use_tracing == 0); PyGenObject *gen = (PyGenObject *)iter; DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); @@ -2810,7 +2803,7 @@ } TARGET(BEFORE_ASYNC_WITH) { - PyObject *mgr = PEEK(1); + PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; PyObject *enter = _PyObject_LookupSpecial(mgr, &_Py_ID(__aenter__)); @@ -2843,14 +2836,14 @@ if (true) goto pop_1_error; } STACK_GROW(1); - POKE(1, res); - POKE(2, exit); + stack_pointer[-1] = res; + stack_pointer[-2] = exit; PREDICT(GET_AWAITABLE); DISPATCH(); } TARGET(BEFORE_WITH) { - PyObject *mgr = PEEK(1); + PyObject *mgr = stack_pointer[-1]; PyObject *exit; PyObject *res; /* pop the context manager, push its __exit__ and the @@ -2886,15 +2879,15 @@ if (true) goto pop_1_error; } STACK_GROW(1); - POKE(1, res); - POKE(2, exit); + stack_pointer[-1] = res; + stack_pointer[-2] = exit; DISPATCH(); } TARGET(WITH_EXCEPT_START) { - PyObject *val = PEEK(1); - PyObject *lasti = PEEK(3); - PyObject *exit_func = PEEK(4); + PyObject *val = stack_pointer[-1]; + PyObject *lasti = stack_pointer[-3]; + PyObject *exit_func = stack_pointer[-4]; PyObject *res; /* At the top of the stack are 4 values: - val: TOP = exc_info() @@ -2917,12 +2910,12 @@ 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); if (res == NULL) goto error; STACK_GROW(1); - POKE(1, res); + stack_pointer[-1] = res; DISPATCH(); } TARGET(PUSH_EXC_INFO) { - PyObject *new_exc = PEEK(1); + PyObject *new_exc = stack_pointer[-1]; PyObject *prev_exc; _PyErr_StackItem *exc_info = tstate->exc_info; if (exc_info->exc_value != NULL) { @@ -2934,13 +2927,13 @@ assert(PyExceptionInstance_Check(new_exc)); exc_info->exc_value = Py_NewRef(new_exc); STACK_GROW(1); - POKE(1, new_exc); - POKE(2, prev_exc); + stack_pointer[-1] = new_exc; + stack_pointer[-2] = prev_exc; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - PyObject *self = PEEK(1); + PyObject *self = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -2964,14 +2957,14 @@ res = self; assert(oparg & 1); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } - JUMPBY(9); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - PyObject *self = PEEK(1); + PyObject *self = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -2987,14 +2980,14 @@ res = self; assert(oparg & 1); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } - JUMPBY(9); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + next_instr += 9; DISPATCH(); } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - PyObject *self = PEEK(1); + PyObject *self = stack_pointer[-1]; PyObject *res2 = NULL; PyObject *res; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -3014,9 +3007,9 @@ res = self; assert(oparg & 1); STACK_GROW(((oparg & 1) ? 1 : 0)); - POKE(1, res); - if (oparg & 1) { POKE(1 + ((oparg & 1) ? 1 : 0), res2); } - JUMPBY(9); + stack_pointer[-1] = res; + if (oparg & 1) { stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))] = res2; } + next_instr += 9; DISPATCH(); } @@ -3030,9 +3023,9 @@ TARGET(CALL) { PREDICTED(CALL); static_assert(INLINE_CACHE_ENTRIES_CALL == 4, "incorrect cache size"); - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; int is_meth = method != NULL; int total_args = oparg; @@ -3107,15 +3100,15 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - PyObject *callable = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -3129,9 +3122,9 @@ TARGET(CALL_PY_EXACT_ARGS) { PREDICTED(CALL_PY_EXACT_ARGS); - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -3160,9 +3153,9 @@ } TARGET(CALL_PY_WITH_DEFAULTS) { - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); uint16_t min_args = read_u16(&next_instr[3].cache); assert(kwnames == NULL); @@ -3197,9 +3190,9 @@ } TARGET(CALL_NO_KW_TYPE_1) { - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *null = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; assert(kwnames == NULL); assert(cframe.use_tracing == 0); @@ -3213,15 +3206,15 @@ Py_DECREF(&PyType_Type); // I.e., callable STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } TARGET(CALL_NO_KW_STR_1) { - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *null = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; assert(kwnames == NULL); assert(cframe.use_tracing == 0); @@ -3236,16 +3229,16 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_TUPLE_1) { - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *null = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; assert(kwnames == NULL); assert(oparg == 1); @@ -3259,16 +3252,16 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BUILTIN_CLASS) { - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; int is_meth = method != NULL; int total_args = oparg; @@ -3293,16 +3286,16 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_BUILTIN_O) { - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ @@ -3334,16 +3327,16 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_BUILTIN_FAST) { - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ @@ -3379,16 +3372,16 @@ */ STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ @@ -3424,16 +3417,16 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_LEN) { - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; assert(cframe.use_tracing == 0); assert(kwnames == NULL); @@ -3462,15 +3455,15 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } TARGET(CALL_NO_KW_ISINSTANCE) { - PyObject **args = &PEEK(oparg); - PyObject *callable = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *callable = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; assert(cframe.use_tracing == 0); assert(kwnames == NULL); @@ -3501,15 +3494,15 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; DISPATCH(); } TARGET(CALL_NO_KW_LIST_APPEND) { - PyObject **args = &PEEK(oparg); - PyObject *self = PEEK(1 + oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *self = stack_pointer[-(1 + oparg)]; + PyObject *method = stack_pointer[-(2 + oparg)]; assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); @@ -3531,8 +3524,8 @@ } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { - PyObject **args = &PEEK(oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; assert(kwnames == NULL); int is_meth = method != NULL; @@ -3566,15 +3559,15 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - PyObject **args = &PEEK(oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; int is_meth = method != NULL; int total_args = oparg; @@ -3606,15 +3599,15 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) { - PyObject **args = &PEEK(oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); @@ -3646,15 +3639,15 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { - PyObject **args = &PEEK(oparg); - PyObject *method = PEEK(2 + oparg); + PyObject **args = (stack_pointer - oparg); + PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; assert(kwnames == NULL); int is_meth = method != NULL; @@ -3685,17 +3678,17 @@ if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); - POKE(1, res); - JUMPBY(4); + stack_pointer[-1] = res; + next_instr += 4; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(CALL_FUNCTION_EX) { PREDICTED(CALL_FUNCTION_EX); - PyObject *kwargs = (oparg & 1) ? PEEK(((oparg & 1) ? 1 : 0)) : NULL; - PyObject *callargs = PEEK(1 + ((oparg & 1) ? 1 : 0)); - PyObject *func = PEEK(2 + ((oparg & 1) ? 1 : 0)); + PyObject *kwargs = (oparg & 1) ? stack_pointer[-(((oparg & 1) ? 1 : 0))] : NULL; + PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; + PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; if (oparg & 1) { // DICT_MERGE is called before this opcode if there are kwargs. @@ -3723,17 +3716,17 @@ if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); - POKE(1, result); + stack_pointer[-1] = result; CHECK_EVAL_BREAKER(); DISPATCH(); } TARGET(MAKE_FUNCTION) { - PyObject *codeobj = PEEK(1); - PyObject *closure = (oparg & 0x08) ? PEEK(1 + ((oparg & 0x08) ? 1 : 0)) : NULL; - PyObject *annotations = (oparg & 0x04) ? PEEK(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0)) : NULL; - PyObject *kwdefaults = (oparg & 0x02) ? PEEK(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0)) : NULL; - PyObject *defaults = (oparg & 0x01) ? PEEK(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0)) : NULL; + PyObject *codeobj = stack_pointer[-1]; + PyObject *closure = (oparg & 0x08) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0))] : NULL; + PyObject *annotations = (oparg & 0x04) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0))] : NULL; + PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; + PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; PyFunctionObject *func_obj = (PyFunctionObject *) @@ -3764,7 +3757,7 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); - POKE(1, func); + stack_pointer[-1] = func; DISPATCH(); } @@ -3792,9 +3785,9 @@ } TARGET(BUILD_SLICE) { - PyObject *step = (oparg == 3) ? PEEK(((oparg == 3) ? 1 : 0)) : NULL; - PyObject *stop = PEEK(1 + ((oparg == 3) ? 1 : 0)); - PyObject *start = PEEK(2 + ((oparg == 3) ? 1 : 0)); + PyObject *step = (oparg == 3) ? stack_pointer[-(((oparg == 3) ? 1 : 0))] : NULL; + PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; + PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; slice = PySlice_New(start, stop, step); Py_DECREF(start); @@ -3803,13 +3796,13 @@ if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); - POKE(1, slice); + stack_pointer[-1] = slice; DISPATCH(); } TARGET(FORMAT_VALUE) { - PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? PEEK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)) : NULL; - PyObject *value = PEEK(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); + PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; + PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); @@ -3857,25 +3850,25 @@ if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } } STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); - POKE(1, result); + stack_pointer[-1] = result; DISPATCH(); } TARGET(COPY) { - PyObject *bottom = PEEK(1 + (oparg-1)); + PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; assert(oparg > 0); top = Py_NewRef(bottom); STACK_GROW(1); - POKE(1, top); + stack_pointer[-1] = top; DISPATCH(); } TARGET(BINARY_OP) { PREDICTED(BINARY_OP); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); - PyObject *rhs = PEEK(1); - PyObject *lhs = PEEK(2); + PyObject *rhs = stack_pointer[-1]; + PyObject *lhs = stack_pointer[-2]; PyObject *res; #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; @@ -3896,17 +3889,17 @@ Py_DECREF(rhs); if (res == NULL) goto pop_2_error; STACK_SHRINK(1); - POKE(1, res); - JUMPBY(1); + stack_pointer[-1] = res; + next_instr += 1; DISPATCH(); } TARGET(SWAP) { - PyObject *top = PEEK(1); - PyObject *bottom = PEEK(2 + (oparg-2)); + PyObject *top = stack_pointer[-1]; + PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; assert(oparg >= 2); - POKE(1, bottom); - POKE(2 + (oparg-2), top); + stack_pointer[-1] = bottom; + stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } diff --git a/Python/import.c b/Python/import.c index fabf03b1c5d698..57d4eea148810f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -465,7 +465,7 @@ _modules_by_index_set(PyInterpreterState *interp, } static int -_modules_by_index_clear(PyInterpreterState *interp, PyModuleDef *def) +_modules_by_index_clear_one(PyInterpreterState *interp, PyModuleDef *def) { Py_ssize_t index = def->m_base.m_index; const char *err = _modules_by_index_check(interp, index); @@ -546,7 +546,7 @@ PyState_RemoveModule(PyModuleDef* def) "PyState_RemoveModule called on module with slots"); return -1; } - return _modules_by_index_clear(tstate->interp, def); + return _modules_by_index_clear_one(tstate->interp, def); } @@ -584,6 +584,109 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) /* extension modules */ /*********************/ +/* + It may help to have a big picture view of what happens + when an extension is loaded. This includes when it is imported + for the first time or via imp.load_dynamic(). + + Here's a summary, using imp.load_dynamic() as the starting point: + + 1. imp.load_dynamic() -> importlib._bootstrap._load() + 2. _load(): acquire import lock + 3. _load() -> importlib._bootstrap._load_unlocked() + 4. _load_unlocked() -> importlib._bootstrap.module_from_spec() + 5. module_from_spec() -> ExtensionFileLoader.create_module() + 6. create_module() -> _imp.create_dynamic() + (see below) + 7. module_from_spec() -> importlib._bootstrap._init_module_attrs() + 8. _load_unlocked(): sys.modules[name] = module + 9. _load_unlocked() -> ExtensionFileLoader.exec_module() + 10. exec_module() -> _imp.exec_dynamic() + (see below) + 11. _load(): release import lock + + + ...for single-phase init modules, where m_size == -1: + + (6). first time (not found in _PyRuntime.imports.extensions): + 1. _imp_create_dynamic_impl() -> import_find_extension() + 2. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec() + 3. _PyImport_LoadDynamicModuleWithSpec(): load + 4. _PyImport_LoadDynamicModuleWithSpec(): call + 5. -> PyModule_Create() -> PyModule_Create2() -> PyModule_CreateInitialized() + 6. PyModule_CreateInitialized() -> PyModule_New() + 7. PyModule_CreateInitialized(): allocate mod->md_state + 8. PyModule_CreateInitialized() -> PyModule_AddFunctions() + 9. PyModule_CreateInitialized() -> PyModule_SetDocString() + 10. PyModule_CreateInitialized(): set mod->md_def + 11. : initialize the module + 12. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + 13. _PyImport_LoadDynamicModuleWithSpec(): set def->m_base.m_init + 14. _PyImport_LoadDynamicModuleWithSpec(): set __file__ + 15. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_FixupExtensionObject() + 16. _PyImport_FixupExtensionObject(): add it to interp->imports.modules_by_index + 17. _PyImport_FixupExtensionObject(): copy __dict__ into def->m_base.m_copy + 18. _PyImport_FixupExtensionObject(): add it to _PyRuntime.imports.extensions + + (6). subsequent times (found in _PyRuntime.imports.extensions): + 1. _imp_create_dynamic_impl() -> import_find_extension() + 2. import_find_extension() -> import_add_module() + 3. if name in sys.modules: use that module + 4. else: + 1. import_add_module() -> PyModule_NewObject() + 2. import_add_module(): set it on sys.modules + 5. import_find_extension(): copy the "m_copy" dict into __dict__ + 6. _imp_create_dynamic_impl() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + + (10). (every time): + 1. noop + + + ...for single-phase init modules, where m_size >= 0: + + (6). not main interpreter and never loaded there - every time (not found in _PyRuntime.imports.extensions): + 1-16. (same as for m_size == -1) + + (6). main interpreter - first time (not found in _PyRuntime.imports.extensions): + 1-16. (same as for m_size == -1) + 17. _PyImport_FixupExtensionObject(): add it to _PyRuntime.imports.extensions + + (6). previously loaded in main interpreter (found in _PyRuntime.imports.extensions): + 1. _imp_create_dynamic_impl() -> import_find_extension() + 2. import_find_extension(): call def->m_base.m_init + 3. import_find_extension(): add the module to sys.modules + + (10). every time: + 1. noop + + + ...for multi-phase init modules: + + (6). every time: + 1. _imp_create_dynamic_impl() -> import_find_extension() (not found) + 2. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec() + 3. _PyImport_LoadDynamicModuleWithSpec(): load module init func + 4. _PyImport_LoadDynamicModuleWithSpec(): call module init func + 5. _PyImport_LoadDynamicModuleWithSpec() -> PyModule_FromDefAndSpec() + 6. PyModule_FromDefAndSpec(): gather/check moduledef slots + 7. if there's a Py_mod_create slot: + 1. PyModule_FromDefAndSpec(): call its function + 8. else: + 1. PyModule_FromDefAndSpec() -> PyModule_NewObject() + 9: PyModule_FromDefAndSpec(): set mod->md_def + 10. PyModule_FromDefAndSpec() -> _add_methods_to_object() + 11. PyModule_FromDefAndSpec() -> PyModule_SetDocString() + + (10). every time: + 1. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic() + 2. if mod->md_state == NULL (including if m_size == 0): + 1. exec_builtin_or_dynamic() -> PyModule_ExecDef() + 2. PyModule_ExecDef(): allocate mod->md_state + 3. if there's a Py_mod_exec slot: + 1. PyModule_ExecDef(): call its function + */ + + /* Make sure name is fully qualified. This is a bit of a hack: when the shared library is loaded, @@ -1007,13 +1110,17 @@ clear_singlephase_extension(PyInterpreterState *interp, /* Clear the PyState_*Module() cache entry. */ if (_modules_by_index_check(interp, def->m_base.m_index) == NULL) { - if (_modules_by_index_clear(interp, def) < 0) { + if (_modules_by_index_clear_one(interp, def) < 0) { return -1; } } /* Clear the cached module def. */ - return _extensions_cache_delete(filename, name); + if (_extensions_cache_delete(filename, name) < 0) { + return -1; + } + + return 0; } diff --git a/Python/initconfig.c b/Python/initconfig.c index deec805a6b1ca4..db7f11e17d6662 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -5,7 +5,7 @@ #include "pycore_interp.h" // _PyInterpreterState.runtime #include "pycore_long.h" // _PY_LONG_MAX_STR_DIGITS_THRESHOLD #include "pycore_pathconfig.h" // _Py_path_config -#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pylifecycle.h" // _Py_PreInitializeFromConfig() #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyThreadState_GET() diff --git a/Python/modsupport.c b/Python/modsupport.c index b9a10dc157e7e5..75698455c88166 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -93,16 +93,12 @@ static PyObject *do_mkvalue(const char**, va_list *, int); static void do_ignore(const char **p_format, va_list *p_va, char endchar, Py_ssize_t n, int flags) { - PyObject *v; - Py_ssize_t i; assert(PyErr_Occurred()); - v = PyTuple_New(n); - for (i = 0; i < n; i++) { - PyObject *exception, *value, *tb, *w; - - PyErr_Fetch(&exception, &value, &tb); - w = do_mkvalue(p_format, p_va, flags); - PyErr_Restore(exception, value, tb); + PyObject *v = PyTuple_New(n); + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *exc = PyErr_GetRaisedException(); + PyObject *w = do_mkvalue(p_format, p_va, flags); + PyErr_SetRaisedException(exc); if (w != NULL) { if (v != NULL) { PyTuple_SET_ITEM(v, i, w); diff --git a/Python/pystate.c b/Python/pystate.c index 32b17fd19e348f..28606e4f32f71c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -3,7 +3,8 @@ #include "Python.h" #include "pycore_ceval.h" -#include "pycore_code.h" // stats +#include "pycore_code.h" // stats +#include "pycore_dtoa.h" // _dtoa_state_INIT() #include "pycore_frame.h" #include "pycore_initconfig.h" #include "pycore_object.h" // _PyType_InitCache() @@ -618,6 +619,18 @@ free_interpreter(PyInterpreterState *interp) e.g. by PyMem_RawCalloc() or memset(), or otherwise pre-initialized. The runtime state is not manipulated. Instead it is assumed that the interpreter is getting added to the runtime. + + Note that the main interpreter was statically initialized as part + of the runtime and most state is already set properly. That leaves + a small number of fields to initialize dynamically, as well as some + that are initialized lazily. + + For subinterpreters we memcpy() the main interpreter in + PyInterpreterState_New(), leaving it in the same mostly-initialized + state. The only difference is that the interpreter has some + self-referential state that is statically initializexd to the + main interpreter. We fix those fields here, in addition + to the other dynamically initialized fields. */ static void @@ -645,6 +658,11 @@ init_interpreter(PyInterpreterState *interp, PyConfig_InitPythonConfig(&interp->config); _PyType_InitCache(interp); + if (interp != &runtime->_main_interpreter) { + /* Fix the self-referential, statically initialized fields. */ + interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); + } + interp->_initialized = 1; } @@ -754,12 +772,19 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) _PyErr_Clear(tstate); } + // Clear the current/main thread state last. HEAD_LOCK(runtime); - // XXX Clear the current/main thread state last. - for (PyThreadState *p = interp->threads.head; p != NULL; p = p->next) { + PyThreadState *p = interp->threads.head; + HEAD_UNLOCK(runtime); + while (p != NULL) { + // See https://github.com/python/cpython/issues/102126 + // Must be called without HEAD_LOCK held as it can deadlock + // if any finalizer tries to acquire that lock. PyThreadState_Clear(p); + HEAD_LOCK(runtime); + p = p->next; + HEAD_UNLOCK(runtime); } - HEAD_UNLOCK(runtime); /* It is possible that any of the objects below have a finalizer that runs Python code or otherwise relies on a thread state diff --git a/Python/specialize.c b/Python/specialize.c index c9555f8ad4dccb..3405d2b0ab0680 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2155,8 +2155,6 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); _PyForIterCache *cache = (_PyForIterCache *)(instr + 1); PyTypeObject *tp = Py_TYPE(iter); - _Py_CODEUNIT next = instr[1+INLINE_CACHE_ENTRIES_FOR_ITER]; - int next_op = _PyOpcode_Deopt[next.op.code]; if (tp == &PyListIter_Type) { instr->op.code = FOR_ITER_LIST; goto success; @@ -2165,7 +2163,7 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) instr->op.code = FOR_ITER_TUPLE; goto success; } - else if (tp == &PyRangeIter_Type && next_op == STORE_FAST) { + else if (tp == &PyRangeIter_Type) { instr->op.code = FOR_ITER_RANGE; goto success; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index b69b803560924c..207abb964bcac9 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -23,7 +23,7 @@ Data members: #include "pycore_namespace.h" // _PyNamespace_New() #include "pycore_object.h" // _PyObject_IS_GC() #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() -#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pylifecycle.h" // _PyErr_WriteUnraisableDefaultHook() #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() @@ -89,12 +89,11 @@ PySys_GetObject(const char *name) { PyThreadState *tstate = _PyThreadState_GET(); - PyObject *exc_type, *exc_value, *exc_tb; - _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb); + PyObject *exc = _PyErr_GetRaisedException(tstate); PyObject *value = _PySys_GetObject(tstate->interp, name); /* XXX Suppress a new exception if it was raised and restore * the old one. */ - _PyErr_Restore(tstate, exc_type, exc_value, exc_tb); + _PyErr_SetRaisedException(tstate, exc); return value; } @@ -203,8 +202,8 @@ sys_audit_tstate(PyThreadState *ts, const char *event, int dtrace = PyDTrace_AUDIT_ENABLED(); - PyObject *exc_type, *exc_value, *exc_tb; - _PyErr_Fetch(ts, &exc_type, &exc_value, &exc_tb); + + PyObject *exc = _PyErr_GetRaisedException(ts); /* Initialize event args now */ if (argFormat && argFormat[0]) { @@ -287,13 +286,11 @@ sys_audit_tstate(PyThreadState *ts, const char *event, Py_XDECREF(eventArgs); if (!res) { - _PyErr_Restore(ts, exc_type, exc_value, exc_tb); + _PyErr_SetRaisedException(ts, exc); } else { assert(_PyErr_Occurred(ts)); - Py_XDECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_tb); + Py_XDECREF(exc); } return res; @@ -3661,12 +3658,11 @@ static void sys_write(PyObject *key, FILE *fp, const char *format, va_list va) { PyObject *file; - PyObject *error_type, *error_value, *error_traceback; char buffer[1001]; int written; PyThreadState *tstate = _PyThreadState_GET(); - _PyErr_Fetch(tstate, &error_type, &error_value, &error_traceback); + PyObject *exc = _PyErr_GetRaisedException(tstate); file = _PySys_GetAttr(tstate, key); written = PyOS_vsnprintf(buffer, sizeof(buffer), format, va); if (sys_pyfile_write(buffer, file) != 0) { @@ -3678,7 +3674,7 @@ sys_write(PyObject *key, FILE *fp, const char *format, va_list va) if (sys_pyfile_write(truncated, file) != 0) fputs(truncated, fp); } - _PyErr_Restore(tstate, error_type, error_value, error_traceback); + _PyErr_SetRaisedException(tstate, exc); } void @@ -3708,7 +3704,7 @@ sys_format(PyObject *key, FILE *fp, const char *format, va_list va) const char *utf8; PyThreadState *tstate = _PyThreadState_GET(); - PyObject *error = _PyErr_GetRaisedException(tstate); + PyObject *exc = _PyErr_GetRaisedException(tstate); file = _PySys_GetAttr(tstate, key); message = PyUnicode_FromFormatV(format, va); if (message != NULL) { @@ -3720,7 +3716,7 @@ sys_format(PyObject *key, FILE *fp, const char *format, va_list va) } Py_DECREF(message); } - _PyErr_SetRaisedException(tstate, error); + _PyErr_SetRaisedException(tstate, exc); } void diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index c7f52b55a5eb99..b760172974c8a5 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -171,19 +171,17 @@ def declare(self, dst: StackEffect, src: StackEffect | None): def assign(self, dst: StackEffect, src: StackEffect): if src.name == UNUSED: return + if src.size: + # Don't write sized arrays -- it's up to the user code. + return cast = self.cast(dst, src) - if m := re.match(r"^PEEK\((.*)\)$", dst.name): - stmt = f"POKE({m.group(1)}, {cast}{src.name});" + if re.match(r"^REG\(oparg(\d+)\)$", dst.name): + self.emit(f"Py_XSETREF({dst.name}, {cast}{src.name});") + else: + stmt = f"{dst.name} = {cast}{src.name};" if src.cond: stmt = f"if ({src.cond}) {{ {stmt} }}" self.emit(stmt) - elif m := re.match(r"^&PEEK\(.*\)$", dst.name): - # The user code is responsible for writing to the output array. - pass - elif m := re.match(r"^REG\(oparg(\d+)\)$", dst.name): - self.emit(f"Py_XSETREF({dst.name}, {cast}{src.name});") - else: - self.emit(f"{dst.name} = {cast}{src.name};") def cast(self, dst: StackEffect, src: StackEffect) -> str: return f"({dst.type or 'PyObject *'})" if src.type != dst.type else "" @@ -292,11 +290,11 @@ def write(self, out: Formatter) -> None: list_effect_size([ieff for ieff in ieffects[: i + 1]]) ) if ieffect.size: - src = StackEffect(f"&PEEK({isize})", "PyObject **") + src = StackEffect(f"(stack_pointer - {maybe_parenthesize(isize)})", "PyObject **") elif ieffect.cond: - src = StackEffect(f"({ieffect.cond}) ? PEEK({isize}) : NULL", "") + src = StackEffect(f"({ieffect.cond}) ? stack_pointer[-{maybe_parenthesize(isize)}] : NULL", "") else: - src = StackEffect(f"PEEK({isize})", "") + src = StackEffect(f"stack_pointer[-{maybe_parenthesize(isize)}]", "") out.declare(ieffect, src) else: # Write input register variable declarations and initializations @@ -324,7 +322,7 @@ def write(self, out: Formatter) -> None: else: out.declare(oeffect, None) - # out.emit(f"JUMPBY(OPSIZE({self.inst.name}) - 1);") + # out.emit(f"next_instr += OPSIZE({self.inst.name}) - 1;") self.write_body(out, 0) @@ -349,9 +347,9 @@ def write(self, out: Formatter) -> None: list_effect_size([oeff for oeff in oeffects[: i + 1]]) ) if oeffect.size: - dst = StackEffect(f"&PEEK({osize})", "PyObject **") + dst = StackEffect(f"stack_pointer - {maybe_parenthesize(osize)}", "PyObject **") else: - dst = StackEffect(f"PEEK({osize})", "") + dst = StackEffect(f"stack_pointer[-{maybe_parenthesize(osize)}]", "") out.assign(dst, oeffect) else: # Write output register assignments @@ -361,7 +359,7 @@ def write(self, out: Formatter) -> None: # Write cache effect if self.cache_offset: - out.emit(f"JUMPBY({self.cache_offset});") + out.emit(f"next_instr += {self.cache_offset};") def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None: """Write the instruction body.""" @@ -1060,17 +1058,13 @@ def write_super(self, sup: SuperInstruction) -> None: with self.wrap_super_or_macro(sup): first = True for comp in sup.parts: - if first: - pass - # self.out.emit("JUMPBY(OPSIZE(opcode) - 1);") - else: - self.out.emit("NEXTOPARG();") - self.out.emit("JUMPBY(1);") - # self.out.emit("JUMPBY(OPSIZE(opcode));") + if not first: + self.out.emit("oparg = (next_instr++)->op.arg;") + # self.out.emit("next_instr += OPSIZE(opcode) - 1;") first = False comp.write_body(self.out, 0) if comp.instr.cache_offset: - self.out.emit(f"JUMPBY({comp.instr.cache_offset});") + self.out.emit(f"next_instr += {comp.instr.cache_offset};") def write_macro(self, mac: MacroInstruction) -> None: """Write code for a macro instruction.""" @@ -1087,7 +1081,7 @@ def write_macro(self, mac: MacroInstruction) -> None: cache_adjust += comp.instr.cache_offset if cache_adjust: - self.out.emit(f"JUMPBY({cache_adjust});") + self.out.emit(f"next_instr += {cache_adjust};") if ( last_instr @@ -1113,7 +1107,7 @@ def wrap_super_or_macro(self, up: SuperOrMacroInstruction): for i, var in reversed(list(enumerate(up.stack))): src = None if i < up.initial_sp: - src = StackEffect(f"PEEK({up.initial_sp - i})", "") + src = StackEffect(f"stack_pointer[-{up.initial_sp - i}]", "") self.out.declare(var, src) yield @@ -1122,7 +1116,7 @@ def wrap_super_or_macro(self, up: SuperOrMacroInstruction): self.out.stack_adjust(up.final_sp - up.initial_sp, [], []) for i, var in enumerate(reversed(up.stack[: up.final_sp]), 1): - dst = StackEffect(f"PEEK({i})", "") + dst = StackEffect(f"stack_pointer[-{i}]", "") self.out.assign(dst, var) self.out.emit(f"DISPATCH();") diff --git a/Tools/freeze/test/freeze.py b/Tools/freeze/test/freeze.py index b4c76ff36a873b..f6a5adb4519fdb 100644 --- a/Tools/freeze/test/freeze.py +++ b/Tools/freeze/test/freeze.py @@ -153,7 +153,7 @@ def prepare(script=None, outdir=None): print(f'configuring python in {builddir}...') cmd = [ os.path.join(srcdir, 'configure'), - *shlex.split(get_config_var(builddir, 'CONFIG_ARGS') or ''), + *shlex.split(get_config_var(srcdir, 'CONFIG_ARGS') or ''), ] ensure_opt(cmd, 'cache-file', os.path.join(outdir, 'python-config.cache')) prefix = os.path.join(outdir, 'python-installation') diff --git a/configure b/configure index 17dc62fb63de3b..557519ad86e06d 100755 --- a/configure +++ b/configure @@ -26844,7 +26844,7 @@ fi as_fn_append MODULE_BLOCK "MODULE__MD5_STATE=$py_cv_module__md5$as_nl" if test "x$py_cv_module__md5" = xyes; then : - + as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" fi @@ -26878,7 +26878,7 @@ fi as_fn_append MODULE_BLOCK "MODULE__SHA1_STATE=$py_cv_module__sha1$as_nl" if test "x$py_cv_module__sha1" = xyes; then : - + as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE$as_nl" fi diff --git a/configure.ac b/configure.ac index bc288b86cfa590..982f669acbcfe5 100644 --- a/configure.ac +++ b/configure.ac @@ -7194,8 +7194,12 @@ PY_STDLIB_MOD_SIMPLE([unicodedata]) dnl By default we always compile these even when OpenSSL is available dnl (issue #14693). The modules are small. -PY_STDLIB_MOD([_md5], [test "$with_builtin_md5" = yes]) -PY_STDLIB_MOD([_sha1], [test "$with_builtin_sha1" = yes]) +PY_STDLIB_MOD([_md5], + [test "$with_builtin_md5" = yes], [], + [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) +PY_STDLIB_MOD([_sha1], + [test "$with_builtin_sha1" = yes], [], + [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE]) PY_STDLIB_MOD([_sha2], [test "$with_builtin_sha2" = yes], [], [-I\$(srcdir)/Modules/_hacl/include -I\$(srcdir)/Modules/_hacl/internal -D_BSD_SOURCE -D_DEFAULT_SOURCE])