Merge branch 'main' into pythongh-114610

.github/CODEOWNERS

@@ -21,6 +21,7 @@ configure*                    @erlend-aasland @corona10
 **/*context*                  @1st1
 **/*genobject*                @markshannon
 **/*hamt*                     @1st1
+**/*jit*                      @brandtbucher
 Objects/set*                  @rhettinger
 Objects/dict*                 @methane @markshannon
 Objects/typevarobject.c       @JelleZijlstra
@@ -37,14 +38,37 @@ Python/ast_opt.c              @isidentical
 Python/bytecodes.c            @markshannon @gvanrossum
 Python/optimizer*.c           @markshannon @gvanrossum
 Lib/test/test_patma.py        @brandtbucher
-Lib/test/test_peepholer.py    @brandtbucher
 Lib/test/test_type_*.py       @JelleZijlstra
 Lib/test/test_capi/test_misc.py  @markshannon @gvanrossum
 Tools/c-analyzer/             @ericsnowcurrently
 
 # dbm
 **/*dbm*                      @corona10 @erlend-aasland @serhiy-storchaka
 
+# runtime state/lifecycle
+**/*pylifecycle*              @ericsnowcurrently
+**/*pystate*                  @ericsnowcurrently
+**/*preconfig*                @ericsnowcurrently
+**/*initconfig*               @ericsnowcurrently
+**/*pathconfig*               @ericsnowcurrently
+**/*sysmodule*                @ericsnowcurrently
+**/*bltinmodule*              @ericsnowcurrently
+**/*gil*                      @ericsnowcurrently
+Include/internal/pycore_runtime.h   @ericsnowcurrently
+Include/internal/pycore_interp.h    @ericsnowcurrently
+Include/internal/pycore_tstate.h    @ericsnowcurrently
+Include/internal/pycore_*_state.h   @ericsnowcurrently
+Include/internal/pycore_*_init.h    @ericsnowcurrently
+Include/internal/pycore_atexit.h    @ericsnowcurrently
+Include/internal/pycore_freelist.h  @ericsnowcurrently
+Include/internal/pycore_global_objects.h  @ericsnowcurrently
+Include/internal/pycore_obmalloc.h  @ericsnowcurrently
+Include/internal/pycore_pymem.h     @ericsnowcurrently
+Modules/main.c                @ericsnowcurrently
+Programs/_bootstrap_python.c  @ericsnowcurrently
+Programs/python.c             @ericsnowcurrently
+Tools/build/generate_global_objects.py  @ericsnowcurrently
+
 # Exceptions
 Lib/traceback.py              @iritkatriel
 Lib/test/test_except*.py      @iritkatriel
@@ -79,7 +103,20 @@ Modules/_hacl/**              @gpshead
 # Import (including importlib).
 **/*import*                   @brettcannon @ericsnowcurrently @ncoghlan @warsaw
 /Python/import.c              @kumaraditya303
-**/*importlib/resources/*      @jaraco @warsaw @FFY00
+Python/dynload_*.c            @ericsnowcurrently
+**/*freeze*                   @ericsnowcurrently
+**/*frozen*                   @ericsnowcurrently
+**/*modsupport*               @ericsnowcurrently
+**/*modulefinder*             @ericsnowcurrently
+**/*moduleobject*             @ericsnowcurrently
+**/*multiphase*               @ericsnowcurrently
+**/*pkgutil*                  @ericsnowcurrently
+**/*pythonrun*                @ericsnowcurrently
+**/*runpy*                    @ericsnowcurrently
+**/*singlephase*              @ericsnowcurrently
+Lib/test/test_module/         @ericsnowcurrently
+Doc/c-api/module.rst          @ericsnowcurrently
+**/*importlib/resources/*     @jaraco @warsaw @FFY00
 **/importlib/metadata/*       @jaraco @warsaw
 
 # Dates and times
@@ -198,6 +235,8 @@ Doc/c-api/stable.rst          @encukou
 Doc/howto/clinic.rst          @erlend-aasland
 
 # Subinterpreters
+**/*interpreteridobject.*     @ericsnowcurrently
+**/*crossinterp*              @ericsnowcurrently
 Lib/test/support/interpreters/  @ericsnowcurrently
 Modules/_xx*interp*module.c   @ericsnowcurrently
 Lib/test/test_interpreters/   @ericsnowcurrently

.github/workflows/jit.yml

@@ -0,0 +1,112 @@
+name: JIT
+on:
+  pull_request:
+    paths: '**jit**'
+  push:
+    paths: '**jit**'
+  workflow_dispatch:
+jobs:
+  jit:
+    name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - i686-pc-windows-msvc/msvc
+          - x86_64-pc-windows-msvc/msvc
+          - x86_64-apple-darwin/clang
+          - x86_64-unknown-linux-gnu/gcc
+          - x86_64-unknown-linux-gnu/clang
+          - aarch64-unknown-linux-gnu/gcc
+          - aarch64-unknown-linux-gnu/clang
+        debug:
+          - true
+          - false
+        llvm:
+          - 16
+        include:
+          - target: i686-pc-windows-msvc/msvc
+            architecture: Win32
+            runner: windows-latest
+            compiler: msvc
+          - target: x86_64-pc-windows-msvc/msvc
+            architecture: x64
+            runner: windows-latest
+            compiler: msvc
+          - target: x86_64-apple-darwin/clang
+            architecture: x86_64
+            runner: macos-latest
+            compiler: clang
+            exclude: test_embed
+          - target: x86_64-unknown-linux-gnu/gcc
+            architecture: x86_64
+            runner: ubuntu-latest
+            compiler: gcc
+          - target: x86_64-unknown-linux-gnu/clang
+            architecture: x86_64
+            runner: ubuntu-latest
+            compiler: clang
+          - target: aarch64-unknown-linux-gnu/gcc
+            architecture: aarch64
+            runner: ubuntu-latest
+            compiler: gcc
+            # These fail because of emulation, not because of the JIT:
+            exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv
+          - target: aarch64-unknown-linux-gnu/clang
+            architecture: aarch64
+            runner: ubuntu-latest
+            compiler: clang
+            # These fail because of emulation, not because of the JIT:
+            exclude: test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv
+    env:
+      CC: ${{ matrix.compiler }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Windows
+        if: runner.os == 'Windows'
+        run: |
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+          ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
+          ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+
+      - name: macOS
+        if: runner.os == 'macOS'
+        run: |
+          brew install llvm@${{ matrix.llvm }}
+          export SDKROOT="$(xcrun --show-sdk-path)"
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+          make all --jobs 3
+          ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+
+      - name: Native Linux
+        if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
+        run: |
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
+          make all --jobs 4
+          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3
+      - name: Emulated Linux
+        if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
+        run: |
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+          ./configure --prefix="$(pwd)/../build"
+          make install --jobs 4
+          make clean --jobs 4
+          export HOST=${{ matrix.architecture }}-linux-gnu
+          sudo apt install --yes "gcc-$HOST" qemu-user
+          ${{ !matrix.debug && matrix.compiler == 'clang' && './configure --enable-optimizations' || '' }}
+          ${{ !matrix.debug && matrix.compiler == 'clang' && 'make profile-run-stamp --jobs 4' || '' }}
+          export CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}"
+          export CPP="$CC --preprocess"
+          export HOSTRUNNER=qemu-${{ matrix.architecture }}
+          export QEMU_LD_PREFIX="/usr/$HOST"
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+          make all --jobs 4
+          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 3600 --verbose2 --verbose3

.github/workflows/mypy.yml

@@ -12,6 +12,7 @@ on:
       - "Tools/build/generate_sbom.py"
       - "Tools/cases_generator/**"
       - "Tools/clinic/**"
+      - "Tools/jit/**"
       - "Tools/peg_generator/**"
       - "Tools/requirements-dev.txt"
       - "Tools/wasm/**"
@@ -38,6 +39,7 @@ jobs:
           "Tools/build/",
           "Tools/cases_generator",
           "Tools/clinic",
+          "Tools/jit",
           "Tools/peg_generator",
           "Tools/wasm",
         ]

.gitignore

@@ -126,6 +126,7 @@ Tools/unicode/data/
 # hendrikmuhs/ccache-action@v1
 /.ccache
 /cross-build/
+/jit_stencils.h
 /platform
 /profile-clean-stamp
 /profile-run-stamp

Doc/c-api/conversion.rst

@@ -48,6 +48,42 @@ The return value (*rv*) for these functions should be interpreted as follows:
 
 The following functions provide locale-independent string to number conversions.
 
+.. c:function:: unsigned long PyOS_strtoul(const char *str, char **ptr, int base)
+
+   Convert the initial part of the string in ``str`` to an :c:expr:`unsigned
+   long` value according to the given ``base``, which must be between ``2`` and
+   ``36`` inclusive, or be the special value ``0``.
+
+   Leading white space and case of characters are ignored.  If ``base`` is zero
+   it looks for a leading ``0b``, ``0o`` or ``0x`` to tell which base.  If
+   these are absent it defaults to ``10``.  Base must be 0 or between 2 and 36
+   (inclusive).  If ``ptr`` is non-``NULL`` it will contain a pointer to the
+   end of the scan.
+
+   If the converted value falls out of range of corresponding return type,
+   range error occurs (:c:data:`errno` is set to :c:macro:`!ERANGE`) and
+   :c:macro:`!ULONG_MAX` is returned.  If no conversion can be performed, ``0``
+   is returned.
+
+   See also the Unix man page :manpage:`strtoul(3)`.
+
+   .. versionadded:: 3.2
+
+
+.. c:function:: long PyOS_strtol(const char *str, char **ptr, int base)
+
+   Convert the initial part of the string in ``str`` to an :c:expr:`long` value
+   according to the given ``base``, which must be between ``2`` and ``36``
+   inclusive, or be the special value ``0``.
+
+   Same as :c:func:`PyOS_strtoul`, but return a :c:expr:`long` value instead
+   and :c:macro:`LONG_MAX` on overflows.
+
+   See also the Unix man page :manpage:`strtol(3)`.
+
+   .. versionadded:: 3.2
+
+
 .. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception)
 
    Convert a string ``s`` to a :c:expr:`double`, raising a Python

Doc/c-api/file.rst

@@ -65,8 +65,14 @@ the :mod:`io` APIs instead.
    Overrides the normal behavior of :func:`io.open_code` to pass its parameter
    through the provided handler.
 
-   The handler is a function of type :c:expr:`PyObject *(\*)(PyObject *path,
-   void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`.
+   The *handler* is a function of type:
+
+   .. c:namespace:: NULL
+   .. c:type:: PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *)
+
+      Equivalent of :c:expr:`PyObject *(\*)(PyObject *path,
+      void *userData)`, where *path* is guaranteed to be
+      :c:type:`PyUnicodeObject`.
 
    The *userData* pointer is passed into the hook function. Since hook
    functions may be called from different runtimes, this pointer should not

Doc/c-api/memoryview.rst

@@ -20,6 +20,17 @@ any other object.
    read/write, otherwise it may be either read-only or read/write at the
    discretion of the exporter.
 
+
+.. c:macro:: PyBUF_READ
+
+   Flag to request a readonly buffer.
+
+
+.. c:macro:: PyBUF_WRITE
+
+   Flag to request a writable buffer.
+
+
 .. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags)
 
    Create a memoryview object using *mem* as the underlying buffer.
@@ -41,6 +52,8 @@ any other object.
    original memory. Otherwise, a copy is made and the memoryview points to a
    new bytes object.
 
+   *buffertype* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`.
+
 
 .. c:function:: int PyMemoryView_Check(PyObject *obj)
 

Doc/c-api/object.rst

@@ -19,6 +19,14 @@ Object Protocol
    to NotImplemented and return it).
 
 
+.. c:macro:: Py_PRINT_RAW
+
+   Flag to be used with multiple functions that print the object (like
+   :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`).
+   If passed, these function would use the :func:`str` of the object
+   instead of the :func:`repr`.
+
+
 .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags)
 
    Print an object *o*, on file *fp*.  Returns ``-1`` on error.  The flags argument
@@ -221,12 +229,8 @@ Object Protocol
 .. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid)
 
    Compare the values of *o1* and *o2* using the operation specified by *opid*,
-   which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`,
-   :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``,
-   ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error,
-   ``0`` if the result is false, ``1`` otherwise. This is the equivalent of the
-   Python expression ``o1 op o2``, where ``op`` is the operator corresponding to
-   *opid*.
+   like :c:func:`PyObject_RichCompare`, but returns ``-1`` on error, ``0`` if
+   the result is false, ``1`` otherwise.
 
 .. note::
    If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool`