Add Python 3.12 (target)

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
           for version in $(./list-versions.py --build); do
               target_dir=../maven/com/chaquo/python/target/$version
               ./download-target.sh $target_dir
-              ./unpackage-target.sh prefix $target_dir
+              ./unpackage-target.sh $target_dir prefix
           done
 
       - name: Install Python requirements
@@ -168,7 +168,6 @@ jobs:
           extra-versions: |
             3.6
             3.7
-            3.12
 
       - name: Download Maven repository
         uses: actions/download-artifact@v3.0.2

product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java

@@ -22,7 +22,7 @@ public class Common {
         PYTHON_VERSIONS.put("3.9.13", "1");
         PYTHON_VERSIONS.put("3.10.6", "1");
         PYTHON_VERSIONS.put("3.11.0", "2");
-        // TODO: once we add 3.12, remove it from extra-versions in ci.yml
+        PYTHON_VERSIONS.put("3.12.0", "0");
     }
 
     public static List<String> PYTHON_VERSIONS_SHORT = new ArrayList<>();

server/pypi/build-wheel.py

@@ -7,7 +7,6 @@
 from dataclasses import dataclass
 from email import generator, message, parser
 from glob import glob
-import multiprocessing
 import os
 from os.path import abspath, basename, dirname, exists, isdir, join, splitext
 from pathlib import Path
@@ -578,7 +577,7 @@ def env_vars(self):
 
             # conda-build variable names defined at
             # https://docs.conda.io/projects/conda-build/en/latest/user-guide/environment-variables.html
-            "CPU_COUNT": str(multiprocessing.cpu_count()),
+            # CPU_COUNT is now in build-common.sh, so the target scripts can use it.
             "PKG_BUILDNUM": str(self.meta["build"]["number"]),
             "PKG_NAME": self.package,
             "PKG_VERSION": self.version,

target/README.md

@@ -0,0 +1,84 @@
+# Chaquopy target
+
+This directory contains scripts to build Python for Android.
+
+
+## Supporting libraries
+
+Before building Python, the correct versions of the supporting libraries must already be
+present in the `prefix` subdirectory:
+
+* Bzip2, libffi and xz use static libraries, so you must build them yourself, using the
+  commands from build-all.sh.
+* SQLite and OpenSSL use dynamic libraries, so you may either build them yourself in the
+  same way, or get pre-built copies using the download-target.sh and unpackage-target.sh
+  scripts, as shown in ci.yml.
+
+
+## Python
+
+Update Common.java with the version you want to build, and the build number you want to
+give it.
+
+Run build-and-package.sh, as shown in build-all.sh. This will create a release in the
+`maven` directory in the root of this repository. If the packaging phase fails, e.g.
+because the version already exists, then rather than doing the whole build again, you
+can re-run package-target.sh directly.
+
+If this is a new major.minor version, do the "Adding a Python version" checklist below.
+
+Run the PythonVersion integration tests.
+
+Use the demo app to run the unit tests on the full set of pre-release devices (see
+release/README.md).
+
+To publish the build, follow the "Public release" instructions in release/README.md.
+Once a version has been published on Maven Central, it cannot be changed, so any fixes
+must be released under a different build number (see Common.java).
+
+
+## Adding a Python version
+
+Add it to Common.java.
+
+Add it to build-all.sh.
+
+In test_gradle_plugin.py:
+* Update the `PYTHON_VERSIONS` assertion.
+* Update `stdlib_native_expected`.
+
+Update documentation:
+* "Python version" in android.rst
+* "Python versions" in versions.rst
+
+To allow running the unit tests, build any packages used by the demo app.
+
+When building the other packages:
+
+* For each package, in dependency order:
+  * Update to the current stable version, unless it's been updated recently, or updating
+    would take a lot of work which wouldn't be justified by user demand.
+  * Review patches and build scripts to see if there are any workarounds which are no
+    longer necessary.
+* When finished the list:
+  * Clear out any bad builds before copying them to the public repository.
+* After release, notify any users who requested new versions.
+
+
+## Removing a Python version
+
+Update all the things listed in the "Adding a Python version" section.
+
+Search source code for `python *[<>=]* *[0-9]` to see if any workarounds can now be
+removed.
+
+Check if any modules can be removed from `BOOTSTRAP_NATIVE_STDLIB` in PythonTasks.kt.
+
+
+## Changing the default Python version
+
+Update `DEFAULT_PYTHON_VERSION` in Common.java.
+
+Update the pythonX.Y scripts in integration/data/BuildPython.
+
+(REST OF LIST TBD)

target/build-all.sh

@@ -10,9 +10,9 @@ done
 
 # Build libraries shared by all Python versions.
 ./for-each-abi.sh bzip2/build.sh 1.0.8
-./for-each-abi.sh libffi/build.sh 3.3
+./for-each-abi.sh libffi/build.sh 3.4.4
 ./for-each-abi.sh sqlite/build.sh 2022 3390200
-./for-each-abi.sh xz/build.sh 5.2.4
+./for-each-abi.sh xz/build.sh 5.4.5
 
 # Build all supported versions of Python, and generate `target` artifacts for Maven.
 #
@@ -29,3 +29,4 @@ python/build-and-package.sh 3.8
 python/build-and-package.sh 3.9
 python/build-and-package.sh 3.10
 python/build-and-package.sh 3.11
+python/build-and-package.sh 3.12

target/build-common.sh

@@ -6,6 +6,16 @@
 : ${abi:=$(basename $prefix)}
 : ${api_level:=21}  # Should match MIN_SDK_VERSION in Common.java.
 
+# Print all messages on stderr so they're visible when running within build-wheel.
+log() {
+    echo "$1" >&2
+}
+
+fail() {
+    log "$1"
+    exit 1
+}
+
 # When moving to a new version of the NDK, carefully review the following:
 #
 # * The release notes (https://developer.android.com/ndk/downloads/revision_history)
@@ -20,8 +30,7 @@ ndk_version=22.1.7171670  # See ndkDir in product/runtime/build.gradle.
 
 ndk=${ANDROID_HOME:?}/ndk/$ndk_version
 if ! [ -e $ndk ]; then
-    # Print all messages on stderr so they're visible when running within build-wheel.
-    echo "Installing NDK: this may take several minutes" >&2
+    log "Installing NDK: this may take several minutes"
     yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager "ndk;$ndk_version"
 fi
 
@@ -40,14 +49,13 @@ case $abi in
         host_triplet=x86_64-linux-android
         ;;
     *)
-        echo "Unknown ABI: '$abi'" >&2
-        exit 1
+        fail "Unknown ABI: '$abi'"
         ;;
 esac
 
 # These variables are based on BuildSystemMaintainers.md above, and
 # $ndk/build/cmake/android.toolchain.cmake.
-toolchain="$ndk/toolchains/llvm/prebuilt/linux-x86_64"
+toolchain=$(echo $ndk/toolchains/llvm/prebuilt/*)
 export AR="$toolchain/bin/llvm-ar"
 export AS="$toolchain/bin/$host_triplet-as"
 export CC="$toolchain/bin/${clang_triplet:-$host_triplet}$api_level-clang"
@@ -58,6 +66,13 @@ export RANLIB="$toolchain/bin/llvm-ranlib"
 export READELF="$toolchain/bin/llvm-readelf"
 export STRIP="$toolchain/bin/llvm-strip"
 
+# The quotes make sure the wildcard in the `toolchain` assignment has been expanded.
+for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; do
+    if ! [ -e "$path" ]; then
+        fail "$path does not exist"
+    fi
+done
+
 export CFLAGS="-I${prefix:?}/include"
 export LDFLAGS="-L${prefix:?}/lib \
 -Wl,--exclude-libs,libgcc.a -Wl,--exclude-libs,libgcc_real.a -Wl,--exclude-libs,libunwind.a \
@@ -78,3 +93,10 @@ esac
 
 export PKG_CONFIG="pkg-config --define-prefix"
 export PKG_CONFIG_LIBDIR="$prefix/lib/pkgconfig"
+
+# conda-build variable name
+if [ $(uname) = "Darwin" ]; then
+    export CPU_COUNT=$(sysctl -n hw.ncpu)
+else
+    export CPU_COUNT=$(nproc)
+fi

target/libffi/build.sh

@@ -12,7 +12,7 @@ version_dir=$recipe_dir/build/$version
 mkdir -p $version_dir
 cd $version_dir
 src_filename=libffi-$version.tar.gz
-wget -c ftp://sourceware.org/pub/libffi/libffi-$version.tar.gz
+wget -c https://github.com/libffi/libffi/releases/download/v$version/$src_filename
 
 build_dir=$version_dir/$abi
 rm -rf $build_dir

target/package-target.sh

@@ -1,31 +1,31 @@
 #!/bin/bash
 set -eu -o pipefail
-shopt -s inherit_errexit
 
-# Positional arguments (order is the same as unpackage-target.sh):
-#  * `prefix` directory to pack from.
+# Positional arguments:
 #  * Maven directory to pack into, e.g. /path/to/com/chaquo/python/target/3.10.6-3. Must
 #    not already exist.
+#  * Multiple `prefix` subdirectories to pack from. Each must be named after an ABI.
 
 this_dir=$(dirname $(realpath $0))
-prefix_dir=$(realpath ${1:?})
-target_dir=$(realpath -m ${2:?})
-
-# If the target looks like a full Maven repository, make sure that its root directory
-# already exists.
-if [[ $(dirname $target_dir) =~ /com/chaquo/python/target$ ]]; then
-    maven_root=$(realpath -m $target_dir/../../../../..)
-    if [ ! -e $maven_root ]; then
-        echo $maven_root does not exist: did you forget to pass it as a Docker volume?
-        exit 1
-    fi
-fi
+target_dir=${1:?}
 
-full_ver=$(basename $target_dir)
-short_ver=$(echo $full_ver | sed -E 's/^([0-9]+\.[0-9]+).*/\1/')
+prefixes=""
+shift
+if [ $# = "0" ]; then
+    echo "Must provide at least one prefix subdirectory"
+    exit 1
+fi
+while [ $# != "0" ]; do
+    prefixes+=" $(realpath $1)"
+    shift
+done
 
 mkdir -p $(dirname $target_dir)
 mkdir "$target_dir"  # Fail if it already exists: we don't want to overwrite things by accident.
+target_dir=$(realpath $target_dir)
+
+full_ver=$(basename $target_dir)
+short_ver=$(echo $full_ver | sed -E 's/^([0-9]+\.[0-9]+).*/\1/')
 target_prefix="$target_dir/target-$full_ver"
 
 cat > "$target_prefix.pom" <<EOF
@@ -70,7 +70,7 @@ rm -rf "$tmp_dir"
 mkdir "$tmp_dir"
 cd "$tmp_dir"
 
-for prefix in $prefix_dir/*; do
+for prefix in $prefixes; do
     unset abi api_level
     . "$this_dir/build-common.sh"
     echo "$abi"
@@ -95,8 +95,8 @@ for prefix in $prefix_dir/*; do
     done
     rm $dynload_dir/*_test*.so
 
-    chmod u+w $(find -name *.so)
-    $STRIP $(find -name *.so)
+    chmod u+w $(find . -name *.so)
+    $STRIP $(find . -name *.so)
 
     abi_zip="$target_prefix-$abi.zip"
     rm -f "$abi_zip"
@@ -114,7 +114,7 @@ rm -r curses idlelib tkinter turtle*
 
 # Remove things which are large and unnecessary.
 rm -r ensurepip pydoc_data
-find -name test -or -name tests | xargs rm -r
+find . -name test -or -name tests | xargs rm -r
 
 # The build generates these files with the version number of the build Python, not the target
 # Python. The source .txt files can't be used instead, because lib2to3 can only load them from
@@ -134,7 +134,7 @@ zip -q -i '*.py' '*.pickle' -r $stdlib_zip .
 
 echo "stdlib-pyc"
 # zipimport doesn't support __pycache__ directories,
-find -name __pycache__ | xargs rm -r
+find . -name __pycache__ | xargs rm -r
 
 # Run compileall from the parent directory: that way the "stdlib/" prefix gets encoded into the
 # .pyc files and will appear in traceback messages.

target/python/build-and-package.sh

@@ -9,5 +9,18 @@ cd $recipe_dir/..
 version_micro=$(./list-versions.py --micro | grep "^$version_short\.")
 version_build=$(./list-versions.py --build | grep "^$version_short\.")
 
-./for-each-abi.sh python/build.sh $version_micro
-./package-target.sh prefix ../maven/com/chaquo/python/target/$version_build
+case $version_short in
+    3.8|3.9|3.10|3.11)
+        abis="armeabi-v7a arm64-v8a x86 x86_64"
+        ;;
+    *)
+        abis="arm64-v8a x86_64"
+        ;;
+esac
+
+for abi in $abis; do
+    python/build.sh prefix/$abi $version_micro
+done
+
+cd prefix
+../package-target.sh ../../maven/com/chaquo/python/target/$version_build $abis