Add Python 3.12 (Gradle plugin)

product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java

@@ -39,8 +39,22 @@ public class Common {
     // Wheel tags (PEP 425).
     public static final String PYTHON_IMPLEMENTATION = "cp";  // CPython
 
-    public static final List<String> ABIS = Arrays.asList
-        ("armeabi-v7a", "arm64-v8a", "x86", "x86_64");
+    public static List<String> supportedAbis(String pythonVersion) {
+        if (!PYTHON_VERSIONS_SHORT.contains(pythonVersion)) {
+            throw new IllegalArgumentException(
+                "Unknown Python version: '" + pythonVersion + "'");
+        }
+
+        List<String> result = new ArrayList<>();
+        result.add("arm64-v8a");
+        result.add("x86_64");
+        if (Arrays.asList("3.8", "3.9", "3.10", "3.11").contains(pythonVersion)) {
+            result.add("armeabi-v7a");
+            result.add("x86");
+        }
+        result.sort(null);  // For testing error messages
+        return result;
+    }
 
     // Subdirectory name to use within assets, getFilesDir() and getCacheDir()
     public static final String ASSET_DIR = "chaquopy";

product/gradle-plugin/src/main/kotlin/PythonPlugin.kt

@@ -225,13 +225,13 @@ class PythonPlugin : Plugin<Project> {
         for ((_, flavor) in variant.productFlavors.reversed()) {
             python.mergeFrom(extension.productFlavors.getByName(flavor))
         }
-        TaskBuilder(this, variant, python, getAbis(variant)).build()
+        TaskBuilder(this, variant, python, getAbis(variant, python)).build()
     }
 
     // variant.externalNativeBuild returns "null if no cmake external build is
     // configured for this variant", so we'll have to determine the abiFilters from
     // the DSL.
-    fun getAbis(variant: Variant): List<String> {
+    fun getAbis(variant: Variant, python: PythonExtension): List<String> {
         // We return the variants in ASCII order. Preserving the order specified in the
         // build.gradle file is not possible, because they're stored in a HashSet.
         val abis = TreeSet(android.defaultConfig.ndk.abiFilters)
@@ -248,11 +248,13 @@ class PythonPlugin : Plugin<Project> {
                 "Variant '${variant.name}': Chaquopy requires ndk.abiFilters: " +
                 "you may want to add it to android.defaultConfig.")
         }
+
+        val supported = Common.supportedAbis(python.version)
         for (abi in abis) {
-            if (abi !in Common.ABIS) {
+            if (abi !in supported) {
                 throw GradleException(
-                    "Variant '${variant.name}': Chaquopy does not support the ABI " +
-                    "'$abi'. Supported ABIs are ${Common.ABIS}.")
+                    "Variant '${variant.name}': Python ${python.version} is not " +
+                    "available for the ABI '$abi'. Supported ABIs are $supported.")
             }
         }
         return ArrayList(abis)

product/gradle-plugin/src/main/kotlin/PythonTasks.kt

@@ -394,13 +394,17 @@ internal class TaskBuilder(
                 // pre-extracted by AndroidPlatform so they can be loaded with the
                 // standard FileFinder. All other native modules are loaded from a .zip using
                 // AssetFinder.
+                //
+                // If this list changes, search for references to this variable name to
+                // find the tests that need to be updated.
                 val BOOTSTRAP_NATIVE_STDLIB = listOf(
                     "_bz2.so",  // zipfile < importer
                     "_ctypes.so",  // java.primitive and importer
                     "_datetime.so",  // calendar < importer (see test_datetime)
                     "_lzma.so",  // zipfile < importer
                     "_random.so",  // random < tempfile < zipimport
-                    "_sha512.so",  // random < tempfile < zipimport
+                    "_sha2.so",  // random < tempfile < zipimport (Python >= 3.12)
+                    "_sha512.so",  // random < tempfile < zipimport (Python <= 3.11)
                     "_struct.so",  // zipfile < importer
                     "binascii.so",  // zipfile < importer
                     "math.so",  // datetime < calendar < importer

product/gradle-plugin/src/main/python/chaquopy/pyc.py

@@ -15,13 +15,14 @@
 import warnings
 
 
-# See importlib._bootstrap_external.MAGIC_NUMBER.
+# See the list in importlib/_bootstrap_external.py.
 MAGIC = {
     "3.7": 3394,
     "3.8": 3413,
     "3.9": 3425,
     "3.10": 3439,
     "3.11": 3495,
+    "3.12": 3531,
 }
 
 

product/gradle-plugin/src/test/integration/data/AbiFilters/invalid_32bit/app/build.gradle

@@ -0,0 +1,20 @@
+apply plugin: 'com.android.application'
+apply plugin: 'com.chaquo.python'
+
+android {
+    namespace "com.chaquo.python.test"
+    compileSdk 23
+    defaultConfig {
+        applicationId "com.chaquo.python.test"
+        minSdk 21
+        targetSdk 23
+        versionCode 1
+        versionName "0.0.1"
+        python {
+            version "3.12"
+        }
+        ndk {
+            abiFilters "x86"
+        }
+    }
+}

product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle

@@ -17,7 +17,7 @@ android {
             version "3.10"
         }
         ndk {
-            abiFilters "x86"
+            abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
         }
     }
 }

product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle

@@ -17,7 +17,7 @@ android {
             version "3.11"
         }
         ndk {
-            abiFilters "x86"
+            abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
         }
     }
 }

product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle

@@ -0,0 +1,23 @@
+plugins {
+    id 'com.android.application'
+    id 'com.chaquo.python'
+}
+
+android {
+    namespace "com.chaquo.python.test"
+    compileSdk 23
+
+    defaultConfig {
+        applicationId "com.chaquo.python.test"
+        minSdk 21
+        targetSdk 23
+        versionCode 1
+        versionName "0.0.1"
+        python {
+            version "3.12"
+        }
+        ndk {
+            abiFilters "arm64-v8a", "x86_64"
+        }
+    }
+}

product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle

@@ -17,7 +17,7 @@ android {
             version "3.8"
         }
         ndk {
-            abiFilters "x86"
+            abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
         }
     }
 }

product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle

@@ -17,7 +17,7 @@ android {
             version "3.9"
         }
         ndk {
-            abiFilters "x86"
+            abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
         }
     }
 }

product/gradle-plugin/src/test/integration/test_gradle_plugin.py

@@ -55,7 +55,7 @@ def list_versions(mode):
 for full_version in list_versions("micro").splitlines():
     version = full_version.rpartition(".")[0]
     PYTHON_VERSIONS[version] = full_version
-assert list(PYTHON_VERSIONS) == ["3.8", "3.9", "3.10", "3.11"]
+assert list(PYTHON_VERSIONS) == ["3.8", "3.9", "3.10", "3.11", "3.12"]
 DEFAULT_PYTHON_VERSION_FULL = PYTHON_VERSIONS[DEFAULT_PYTHON_VERSION]
 
 NON_DEFAULT_PYTHON_VERSION = "3.10"
@@ -398,18 +398,24 @@ class PythonVersion(GradleTestCase):
     # To allow a quick check of the setting, this test only covers two versions.
     def test_change(self):
         run = self.RunGradle("base", run=False)
-        for version in ["3.8", "3.9"]:
+        for version in [DEFAULT_PYTHON_VERSION, NON_DEFAULT_PYTHON_VERSION]:
             self.check_version(run, version)
 
     # Test all versions not covered by test_change.
     def test_others(self):
         run = self.RunGradle("base", run=False)
-        for version in ["3.10", "3.11"]:
-            self.check_version(run, version)
+        for version in PYTHON_VERSIONS:
+            if version not in [DEFAULT_PYTHON_VERSION, NON_DEFAULT_PYTHON_VERSION]:
+                self.check_version(run, version)
 
     def check_version(self, run, version):
         with self.subTest(version=version):
-            run.rerun(f"PythonVersion/{version}", python_version=version)
+            # Make sure every ABI has the full set of native stdlib module files.
+            abis = ["arm64-v8a", "x86_64"]
+            if version in ["3.8", "3.9", "3.10", "3.11"]:
+                abis += ["armeabi-v7a", "x86"]
+            run.rerun(f"PythonVersion/{version}", python_version=version, abis=abis)
+
             if version == DEFAULT_PYTHON_VERSION:
                 self.assertNotInLong(self.WARNING.format(".*"), run.stdout, re=True)
             else:
@@ -441,9 +447,17 @@ def test_missing(self):
 
     def test_invalid(self):
         run = self.RunGradle("base", "AbiFilters/invalid", succeed=False)
-        self.assertInLong("Variant 'debug': Chaquopy does not support the ABI 'armeabi'. "
-                          "Supported ABIs are [armeabi-v7a, arm64-v8a, x86, x86_64].",
-                          run.stderr)
+        self.assertInLong(
+            "Variant 'debug': Python 3.8 is not available for the ABI 'armeabi'. "
+            "Supported ABIs are [arm64-v8a, armeabi-v7a, x86, x86_64].",
+            run.stderr)
+
+    def test_invalid_32bit(self):
+        run = self.RunGradle("base", "AbiFilters/invalid_32bit", succeed=False)
+        self.assertInLong(
+            "Variant 'debug': Python 3.12 is not available for the ABI 'x86'. "
+            "Supported ABIs are [arm64-v8a, x86_64].",
+            run.stderr)
 
     def test_all(self):  # Also tests making a change.
         run = self.RunGradle("base", abis=["x86"])
@@ -1734,21 +1748,27 @@ def check_assets(self, apk_dir, kwargs):
         with ZipFile(join(asset_dir, "bootstrap.imy")) as bootstrap_zip:
             self.check_pyc(bootstrap_zip, "java/__init__.pyc", kwargs)
 
+        python_version_info = tuple(int(x) for x in python_version.split("."))
+        stdlib_bootstrap_expected = {
+            # This is the list from our minimum Python version. For why each of these
+            # modules is needed, see BOOTSTRAP_NATIVE_STDLIB in PythonTasks.kt.
+            "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so",
+            "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so",
+        }
+        if python_version_info >= (3, 12):
+            stdlib_bootstrap_expected -= {"_sha512.so"}
+            stdlib_bootstrap_expected |= {"_sha2.so"}
+
         bootstrap_native_dir = join(asset_dir, "bootstrap-native")
         self.test.assertCountEqual(abis, os.listdir(bootstrap_native_dir))
         for abi in abis:
             abi_dir = join(bootstrap_native_dir, abi)
-            self.test.assertCountEqual(
-                # For why each of these modules are needed, see BOOTSTRAP_NATIVE_STDLIB
-                # in PythonTasks.kt.
-                ["java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so",
-                 "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so"],
-                os.listdir(abi_dir))
-            self.check_python_so(join(abi_dir, "_ctypes.so"), python_version, abi)
+            self.test.assertCountEqual(stdlib_bootstrap_expected, os.listdir(abi_dir))
+            self.check_so(join(abi_dir, "_ctypes.so"), python_version, abi)
 
             java_dir = join(abi_dir, "java")
             self.test.assertCountEqual(["chaquopy.so"], os.listdir(java_dir))
-            self.check_python_so(join(java_dir, "chaquopy.so"), python_version, abi)
+            self.check_so(join(java_dir, "chaquopy.so"), python_version, abi)
 
         # Python stdlib
         with ZipFile(join(asset_dir, "stdlib-common.imy")) as stdlib_zip:
@@ -1777,14 +1797,16 @@ def check_assets(self, apk_dir, kwargs):
             "audioop.so", "cmath.so", "fcntl.so", "ossaudiodev.so", "parser.so",
             "pyexpat.so", "resource.so", "select.so", "syslog.so", "termios.so",
             "unicodedata.so", "xxlimited.so"}
-        python_version_info = tuple(int(x) for x in python_version.split("."))
         if python_version_info >= (3, 9):
             stdlib_native_expected |= {"_zoneinfo.so"}
         if python_version_info >= (3, 10):
             stdlib_native_expected -= {"parser.so"}
             stdlib_native_expected |= {"xxlimited_35.so"}
         if python_version_info >= (3, 11):
             stdlib_native_expected |= {"_typing.so"}
+        if python_version_info >= (3, 12):
+            stdlib_native_expected -= {"_sha256.so", "_typing.so"}
+            stdlib_native_expected |= {"_xxinterpchannels.so", "xxsubtype.so"}
 
         for abi in abis:
             stdlib_native_zip = ZipFile(join(asset_dir, f"stdlib-{abi}.imy"))
@@ -1793,7 +1815,7 @@ def check_assets(self, apk_dir, kwargs):
             with TemporaryDirectory() as tmp_dir:
                 test_module = "_asyncio.so"
                 stdlib_native_zip.extract(test_module, tmp_dir)
-                self.check_python_so(join(tmp_dir, test_module), python_version, abi)
+                self.check_so(join(tmp_dir, test_module), python_version, abi)
 
         # build.json
         with open(join(asset_dir, "build.json")) as build_json_file:
@@ -1812,13 +1834,14 @@ def check_assets(self, apk_dir, kwargs):
             build_json["assets"])
 
     def check_pyc(self, zip_file, pyc_filename, kwargs):
-        # See importlib._bootstrap_external.MAGIC_NUMBER.
+        # See the list in importlib/_bootstrap_external.py.
         MAGIC = {
             "3.7": 3394,
             "3.8": 3413,
             "3.9": 3425,
             "3.10": 3439,
             "3.11": 3495,
+            "3.12": 3531,
         }
         with zip_file.open(pyc_filename) as pyc_file:
             self.test.assertEqual(
@@ -1836,9 +1859,9 @@ def check_lib(self, lib_dir, kwargs):
                  f"libpython{kwargs['python_version']}.so", "libssl_chaquopy.so",
                  "libsqlite3_chaquopy.so"],
                 os.listdir(abi_dir))
-            self.check_python_so(join(abi_dir, "libchaquopy_java.so"), python_version, abi)
+            self.check_so(join(abi_dir, "libchaquopy_java.so"), python_version, abi)
 
-    def check_python_so(self, so_filename, python_version, abi):
+    def check_so(self, so_filename, python_version, abi):
         libpythons = []
         with open(so_filename, "rb") as so_file:
             ef = ELFFile(so_file)
@@ -1853,7 +1876,11 @@ def check_python_so(self, so_filename, python_version, abi):
                 if tag.entry.d_tag == "DT_NEEDED" and \
                    tag.needed.startswith("libpython"):
                     libpythons.append(tag.needed)
-        self.test.assertEqual([f"libpython{python_version}.so"], libpythons)
+
+        # Python 3.12 doesn't link its stdlib modules against libpython. But we'll make
+        # sure that *if* there's a libpython, it's the correct version.
+        if libpythons:
+            self.test.assertEqual([f"libpython{python_version}.so"], libpythons)
 
     def dump_run(self, msg):
         self.test.fail(msg + "\n" +

product/runtime/build.gradle

@@ -228,7 +228,10 @@ if (!(cmakeBuildType in KNOWN_BUILD_TYPES)) {
         "Unknown build type '$cmakeBuildType'; valid values are $KNOWN_BUILD_TYPES")
 }
 
-(["host"] + Common.ABIS).each { abi ->
+(["host"] + Common.supportedAbis("3.11")).each { abi ->
+    def pyVersions = Common.PYTHON_VERSIONS_SHORT.findAll {
+        Common.supportedAbis(it).contains(abi)
+    }
     def pyLibSuffix = ".so"
     def cmakeBuildSubdir = "$buildDir/cmake/$abi"
     def cmake = tasks.register("cmake-$abi", Exec) {
@@ -273,11 +276,12 @@ if (!(cmakeBuildType in KNOWN_BUILD_TYPES)) {
             // GitHub Actions runner. Ideally it would also match the version in
             // target/build-common.sh, but the latter is more difficult to change.
             def ndkDir = sdkPath("ndk/26.1.10909125")
+
             def prefixDir = "$projectDir/../../target/prefix/$abi"
             args "-DCMAKE_TOOLCHAIN_FILE=$ndkDir/build/cmake/android.toolchain.cmake",
                  "-DANDROID_ABI=$abi", "-DANDROID_STL=system",
                  "-DANDROID_NATIVE_API_LEVEL=$Common.MIN_SDK_VERSION",
-                 "-DCHAQUOPY_PYTHON_VERSIONS=${Common.PYTHON_VERSIONS_SHORT.join(';')}",
+                 "-DCHAQUOPY_PYTHON_VERSIONS=${pyVersions.join(';')}",
                  "-DCHAQUOPY_INCLUDE_PYTHON=$prefixDir/include",
                  "-DCHAQUOPY_LIB_DIRS=$prefixDir/lib"
         }
@@ -294,7 +298,7 @@ if (!(cmakeBuildType in KNOWN_BUILD_TYPES)) {
     }
     if (abi != "host") {
         for (name in ["chaquopy", "libchaquopy_java"]) {
-            for (pyVersion in Common.PYTHON_VERSIONS_SHORT) {
+            for (pyVersion in pyVersions) {
                 addArtifact(
                     cmakeBuild, new File(cmakeBuildSubdir, "$name-${pyVersion}.so"),
                     pyVersion, abi)

product/runtime/requirements-build.txt

@@ -1 +1 @@
-Cython==0.29.32
+Cython==0.29.36

release/bundle.sh

@@ -5,7 +5,7 @@ version="${1:?}"
 
 cd "$(dirname $(realpath $0))/../maven"
 
-find -name $version | while read dir; do (
+find . -name $version | while read dir; do (
     cd $dir
 
     # Maven Central adds hashes to every file in the bundle, even existing hash files.

target/README.md

@@ -43,9 +43,9 @@ Add it to Common.java.
 
 Add it to build-all.sh.
 
-In test_gradle_plugin.py:
-* Update the `PYTHON_VERSIONS` assertion.
-* Update `stdlib_native_expected`.
+In test_gradle_plugin.py, update the `PYTHON_VERSIONS` assertion.
+
+Update the `MAGIC` lists in test_gradle_plugin.py and pyc.py.
 
 Update documentation:
 * "Python version" in android.rst