From 8dd172d105d61591572af418f4e7f221eaf1df5e Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 21 Nov 2023 13:46:14 +0000 Subject: [PATCH] Add Python 3.12 (Gradle plugin) --- .../com/chaquo/python/internal/Common.java | 18 ++++- .../src/main/kotlin/PythonPlugin.kt | 12 ++-- .../src/main/kotlin/PythonTasks.kt | 6 +- .../src/main/python/chaquopy/pyc.py | 3 +- .../AbiFilters/invalid_32bit/app/build.gradle | 20 ++++++ .../data/PythonVersion/3.10/app/build.gradle | 2 +- .../data/PythonVersion/3.11/app/build.gradle | 2 +- .../data/PythonVersion/3.12/app/build.gradle | 23 ++++++ .../data/PythonVersion/3.8/app/build.gradle | 2 +- .../data/PythonVersion/3.9/app/build.gradle | 2 +- .../test/integration/test_gradle_plugin.py | 71 +++++++++++++------ product/runtime/build.gradle | 10 ++- product/runtime/requirements-build.txt | 2 +- release/bundle.sh | 2 +- target/README.md | 6 +- 15 files changed, 138 insertions(+), 43 deletions(-) create mode 100644 product/gradle-plugin/src/test/integration/data/AbiFilters/invalid_32bit/app/build.gradle create mode 100644 product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle mode change 100644 => 100755 release/bundle.sh diff --git a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java index 01fa49d8c9..73dd824179 100644 --- a/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java +++ b/product/buildSrc/src/main/java/com/chaquo/python/internal/Common.java @@ -39,8 +39,22 @@ public class Common { // Wheel tags (PEP 425). public static final String PYTHON_IMPLEMENTATION = "cp"; // CPython - public static final List ABIS = Arrays.asList - ("armeabi-v7a", "arm64-v8a", "x86", "x86_64"); + public static List supportedAbis(String pythonVersion) { + if (!PYTHON_VERSIONS_SHORT.contains(pythonVersion)) { + throw new IllegalArgumentException( + "Unknown Python version: '" + pythonVersion + "'"); + } + + List result = new ArrayList<>(); + result.add("arm64-v8a"); + result.add("x86_64"); + if (Arrays.asList("3.8", "3.9", "3.10", "3.11").contains(pythonVersion)) { + result.add("armeabi-v7a"); + result.add("x86"); + } + result.sort(null); // For testing error messages + return result; + } // Subdirectory name to use within assets, getFilesDir() and getCacheDir() public static final String ASSET_DIR = "chaquopy"; diff --git a/product/gradle-plugin/src/main/kotlin/PythonPlugin.kt b/product/gradle-plugin/src/main/kotlin/PythonPlugin.kt index 51b9b48bbc..7baabcbfff 100644 --- a/product/gradle-plugin/src/main/kotlin/PythonPlugin.kt +++ b/product/gradle-plugin/src/main/kotlin/PythonPlugin.kt @@ -225,13 +225,13 @@ class PythonPlugin : Plugin { for ((_, flavor) in variant.productFlavors.reversed()) { python.mergeFrom(extension.productFlavors.getByName(flavor)) } - TaskBuilder(this, variant, python, getAbis(variant)).build() + TaskBuilder(this, variant, python, getAbis(variant, python)).build() } // variant.externalNativeBuild returns "null if no cmake external build is // configured for this variant", so we'll have to determine the abiFilters from // the DSL. - fun getAbis(variant: Variant): List { + fun getAbis(variant: Variant, python: PythonExtension): List { // We return the variants in ASCII order. Preserving the order specified in the // build.gradle file is not possible, because they're stored in a HashSet. val abis = TreeSet(android.defaultConfig.ndk.abiFilters) @@ -248,11 +248,13 @@ class PythonPlugin : Plugin { "Variant '${variant.name}': Chaquopy requires ndk.abiFilters: " + "you may want to add it to android.defaultConfig.") } + + val supported = Common.supportedAbis(python.version) for (abi in abis) { - if (abi !in Common.ABIS) { + if (abi !in supported) { throw GradleException( - "Variant '${variant.name}': Chaquopy does not support the ABI " + - "'$abi'. Supported ABIs are ${Common.ABIS}.") + "Variant '${variant.name}': Python ${python.version} is not " + + "available for the ABI '$abi'. Supported ABIs are $supported.") } } return ArrayList(abis) diff --git a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt index b1d4f70388..570fb146b0 100644 --- a/product/gradle-plugin/src/main/kotlin/PythonTasks.kt +++ b/product/gradle-plugin/src/main/kotlin/PythonTasks.kt @@ -394,13 +394,17 @@ internal class TaskBuilder( // pre-extracted by AndroidPlatform so they can be loaded with the // standard FileFinder. All other native modules are loaded from a .zip using // AssetFinder. + // + // If this list changes, search for references to this variable name to + // find the tests that need to be updated. val BOOTSTRAP_NATIVE_STDLIB = listOf( "_bz2.so", // zipfile < importer "_ctypes.so", // java.primitive and importer "_datetime.so", // calendar < importer (see test_datetime) "_lzma.so", // zipfile < importer "_random.so", // random < tempfile < zipimport - "_sha512.so", // random < tempfile < zipimport + "_sha2.so", // random < tempfile < zipimport (Python >= 3.12) + "_sha512.so", // random < tempfile < zipimport (Python <= 3.11) "_struct.so", // zipfile < importer "binascii.so", // zipfile < importer "math.so", // datetime < calendar < importer diff --git a/product/gradle-plugin/src/main/python/chaquopy/pyc.py b/product/gradle-plugin/src/main/python/chaquopy/pyc.py index 5345d222ff..dd9e705cc2 100644 --- a/product/gradle-plugin/src/main/python/chaquopy/pyc.py +++ b/product/gradle-plugin/src/main/python/chaquopy/pyc.py @@ -15,13 +15,14 @@ import warnings -# See importlib._bootstrap_external.MAGIC_NUMBER. +# See the list in importlib/_bootstrap_external.py. MAGIC = { "3.7": 3394, "3.8": 3413, "3.9": 3425, "3.10": 3439, "3.11": 3495, + "3.12": 3531, } diff --git a/product/gradle-plugin/src/test/integration/data/AbiFilters/invalid_32bit/app/build.gradle b/product/gradle-plugin/src/test/integration/data/AbiFilters/invalid_32bit/app/build.gradle new file mode 100644 index 0000000000..383b455ca4 --- /dev/null +++ b/product/gradle-plugin/src/test/integration/data/AbiFilters/invalid_32bit/app/build.gradle @@ -0,0 +1,20 @@ +apply plugin: 'com.android.application' +apply plugin: 'com.chaquo.python' + +android { + namespace "com.chaquo.python.test" + compileSdk 23 + defaultConfig { + applicationId "com.chaquo.python.test" + minSdk 21 + targetSdk 23 + versionCode 1 + versionName "0.0.1" + python { + version "3.12" + } + ndk { + abiFilters "x86" + } + } +} diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle index 8c4e96b49d..4925805c37 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.10/app/build.gradle @@ -17,7 +17,7 @@ android { version "3.10" } ndk { - abiFilters "x86" + abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64" } } } diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle index 95c1d36950..cc9115e08b 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.11/app/build.gradle @@ -17,7 +17,7 @@ android { version "3.11" } ndk { - abiFilters "x86" + abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64" } } } diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle new file mode 100644 index 0000000000..07b9871dd6 --- /dev/null +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.12/app/build.gradle @@ -0,0 +1,23 @@ +plugins { + id 'com.android.application' + id 'com.chaquo.python' +} + +android { + namespace "com.chaquo.python.test" + compileSdk 23 + + defaultConfig { + applicationId "com.chaquo.python.test" + minSdk 21 + targetSdk 23 + versionCode 1 + versionName "0.0.1" + python { + version "3.12" + } + ndk { + abiFilters "arm64-v8a", "x86_64" + } + } +} diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle index 262f593743..7eca01021c 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.8/app/build.gradle @@ -17,7 +17,7 @@ android { version "3.8" } ndk { - abiFilters "x86" + abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64" } } } diff --git a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle index 419596af97..d38e482c86 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonVersion/3.9/app/build.gradle @@ -17,7 +17,7 @@ android { version "3.9" } ndk { - abiFilters "x86" + abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64" } } } diff --git a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py index bc74eb8a68..62fb46e58b 100644 --- a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py +++ b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py @@ -55,7 +55,7 @@ def list_versions(mode): for full_version in list_versions("micro").splitlines(): version = full_version.rpartition(".")[0] PYTHON_VERSIONS[version] = full_version -assert list(PYTHON_VERSIONS) == ["3.8", "3.9", "3.10", "3.11"] +assert list(PYTHON_VERSIONS) == ["3.8", "3.9", "3.10", "3.11", "3.12"] DEFAULT_PYTHON_VERSION_FULL = PYTHON_VERSIONS[DEFAULT_PYTHON_VERSION] NON_DEFAULT_PYTHON_VERSION = "3.10" @@ -398,18 +398,24 @@ class PythonVersion(GradleTestCase): # To allow a quick check of the setting, this test only covers two versions. def test_change(self): run = self.RunGradle("base", run=False) - for version in ["3.8", "3.9"]: + for version in [DEFAULT_PYTHON_VERSION, NON_DEFAULT_PYTHON_VERSION]: self.check_version(run, version) # Test all versions not covered by test_change. def test_others(self): run = self.RunGradle("base", run=False) - for version in ["3.10", "3.11"]: - self.check_version(run, version) + for version in PYTHON_VERSIONS: + if version not in [DEFAULT_PYTHON_VERSION, NON_DEFAULT_PYTHON_VERSION]: + self.check_version(run, version) def check_version(self, run, version): with self.subTest(version=version): - run.rerun(f"PythonVersion/{version}", python_version=version) + # Make sure every ABI has the full set of native stdlib module files. + abis = ["arm64-v8a", "x86_64"] + if version in ["3.8", "3.9", "3.10", "3.11"]: + abis += ["armeabi-v7a", "x86"] + run.rerun(f"PythonVersion/{version}", python_version=version, abis=abis) + if version == DEFAULT_PYTHON_VERSION: self.assertNotInLong(self.WARNING.format(".*"), run.stdout, re=True) else: @@ -441,9 +447,17 @@ def test_missing(self): def test_invalid(self): run = self.RunGradle("base", "AbiFilters/invalid", succeed=False) - self.assertInLong("Variant 'debug': Chaquopy does not support the ABI 'armeabi'. " - "Supported ABIs are [armeabi-v7a, arm64-v8a, x86, x86_64].", - run.stderr) + self.assertInLong( + "Variant 'debug': Python 3.8 is not available for the ABI 'armeabi'. " + "Supported ABIs are [arm64-v8a, armeabi-v7a, x86, x86_64].", + run.stderr) + + def test_invalid_32bit(self): + run = self.RunGradle("base", "AbiFilters/invalid_32bit", succeed=False) + self.assertInLong( + "Variant 'debug': Python 3.12 is not available for the ABI 'x86'. " + "Supported ABIs are [arm64-v8a, x86_64].", + run.stderr) def test_all(self): # Also tests making a change. run = self.RunGradle("base", abis=["x86"]) @@ -1734,21 +1748,27 @@ def check_assets(self, apk_dir, kwargs): with ZipFile(join(asset_dir, "bootstrap.imy")) as bootstrap_zip: self.check_pyc(bootstrap_zip, "java/__init__.pyc", kwargs) + python_version_info = tuple(int(x) for x in python_version.split(".")) + stdlib_bootstrap_expected = { + # This is the list from our minimum Python version. For why each of these + # modules is needed, see BOOTSTRAP_NATIVE_STDLIB in PythonTasks.kt. + "java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so", + "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so", + } + if python_version_info >= (3, 12): + stdlib_bootstrap_expected -= {"_sha512.so"} + stdlib_bootstrap_expected |= {"_sha2.so"} + bootstrap_native_dir = join(asset_dir, "bootstrap-native") self.test.assertCountEqual(abis, os.listdir(bootstrap_native_dir)) for abi in abis: abi_dir = join(bootstrap_native_dir, abi) - self.test.assertCountEqual( - # For why each of these modules are needed, see BOOTSTRAP_NATIVE_STDLIB - # in PythonTasks.kt. - ["java", "_bz2.so", "_ctypes.so", "_datetime.so", "_lzma.so", "_random.so", - "_sha512.so", "_struct.so", "binascii.so", "math.so", "mmap.so", "zlib.so"], - os.listdir(abi_dir)) - self.check_python_so(join(abi_dir, "_ctypes.so"), python_version, abi) + self.test.assertCountEqual(stdlib_bootstrap_expected, os.listdir(abi_dir)) + self.check_so(join(abi_dir, "_ctypes.so"), python_version, abi) java_dir = join(abi_dir, "java") self.test.assertCountEqual(["chaquopy.so"], os.listdir(java_dir)) - self.check_python_so(join(java_dir, "chaquopy.so"), python_version, abi) + self.check_so(join(java_dir, "chaquopy.so"), python_version, abi) # Python stdlib with ZipFile(join(asset_dir, "stdlib-common.imy")) as stdlib_zip: @@ -1777,7 +1797,6 @@ def check_assets(self, apk_dir, kwargs): "audioop.so", "cmath.so", "fcntl.so", "ossaudiodev.so", "parser.so", "pyexpat.so", "resource.so", "select.so", "syslog.so", "termios.so", "unicodedata.so", "xxlimited.so"} - python_version_info = tuple(int(x) for x in python_version.split(".")) if python_version_info >= (3, 9): stdlib_native_expected |= {"_zoneinfo.so"} if python_version_info >= (3, 10): @@ -1785,6 +1804,9 @@ def check_assets(self, apk_dir, kwargs): stdlib_native_expected |= {"xxlimited_35.so"} if python_version_info >= (3, 11): stdlib_native_expected |= {"_typing.so"} + if python_version_info >= (3, 12): + stdlib_native_expected -= {"_sha256.so", "_typing.so"} + stdlib_native_expected |= {"_xxinterpchannels.so", "xxsubtype.so"} for abi in abis: stdlib_native_zip = ZipFile(join(asset_dir, f"stdlib-{abi}.imy")) @@ -1793,7 +1815,7 @@ def check_assets(self, apk_dir, kwargs): with TemporaryDirectory() as tmp_dir: test_module = "_asyncio.so" stdlib_native_zip.extract(test_module, tmp_dir) - self.check_python_so(join(tmp_dir, test_module), python_version, abi) + self.check_so(join(tmp_dir, test_module), python_version, abi) # build.json with open(join(asset_dir, "build.json")) as build_json_file: @@ -1812,13 +1834,14 @@ def check_assets(self, apk_dir, kwargs): build_json["assets"]) def check_pyc(self, zip_file, pyc_filename, kwargs): - # See importlib._bootstrap_external.MAGIC_NUMBER. + # See the list in importlib/_bootstrap_external.py. MAGIC = { "3.7": 3394, "3.8": 3413, "3.9": 3425, "3.10": 3439, "3.11": 3495, + "3.12": 3531, } with zip_file.open(pyc_filename) as pyc_file: self.test.assertEqual( @@ -1836,9 +1859,9 @@ def check_lib(self, lib_dir, kwargs): f"libpython{kwargs['python_version']}.so", "libssl_chaquopy.so", "libsqlite3_chaquopy.so"], os.listdir(abi_dir)) - self.check_python_so(join(abi_dir, "libchaquopy_java.so"), python_version, abi) + self.check_so(join(abi_dir, "libchaquopy_java.so"), python_version, abi) - def check_python_so(self, so_filename, python_version, abi): + def check_so(self, so_filename, python_version, abi): libpythons = [] with open(so_filename, "rb") as so_file: ef = ELFFile(so_file) @@ -1853,7 +1876,11 @@ def check_python_so(self, so_filename, python_version, abi): if tag.entry.d_tag == "DT_NEEDED" and \ tag.needed.startswith("libpython"): libpythons.append(tag.needed) - self.test.assertEqual([f"libpython{python_version}.so"], libpythons) + + # Python 3.12 doesn't link its stdlib modules against libpython. But we'll make + # sure that *if* there's a libpython, it's the correct version. + if libpythons: + self.test.assertEqual([f"libpython{python_version}.so"], libpythons) def dump_run(self, msg): self.test.fail(msg + "\n" + diff --git a/product/runtime/build.gradle b/product/runtime/build.gradle index a3e828ea8b..50fa972f75 100644 --- a/product/runtime/build.gradle +++ b/product/runtime/build.gradle @@ -228,7 +228,10 @@ if (!(cmakeBuildType in KNOWN_BUILD_TYPES)) { "Unknown build type '$cmakeBuildType'; valid values are $KNOWN_BUILD_TYPES") } -(["host"] + Common.ABIS).each { abi -> +(["host"] + Common.supportedAbis("3.11")).each { abi -> + def pyVersions = Common.PYTHON_VERSIONS_SHORT.findAll { + Common.supportedAbis(it).contains(abi) + } def pyLibSuffix = ".so" def cmakeBuildSubdir = "$buildDir/cmake/$abi" def cmake = tasks.register("cmake-$abi", Exec) { @@ -273,11 +276,12 @@ if (!(cmakeBuildType in KNOWN_BUILD_TYPES)) { // GitHub Actions runner. Ideally it would also match the version in // target/build-common.sh, but the latter is more difficult to change. def ndkDir = sdkPath("ndk/26.1.10909125") + def prefixDir = "$projectDir/../../target/prefix/$abi" args "-DCMAKE_TOOLCHAIN_FILE=$ndkDir/build/cmake/android.toolchain.cmake", "-DANDROID_ABI=$abi", "-DANDROID_STL=system", "-DANDROID_NATIVE_API_LEVEL=$Common.MIN_SDK_VERSION", - "-DCHAQUOPY_PYTHON_VERSIONS=${Common.PYTHON_VERSIONS_SHORT.join(';')}", + "-DCHAQUOPY_PYTHON_VERSIONS=${pyVersions.join(';')}", "-DCHAQUOPY_INCLUDE_PYTHON=$prefixDir/include", "-DCHAQUOPY_LIB_DIRS=$prefixDir/lib" } @@ -294,7 +298,7 @@ if (!(cmakeBuildType in KNOWN_BUILD_TYPES)) { } if (abi != "host") { for (name in ["chaquopy", "libchaquopy_java"]) { - for (pyVersion in Common.PYTHON_VERSIONS_SHORT) { + for (pyVersion in pyVersions) { addArtifact( cmakeBuild, new File(cmakeBuildSubdir, "$name-${pyVersion}.so"), pyVersion, abi) diff --git a/product/runtime/requirements-build.txt b/product/runtime/requirements-build.txt index 44e1d680ff..a521eeabc1 100644 --- a/product/runtime/requirements-build.txt +++ b/product/runtime/requirements-build.txt @@ -1 +1 @@ -Cython==0.29.32 +Cython==0.29.36 diff --git a/release/bundle.sh b/release/bundle.sh old mode 100644 new mode 100755 index 44edc3ae87..8869824f50 --- a/release/bundle.sh +++ b/release/bundle.sh @@ -5,7 +5,7 @@ version="${1:?}" cd "$(dirname $(realpath $0))/../maven" -find -name $version | while read dir; do ( +find . -name $version | while read dir; do ( cd $dir # Maven Central adds hashes to every file in the bundle, even existing hash files. diff --git a/target/README.md b/target/README.md index cbe33eff66..fcba7d8e34 100644 --- a/target/README.md +++ b/target/README.md @@ -43,9 +43,9 @@ Add it to Common.java. Add it to build-all.sh. -In test_gradle_plugin.py: -* Update the `PYTHON_VERSIONS` assertion. -* Update `stdlib_native_expected`. +In test_gradle_plugin.py, update the `PYTHON_VERSIONS` assertion. + +Update the `MAGIC` lists in test_gradle_plugin.py and pyc.py. Update documentation: * "Python version" in android.rst