Require meta.yaml 'cmake' requirements to have a version number:

The recipes updated in this commit have been built successfully for Python 3.8 and armeabi-v7a, but not tested. The other CMake-based recipes have various problems such as #790.

server/pypi/README.md

@@ -55,9 +55,6 @@ Depending on which package you're building, you may also need additional tools.
 these can be installed using your distribution. Some of them have special entries in the
 `build` requirements section of meta.yaml:
 
-* `cmake`: A `chaquopy.toolchain.cmake` file will be generated in the build directory
-  for use with `-DCMAKE_TOOLCHAIN_FILE`.
-
 * `fortran`: You must install the Fortran compiler from
   [here](https://github.com/mzakharo/android-gfortran/releases/tag/r21e). Create a
   `fortran` subdirectory in the same directory as this README, and unpack the .bz2 files

server/pypi/build-wheel.py

@@ -90,6 +90,12 @@ def main(self):
                     pass
                 else:
                     self.non_python_build_reqs.add(name)
+                    if name == "cmake":
+                        raise CommandError(
+                            "meta.yaml 'cmake' requirements must now have a version "
+                            "number. If you specify version 3.21 or later, you can "
+                            "probably remove references to CMAKE_TOOLCHAIN_FILE from "
+                            "the recipe.")
 
             self.needs_python = self.needs_target = (self.meta["source"] == "pypi")
             for name in ["openssl", "python", "sqlite"]:
@@ -163,8 +169,8 @@ def unpack_and_build(self):
                 if not src_is_pyproject:
                     pyproject_toml.unlink()
 
-            if not self.no_unpack:
-                self.create_build_env()
+        if not self.no_unpack:
+            self.create_build_env()
 
         if self.no_build:
             log("Skipping build due to --no-build")
@@ -235,37 +241,41 @@ def find_target(self):
         self.target_zip = zips[0]
 
     def create_build_env(self):
+        python_ver = self.python or ".".join(map(str, sys.version_info[:2]))
+
         # Installing Python's bundled pip and setuptools into a new environment takes
         # about 3.5 seconds on Python 3.8, and 6 seconds on Python 3.11. To avoid this,
         # we create one bootstrap environment per Python version, shared between all
         # packages, and use that to install the build environments.
         os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
-        bootstrap_env = self.get_bootstrap_env()
+        bootstrap_env = self.get_bootstrap_env(python_ver)
         ensure_empty(self.build_env)
-        run(f"python{self.python} -m venv --without-pip {self.build_env}")
+        run(f"python{python_ver} -m venv --without-pip {self.build_env}")
 
         # In case meta.yaml and pyproject.toml have requirements for the same package,
         # listing the more specific requirements first will help pip find a solution
         # faster.
-        build_reqs = ([f"{package}=={version}"
-                       for package, version in self.get_requirements("build")]
-                      + list(self.builder.build_system_requires))
+        build_reqs = [f"{package}=={version}"
+                      for package, version in self.get_requirements("build")]
+        if self.needs_python:
+            build_reqs += list(self.builder.build_system_requires)
 
         def pip_install(requirements):
             if not requirements:
                 return
-            run(f"{bootstrap_env}/bin/pip --python {self.builder.python_executable} "
+            run(f"{bootstrap_env}/bin/pip --python {self.build_env}/bin/python "
                 f"install " + " ".join(shlex.quote(req) for req in requirements))
 
         # In the common case where get_requires_for_build only returns things which were
         # already in build_system_requires, we can avoid running pip a second time.
         pip_install(build_reqs)
-        with self.env_vars():
-            requires_for_build = self.builder.get_requires_for_build("wheel")
-        pip_install(requires_for_build - set(build_reqs))
+        if self.needs_python:
+            with self.env_vars():
+                requires_for_build = self.builder.get_requires_for_build("wheel")
+            pip_install(requires_for_build - set(build_reqs))
 
-    def get_bootstrap_env(self):
-        bootstrap_env = f"{PYPI_DIR}/build/_bootstrap/{self.python}"
+    def get_bootstrap_env(self, python_ver):
+        bootstrap_env = f"{PYPI_DIR}/build/_bootstrap/{python_ver}"
         pip_version = "23.2.1"
 
         def check_bootstrap_env():
@@ -283,7 +293,7 @@ def check_bootstrap_env():
                 log("Invalid bootstrap environment: recreating it")
                 ensure_empty(bootstrap_env)
 
-        run(f"python{self.python} -m venv {bootstrap_env}")
+        run(f"python{python_ver} -m venv {bootstrap_env}")
         run(f"{bootstrap_env}/bin/pip install pip=={pip_version}")
         check_bootstrap_env()
         return bootstrap_env
@@ -548,6 +558,8 @@ def get_common_env_vars(self, env):
     def get_python_env_vars(self, env, pypi_env):
         # Adding host_env to PYTHONPATH allows setup.py to import requirements, for
         # example to call numpy.get_include().
+        #
+        # build_env is included implicitly, but at a lower priority.
         env["PYTHONPATH"] = os.pathsep.join([f"{pypi_env}/lib/python", self.host_env])
         env["CHAQUOPY_PYTHON"] = self.python
 
@@ -610,8 +622,9 @@ def env_vars(self):
             key, value = var.split("=")
             env[key] = value
 
-        if "cmake" in self.non_python_build_reqs:
-            self.generate_cmake_toolchain(env)
+        # We do this unconditionally, because we don't know whether the package requires
+        # CMake (it may be listed in the pyproject.toml build-system requires).
+        self.generate_cmake_toolchain(env)
 
         if self.verbose:
             log("Environment set as follows:\n" +
@@ -633,12 +646,9 @@ def generate_cmake_toolchain(self, env):
         ndk = abspath(f"{env['AR']}/../../../../../..")
         toolchain_filename = join(self.build_dir, "chaquopy.toolchain.cmake")
 
-        # This environment variable requires CMake 3.21 or later, so until we can rely on
-        # that being available, we'll still need to patch packages to pass it on the
-        # command line.
+        # This environment variable requires CMake 3.21 or later.
         env["CMAKE_TOOLCHAIN_FILE"] = toolchain_filename
 
-        log(f"Generating {toolchain_filename}")
         with open(toolchain_filename, "w") as toolchain_file:
             print(dedent(f"""\
                 set(ANDROID_ABI {self.abi})

server/pypi/meta-schema.yaml

@@ -63,7 +63,6 @@ properties:
 
       # Requirements which must be installed in the build environment. One of the following:
       #
-      #   * `cmake`: see README.md
       #   * `fortran`: see README.md
       #   * `<package> <version>`: A Python package, which will be installed using pip
       build:

server/pypi/packages/chaquopy-crc32c/build.sh

@@ -6,7 +6,6 @@ cd build
 rm -f CMakeCache.txt  # For rerunning with build-wheel.py --no-unpack.
 
 cmake .. -DCRC32C_BUILD_TESTS=0 -DCRC32C_BUILD_BENCHMARKS=0 -DCRC32C_USE_GLOG=0 \
-      -DCMAKE_TOOLCHAIN_FILE="$SRC_DIR/../chaquopy.toolchain.cmake" \
       -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=1 -DCMAKE_INSTALL_PREFIX="$PREFIX"
 
 make -j $(nproc)

server/pypi/packages/chaquopy-crc32c/meta.yaml

@@ -7,10 +7,13 @@ package:
   name: chaquopy-crc32c
   version: {{ version }}
 
+build:
+  number: 1
+
 source:
   git_url: https://github.com/google/crc32c
   git_rev: {{ version }}
 
 requirements:
   build:
-    - cmake
+    - cmake 3.28.1

server/pypi/packages/chaquopy-proj/build.sh

@@ -3,7 +3,7 @@ set -eu
 
 mkdir build
 cd build
-cmake .. -DCMAKE_TOOLCHAIN_FILE="$SRC_DIR/../chaquopy.toolchain.cmake" \
+cmake .. \
       -DCMAKE_INSTALL_PREFIX="$PREFIX" \
       -DBUILD_TESTING=OFF
 cmake --build . -j $CPU_COUNT

server/pypi/packages/chaquopy-proj/meta.yaml

@@ -11,14 +11,14 @@ package:
   version: {{ version }}
 
 build:
-  number: 1
+  number: 2
 
 source:
   url: https://download.osgeo.org/proj/proj-{{ version }}.tar.gz
 
 requirements:
   build:
-    - cmake
+    - cmake 3.28.1
   host:
     - chaquopy-libtiff 4.5.0
     - chaquopy-curl{{ openssl_suffix }} 7.76.1

server/pypi/packages/sentencepiece/bundled.patch

@@ -0,0 +1,36 @@
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index f7e4d50..3aba9f3 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -61,6 +61,12 @@ if (SPM_USE_BUILTIN_PROTOBUF)
+     ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/zero_copy_stream.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/zero_copy_stream_impl.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite/zero_copy_stream_impl_lite.cc)
++
++  # Chaquopy
++  if (ANDROID)
++    list(APPEND SPM_LIBS log)  # Used by protobuf
++  endif()
++
+   if (MSVC)
+     add_definitions("/DHAVE_PTHREAD /wd4018 /wd4514")
+   else()
+diff --git a/src/util.h b/src/util.h
+index 18d6e9c..3f94c9a 100644
+--- a/src/util.h
++++ b/src/util.h
+@@ -36,9 +36,13 @@
+ #include <pthread.h>
+ #endif
+
+-#if !defined(__APPLE__) && !defined(_WIN32) && BYTE_ORDER == __BIG_ENDIAN
++// Chaquopy: add missing #include
++#if !defined(__APPLE__) && !defined(_WIN32)
++#include <endian.h>
++#if defined(BYTE_ORDER) && defined(__BIG_ENDIAN) && BYTE_ORDER == __BIG_ENDIAN
+ #define IS_BIG_ENDIAN
+ #endif
++#endif
+
+ namespace sentencepiece {
+ template <typename T>

server/pypi/packages/sentencepiece/meta.yaml

@@ -3,11 +3,11 @@ package:
   version: "0.1.95"
 
 build:
-  number: 2
+  number: 3
 
 requirements:
   build:
-    - cmake
+    - cmake 3.28.1
 
 about:
-  license_file: bundled/sentencepiece/LICENSE
+  license_file: bundled/sentencepiece/LICENSE

server/pypi/packages/sentencepiece/patches/chaquopy.patch

@@ -11,12 +11,8 @@
 
  setup(
 --- src-original/build_bundled.sh	2021-01-09 15:24:24.000000000 +0000
-+++ src/build_bundled.sh	2021-01-12 13:35:07.595453948 +0000
-@@ -1,18 +1,20 @@
- #!/bin/sh
-
- VERSION="$1"
-+export LDFLAGS="$LDFLAGS -llog"  # Chaquopy
++++ src/build_bundled.sh	2023-12-16 22:18:22.971159515 +0000
+@@ -4,12 +4,12 @@
 
  mkdir bundled
  cd bundled
@@ -28,33 +24,7 @@
 +  -b v"${VERSION}" --depth 1
 
  cd sentencepiece
-+patch -p1 -i $SRC_DIR/util.patch  # Chaquopy: fix missing #include.
++patch -p1 -i $RECIPE_DIR/bundled.patch  # Chaquopy
  mkdir build
  cd build
--cmake .. -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=../..
-+cmake .. -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=../.. \
-+      -DCMAKE_TOOLCHAIN_FILE=$SRC_DIR/../chaquopy.toolchain.cmake  # Chaquopy
- make -j $(nproc)
- make install
- cd ../..
---- src-original/util.patch	1970-01-01 00:00:00.000000000 +0000
-+++ src/util.patch	2021-01-12 13:30:53.587344655 +0000
-@@ -0,0 +1,18 @@
-+diff --git a/src/util.h b/src/util.h
-+index 18d6e9c..08c2e00 100644
-+--- a/src/util.h
-++++ b/src/util.h
-+@@ -36,9 +36,12 @@
-+ #include <pthread.h>
-+ #endif
-+ 
-+-#if !defined(__APPLE__) && !defined(_WIN32) && BYTE_ORDER == __BIG_ENDIAN
-++#if !defined(__APPLE__) && !defined(_WIN32)
-++#include <endian.h>
-++#if defined(BYTE_ORDER) && defined(__BIG_ENDIAN) && BYTE_ORDER == __BIG_ENDIAN
-+ #define IS_BIG_ENDIAN
-+ #endif
-++#endif
-+ 
-+ namespace sentencepiece {
-+ template <typename T>
+ cmake .. -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=../..