Merge pull request #937 from pasdeloup/pr-fix_scipy

Fixes to build Scipy

server/pypi/.gitignore

@@ -1,2 +1,3 @@
 /build
+/fortran
 /packages/*/build

server/pypi/README.md

@@ -51,7 +51,17 @@ Use your distribution's package manager to install the following build tools:
 * patch
 * patchelf
 
-Depending on which package you're building, you may also need additional tools.
+Depending on which package you're building, you may also need additional tools. Most of
+these can be installed using your distribution. Some of them have special entries in the
+`build` requirements section of meta.yaml:
+
+* `cmake`: A `chaquopy.toolchain.cmake` file will be generated in the build directory
+  for use with `-DCMAKE_TOOLCHAIN_FILE`.
+
+* `fortran`: You must install the Fortran compiler from
+  [here](https://github.com/mzakharo/android-gfortran/releases/tag/r21e). Create a
+  `fortran` subdirectory in the same directory as this README, and unpack the .bz2 files
+  into it.
 
 
 ## Building a package

server/pypi/build-wheel.py

@@ -77,10 +77,14 @@ def main(self):
             self.name_version = (normalize_name_wheel(self.package) + "-" +
                                  normalize_version(self.version))
 
-            self.needs_cmake = False
-            if "cmake" in self.meta["requirements"]["build"]:
-                self.meta["requirements"]["build"].remove("cmake")
-                self.needs_cmake = True
+            self.non_python_build_reqs = set()
+            for name in ["cmake", "fortran"]:
+                try:
+                    self.meta["requirements"]["build"].remove(name)
+                except ValueError:
+                    pass
+                else:
+                    self.non_python_build_reqs.add(name)
 
             self.needs_python = self.needs_target = (self.meta["source"] == "pypi")
             for name in ["openssl", "python", "sqlite"]:
@@ -460,6 +464,18 @@ def update_env(self):
         # See env/bin/pkg-config.
         del env["PKG_CONFIG"]
 
+        if "fortran" in self.non_python_build_reqs:
+            tool_prefix = ABIS[self.abi].tool_prefix
+            toolchain = self.abi if self.abi in ["x86", "x86_64"] else tool_prefix
+            gfortran = f"{PYPI_DIR}/fortran/{toolchain}-4.9/bin/{tool_prefix}-gfortran"
+            if not exists(gfortran):
+                raise CommandError(f"This package requries a Fortran compiler, but "
+                                   f"{gfortran} does not exist. See README.md.")
+
+            env["FC"] = gfortran  # Used by OpenBLAS
+            env["F77"] = env["F90"] = gfortran  # Used by numpy.distutils
+            env["FARCH"] = env["CFLAGS"]  # Used by numpy.distutils
+
         env_dir = f"{PYPI_DIR}/env"
         env["PATH"] = os.pathsep.join([
             f"{env_dir}/bin",
@@ -523,7 +539,7 @@ def update_env(self):
             key, value = var.split("=")
             env[key] = value
 
-        if self.needs_cmake:
+        if "cmake" in self.non_python_build_reqs:
             self.generate_cmake_toolchain(env)
 
         if self.verbose:

server/pypi/env/lib/python/sitecustomize.py

@@ -32,8 +32,10 @@ def run_command_override(self, command):
 # Remove include and library directories which are not in known safe locations.
 # Monkey-patching at this level handles both default paths added by distutils itself,
 # and paths added explicitly by package build scripts.
+# We must also allow include directories from env/ dir 
+# as they contain pybind11/pybind11.h headers needed to build some packages like scipy
 src_dir = os.environ["SRC_DIR"]
-valid_dirs = [abspath(path) for path in [src_dir, f"{src_dir}/../requirements"]]
+valid_dirs = [abspath(path) for path in [src_dir, f"{src_dir}/../requirements", f"{src_dir}/../env"]]
 
 def filter_dirs(dir_type, dirs):
     result = []

server/pypi/meta-schema.yaml

@@ -63,9 +63,9 @@ properties:
 
       # Requirements which must be installed in the build environment. One of the following:
       #
-      # * `<package> <version>`: A Python package.
-      # * `cmake`: indicates that CMake is used in the build. A `chaquopy.toolchain.cmake` file
-      #   will be generated in the build directory for use with `-DCMAKE_TOOLCHAIN_FILE`.
+      #   * `cmake`: see README.md
+      #   * `fortran`: see README.md
+      #   * `<package> <version>`: A Python package, which will be installed using pip
       build:
         type: array
         default: []

server/pypi/packages/chaquopy-openblas/meta.yaml

@@ -9,5 +9,7 @@ build:
   number: 5
 
 requirements:
+  build:
+    - fortran
   host:
     - chaquopy-libgfortran 4.9

server/pypi/packages/scipy/meta.yaml

@@ -1,13 +1,20 @@
+{% if PY_VER == "3.8" %}
+    {% set numpy_version = "1.19.5" %}
+{% else %}
+    {% set numpy_version = "1.23.3" %}
+{% endif %}
+
 package:
   name: scipy
   version: 1.4.1
 
 requirements:
   build:
+    - fortran
     # setup.py has pybind11 in setup_requires, but this doesn't work because the headers aren't
     # unpacked correctly (https://github.com/pybind/python_example/issues/16).
     - pybind11 2.4.3
   host:
     - chaquopy-libgfortran 4.9
     - chaquopy-openblas 0.2.20
-    - numpy 1.17.4
+    - numpy {{ numpy_version }}

target/build-common.sh

@@ -7,10 +7,17 @@
 : ${api_level:=21}  # Should match MIN_SDK_VERSION in Common.java.
 
 # When moving to a new version of the NDK, carefully review the following:
+#
 # * The release notes (https://developer.android.com/ndk/downloads/revision_history)
+#
 # * https://android.googlesource.com/platform/ndk/+/ndk-release-rXX/docs/BuildSystemMaintainers.md,
 #   where XX is the NDK version. Do a diff against the version you're upgrading from.
+#
+# * According to https://github.com/kivy/python-for-android/pull/2615, the mzakharo
+#   build of gfortran is not compatible with NDK version 23, which is the version in
+#   which they removed the GNU binutils.
 ndk_version=22.1.7171670  # See ndkDir in product/runtime/build.gradle.
+
 ndk=${ANDROID_HOME:?}/ndk/$ndk_version
 if ! [ -e $ndk ]; then
     # Print all messages on stderr so they're visible when running within build-wheel.