Fix recipes which assumed $CC ends with "gcc":

chaquopy-libogg chaquopy-flac chaquopy-geos chaquopy-hdf5 chaquopy-libjpeg chaquopy-libpng chaquopy-libraw chaquopy-libsndfile chaquopy-libvorbis chaquopy-libxml2 chaquopy-libxslt chaquopy-secp256k1 chaquopy-ta-lib gevent Confirmed all these packages now build on both armeabi-v7a and x86_64, but did no further testing.
chaquo · Sep 12, 2023 · 81c2529 · 81c2529
1 parent d1f3567
commit 81c2529
Show file tree

Hide file tree

Showing 18 changed files with 50 additions and 85 deletions.
diff --git a/server/pypi/build-wheel.py b/server/pypi/build-wheel.py
@@ -138,6 +138,9 @@ def unpack_and_build(self):
             ensure_empty(self.build_dir)
             self.unpack_source()
             self.apply_patches()
+            self.create_host_env()
+
+        self.update_env()
 
         # ProjectBuilder requires at least one of pyproject.toml or setup.py to exist,
         # which may not be the case for packages built using build.sh (e.g.
@@ -155,16 +158,12 @@ def unpack_and_build(self):
                 if not src_is_pyproject:
                     pyproject_toml.unlink()
 
-        if not self.no_unpack:
-            os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
-            if self.needs_python:
+            if not self.no_unpack:
                 self.create_build_env(src_is_pyproject)
-            self.create_host_env()
 
         if self.no_build:
             log("Skipping build due to --no-build")
         else:
-            self.update_env()
             self.create_dummy_libs()
             wheel_filename = self.build_wheel()
             self.fix_wheel(wheel_filename)
@@ -245,8 +244,8 @@ def pip_install(requirements):
             run(f"{bootstrap_env}/bin/pip --python {self.builder.python_executable} "
                 f"install " + " ".join(shlex.quote(req) for req in requirements))
 
-        # In the common case where get_requires_for_build only returns "wheel", which
-        # was already in build_system_requires, we can avoid running pip a second time.
+        # In the common case where get_requires_for_build only returns things which were
+        # already in build_system_requires, we can avoid running pip a second time.
         pip_install(build_reqs)
         if src_is_pyproject:
             pip_install(self.builder.get_requires_for_build("wheel") - set(build_reqs))
@@ -489,6 +488,7 @@ def update_env(self):
         # See env/bin/pkg-config.
         del env["PKG_CONFIG"]
 
+        compiler_vars = ["CC", "CXX", "LD"]
         if "fortran" in self.non_python_build_reqs:
             tool_prefix = ABIS[self.abi].tool_prefix
             toolchain = self.abi if self.abi in ["x86", "x86_64"] else tool_prefix
@@ -497,6 +497,7 @@ def update_env(self):
                 raise CommandError(f"This package requries a Fortran compiler, but "
                                    f"{gfortran} does not exist. See README.md.")
 
+            compiler_vars += ["FC", "F77", "F90"]
             env["FC"] = gfortran  # Used by OpenBLAS
             env["F77"] = env["F90"] = gfortran  # Used by numpy.distutils
             env["FARCH"] = env["CFLAGS"]  # Used by numpy.distutils
@@ -509,21 +510,17 @@ def update_env(self):
 
         # Wrap compiler and linker commands with a script which removes include and
         # library directories which are not in known safe locations.
-        for var in ["CC", "CXX", "FC", "F77", "F90", "LD"]:
-            try:
-                real_path = env[var]
-            except KeyError:
-                pass
-            else:
-                wrapper_path = join(ensure_dir(f"{self.build_env}/bin"),
-                                    basename(real_path))
-                with open(wrapper_path, "w") as wrapper_file:
-                    print(dedent(f"""\
-                        #!/bin/sh
-                        exec "{PYPI_DIR}/compiler-wrapper.py" "{real_path}" "$@"
-                        """), file=wrapper_file)
-                os.chmod(wrapper_path, 0o755)
-                env[var] = wrapper_path
+        for var in compiler_vars:
+            real_path = env[var]
+            wrapper_path = join(ensure_dir(f"{self.build_dir}/wrappers"),
+                                basename(real_path))
+            with open(wrapper_path, "w") as wrapper_file:
+                print(dedent(f"""\
+                    #!/bin/sh
+                    exec "{PYPI_DIR}/compiler-wrapper.py" "{real_path}" "$@"
+                    """), file=wrapper_file)
+            os.chmod(wrapper_path, 0o755)
+            env[var] = wrapper_path
 
         # Adding host_env to PYTHONPATH allows setup.py to import requirements, for example to
         # call numpy.get_include().
@@ -551,6 +548,7 @@ def update_env(self):
             assert_exists(self.python_lib)
             self.standard_libs.append(libpython)
 
+            env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
             env["CHAQUOPY_PYTHON"] = self.python
             # Use -idirafter so that package-specified -I directories take priority (e.g.
             # in grpcio and typed-ast).
@@ -566,6 +564,7 @@ def update_env(self):
             "HOST": ABIS[self.abi].tool_prefix,
 
             # Overrides sysconfig.get_platform and distutils.util.get_platform.
+            # TODO: consider replacing this with crossenv.
             "_PYTHON_HOST_PLATFORM": f"linux_{ABIS[self.abi].uname_machine}",
 
             # conda-build variable names defined at
@@ -586,9 +585,9 @@ def update_env(self):
             self.generate_cmake_toolchain(env)
 
         if self.verbose:
-            # Format variables so they can be pasted into a shell when troubleshooting.
             log("Environment set as follows:\n" +
-                "\n".join(f"export {key}='{value}'" for key, value in env.items()))
+                "\n".join(f"export {key}={shlex.quote(value)}"
+                          for key, value in env.items()))
         os.environ.update(env)
 
     def generate_cmake_toolchain(self, env):

diff --git a/server/pypi/env/lib/python/sitecustomize.py b/server/pypi/env/lib/python/sitecustomize.py
@@ -1,13 +1,16 @@
-# build-wheel sets PYTHONPATH to ensure this file is imported on startup in pip and all of
+# build-wheel sets PYTHONPATH to ensure this file is imported on startup in all of
 # its Python subprocesses.
 
 import os
+import shlex
 import sys
 
-from setuptools._distutils import sysconfig, util
+try:
+    from setuptools._distutils import sysconfig
+except ImportError:
+    from distutils import sysconfig
 
 
-# Override the CFLAGS from the build Python sysconfigdata file.
 # TODO: look into using crossenv to extract this from the Android sysconfigdata.
 sysconfig.get_config_vars()  # Ensure _config_vars has been initialized.
 sysconfig._config_vars["CFLAGS"] = \
@@ -21,7 +24,7 @@ def customize_compiler_override(compiler):
     customize_compiler_original(compiler)
     ldflags = os.environ["LDFLAGS"]
     if ldflags not in " ".join(compiler.linker_exe):
-        compiler.linker_exe += util.split_quoted(ldflags)
+        compiler.linker_exe += shlex.split(ldflags)
 
 sysconfig.customize_compiler = customize_compiler_override
 

diff --git a/server/pypi/packages/chaquopy-flac/build.sh b/server/pypi/packages/chaquopy-flac/build.sh
@@ -1,9 +1,7 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
-
-./configure --host=$HOST_TRIPLET --prefix=$PREFIX
+./configure --host=$HOST --prefix=$PREFIX
 make -j $CPU_COUNT
 make install
 

diff --git a/server/pypi/packages/chaquopy-geos/build.sh b/server/pypi/packages/chaquopy-geos/build.sh
@@ -1,10 +1,8 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
-
 config_args=""
-if [ "$CHAQUOPY_ABI" = "armeabi-v7a" ]; then
+if [ "$HOST" = "arm-linux-androideabi" ]; then
     # On this ABI only, we get the following linker errors:
     #
     # ld: error: noding/.libs/libnoding.a(BasicSegmentString.o): multiple definition of 'typeinfo for geos::noding::BasicSegmentString'
@@ -16,7 +14,7 @@ if [ "$CHAQUOPY_ABI" = "armeabi-v7a" ]; then
     config_args+=" --disable-inline"
 fi
 
-./configure --host=$HOST_TRIPLET --prefix=$PREFIX $config_args
+./configure --host=$HOST --prefix=$PREFIX $config_args
 make -j $CPU_COUNT
 make install
 

diff --git a/server/pypi/packages/chaquopy-hdf5/build.sh b/server/pypi/packages/chaquopy-hdf5/build.sh
@@ -21,8 +21,7 @@ fi
 # Some internal libraries can't be built with this flag.
 LDFLAGS=$(echo $LDFLAGS | sed 's/-Wl,--no-undefined//')
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
-./configure --host=$HOST_TRIPLET --prefix=$PREFIX
+./configure --host=$HOST --prefix=$PREFIX
 make -j $CPU_COUNT
 make install
 

diff --git a/server/pypi/packages/chaquopy-hdf5/meta.yaml b/server/pypi/packages/chaquopy-hdf5/meta.yaml
@@ -8,4 +8,4 @@ build:
   number: 1
 
 source:
-  url: https://github.com/live-clones/hdf5/archive/hdf5-{{ version | replace(".", "_") }}.tar.gz
+  url: https://github.com/hdfgroup/hdf5/archive/hdf5-{{ version | replace(".", "_") }}.tar.gz
diff --git a/server/pypi/packages/chaquopy-libjpeg/build.sh b/server/pypi/packages/chaquopy-libjpeg/build.sh
@@ -1,9 +1,8 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
 # SIMD is only available for x86, so disable for consistency between ABIs.
-./configure --host=$HOST_TRIPLET --without-turbojpeg --without-simd
+./configure --host=$HOST --without-turbojpeg --without-simd
 make -j $CPU_COUNT
 make install prefix=$PREFIX
 

diff --git a/server/pypi/packages/chaquopy-libogg/build.sh b/server/pypi/packages/chaquopy-libogg/build.sh
@@ -1,9 +1,7 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
-
-./configure --host=$HOST_TRIPLET --prefix=$PREFIX
+./configure --host=$HOST --prefix=$PREFIX
 make -j $CPU_COUNT
 make install
 

diff --git a/server/pypi/packages/chaquopy-libpng/build.sh b/server/pypi/packages/chaquopy-libpng/build.sh
@@ -1,8 +1,7 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
-./configure --host=$HOST_TRIPLET
+./configure --host=$HOST
 make -j $CPU_COUNT
 make install prefix=$PREFIX
 

diff --git a/server/pypi/packages/chaquopy-libraw/build.sh b/server/pypi/packages/chaquopy-libraw/build.sh
@@ -1,9 +1,10 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
+# https://stackoverflow.com/a/33279062
+touch aclocal.m4 configure Makefile.am Makefile.in
 
-./configure --host=$HOST_TRIPLET --disable-static --disable-openmp --disable-examples
+./configure --host=$HOST --disable-static --disable-openmp --disable-examples
 make -j $CPU_COUNT
 make install prefix=$PREFIX
 

diff --git a/server/pypi/packages/chaquopy-libraw/patches/chaquopy.patch b/server/pypi/packages/chaquopy-libraw/patches/chaquopy.patch
@@ -1,30 +1,12 @@
 --- src-original/libraw/libraw.h	2020-10-15 05:06:07.000000000 +0000
 +++ src/libraw/libraw.h	2021-10-09 10:13:52.377222725 +0000
-@@ -21,7 +21,26 @@
+@@ -21,7 +21,8 @@
  #define _LIBRAW_CLASS_H
 
  #ifdef __linux__
 -#define _FILE_OFFSET_BITS 64
 +// Chaquopy disabled: see https://android.googlesource.com/platform/bionic/+/master/docs/32-bit-abi.md
 +// #define _FILE_OFFSET_BITS 64
-+#endif
-+
-+// Chaquopy: based on bionic/libc/bionic/swab.cpp.
-+#if __ANDROID_API__ < 28
-+#include <stdint.h>
-+#include <sys/types.h>
-+
-+static void swab(const void* void_src, void* void_dst, ssize_t byte_count) {
-+  const uint8_t* src = (uint8_t*)void_src;
-+  uint8_t* dst = (uint8_t*)void_dst;
-+  while (byte_count > 1) {
-+    uint8_t x = *src++;
-+    uint8_t y = *src++;
-+    *dst++ = y;
-+    *dst++ = x;
-+    byte_count -= 2;
-+  }
-+}
  #endif
 
  /* maximum file size to use LibRaw_file_datastream (fully buffered) I/O */

diff --git a/server/pypi/packages/chaquopy-libsndfile/build.sh b/server/pypi/packages/chaquopy-libsndfile/build.sh
@@ -1,9 +1,7 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
-
-./configure --host=$HOST_TRIPLET --prefix=$PREFIX
+./configure --host=$HOST --prefix=$PREFIX
 make -j $CPU_COUNT
 make install
 

diff --git a/server/pypi/packages/chaquopy-libvorbis/build.sh b/server/pypi/packages/chaquopy-libvorbis/build.sh
@@ -1,9 +1,7 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
-
-./configure --host=$HOST_TRIPLET --prefix=$PREFIX
+./configure --host=$HOST --prefix=$PREFIX
 make -j $CPU_COUNT
 make install
 

diff --git a/server/pypi/packages/chaquopy-libxml2/build.sh b/server/pypi/packages/chaquopy-libxml2/build.sh
@@ -1,8 +1,7 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
-./configure --host=$HOST_TRIPLET --prefix=$PREFIX --without-python
+./configure --host=$HOST --prefix=$PREFIX --without-python
 make -j $CPU_COUNT
 make install
 

diff --git a/server/pypi/packages/chaquopy-libxslt/build.sh b/server/pypi/packages/chaquopy-libxslt/build.sh
@@ -1,8 +1,7 @@
 #!/bin/bash
 set -eu
 
-host_triplet=$(basename $CC | sed 's/-gcc$//')
-./configure --host=$host_triplet --prefix=$PREFIX --without-python
+./configure --host=$HOST --prefix=$PREFIX --without-crypto --without-python
 make -j $CPU_COUNT V=1
 make install
 

diff --git a/server/pypi/packages/chaquopy-secp256k1/build.sh b/server/pypi/packages/chaquopy-secp256k1/build.sh
@@ -1,11 +1,9 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
-
 # Configure command based on coincurve's setup.py.
 ./autogen.sh
-./configure --host=$HOST_TRIPLET --enable-shared --disable-static \
+./configure --host=$HOST --enable-shared --disable-static \
             --disable-dependency-tracking --with-pic --enable-module-recovery --disable-jni \
             --enable-experimental --enable-module-ecdh --enable-benchmark=no
 make -j $CPU_COUNT

diff --git a/server/pypi/packages/chaquopy-ta-lib/build.sh b/server/pypi/packages/chaquopy-ta-lib/build.sh
@@ -1,8 +1,7 @@
 #!/bin/bash
 set -eu
 
-HOST_TRIPLET=$(basename $CC | sed 's/-gcc$//')
-./configure --host=$HOST_TRIPLET --prefix=$PREFIX
+./configure --host=$HOST --prefix=$PREFIX
 make  # The build breaks with -j.
 make install
 

diff --git a/server/pypi/packages/gevent/patches/chaquopy.patch b/server/pypi/packages/gevent/patches/chaquopy.patch
@@ -1,19 +1,17 @@
 diff -ur src-original/_setuplibev.py src/_setuplibev.py
 --- src-original/_setuplibev.py	2019-01-04 11:51:44.000000000 +0000
 +++ src/_setuplibev.py	2019-08-05 14:16:18.730590045 +0000
-@@ -29,9 +29,14 @@
+@@ -29,9 +29,12 @@
  # the build/temp.XXX/libev/ directory. If we're building from a
  # source checkout on pypy, OLDPWD will be the location of setup.py
  # and the PyPy branch will clean it up.
 +
 +# Chaquopy: added --host.
 +import os
-+host_triplet = os.path.basename(os.environ["CC"]).replace("-gcc", "")
-+
  libev_configure_command = ' '.join([
      "(cd ", quoted_dep_abspath('libev'),
 -    " && sh ./configure ",
-+    " && sh ./configure --host=" + host_triplet,
++    " && sh ./configure --host=" + os.environ["HOST"],
      " && cp config.h \"$OLDPWD\"",
      ")",
      '> configure-output.txt'