Merge pull request #949 from pasdeloup/pr-scipy_1.5.4

Update SciPy version

server/pypi/build-wheel.py

@@ -507,6 +507,24 @@ def update_env(self):
             f"{self.host_env}/chaquopy/bin",  # For "-config" scripts.
             os.environ["PATH"]])
 
+        # Wrap compiler and linker commands with a script which removes include and
+        # library directories which are not in known safe locations.
+        for var in ["CC", "CXX", "FC", "F77", "F90", "LD"]:
+            try:
+                real_path = env[var]
+            except KeyError:
+                pass
+            else:
+                wrapper_path = join(ensure_dir(f"{self.build_env}/bin"),
+                                    basename(real_path))
+                with open(wrapper_path, "w") as wrapper_file:
+                    print(dedent(f"""\
+                        #!/bin/sh
+                        exec "{PYPI_DIR}/compiler-wrapper.py" "{real_path}" "$@"
+                        """), file=wrapper_file)
+                os.chmod(wrapper_path, 0o755)
+                env[var] = wrapper_path
+
         # Adding host_env to PYTHONPATH allows setup.py to import requirements, for example to
         # call numpy.get_include().
         pythonpath = [f"{env_dir}/lib/python", self.host_env]
@@ -574,7 +592,7 @@ def update_env(self):
         os.environ.update(env)
 
     def generate_cmake_toolchain(self, env):
-        ndk = abspath(f"{env['CC']}/../../../../../..")
+        ndk = abspath(f"{env['AR']}/../../../../../..")
         toolchain_filename = join(self.build_dir, "chaquopy.toolchain.cmake")
 
         # This environment variable requires CMake 3.21 or later, so until we can rely on

server/pypi/compiler-wrapper.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+#
+# Filter compiler command line to remove include and library directories which are not
+# in known safe locations.
+
+import os
+from os.path import abspath, commonpath, dirname
+import sys
+
+
+# This covers the source directory, the host environment and the build environment.
+valid_dirs = [abspath(f"{dirname(sys.argv[0])}/../..")]
+
+def is_valid(dir, prefix):
+    if any(commonpath([vd, abspath(dir)]) == vd for vd in valid_dirs):
+        return True
+    else:
+        print(f"Chaquopy: ignored invalid {prefix} directory: {dir!r}", file=sys.stderr)
+        return False
+
+
+def main():
+    args_in = sys.argv[1:]
+    args_out = []
+
+    i = 0
+    while i < len(args_in):
+        arg = args_in[i]
+        for prefix in ["-I", "-L"]:
+            if arg.startswith(prefix):
+                if arg == prefix:
+                    i += 1
+                    dir = args_in[i]
+                    if is_valid(dir, prefix):
+                        args_out += [arg, dir]
+                else:
+                    dir = arg[2:]
+                    if is_valid(dir, prefix):
+                        args_out.append(arg)
+                break
+        else:
+            args_out.append(arg)
+
+        i += 1
+
+    os.execv(args_out[0], args_out)
+
+
+if __name__ == "__main__":
+    main()

server/pypi/env/lib/python/sitecustomize.py

@@ -1,67 +1,10 @@
 # build-wheel sets PYTHONPATH to ensure this file is imported on startup in pip and all of
 # its Python subprocesses.
-#
-# build-wheel currently disables all pyproject.toml files by renaming them. At some point
-# we'll have to stop doing this in order to enable PEP 517 builds. However, pip overrides
-# PYTHONPATH when launching a PEP 517 build, which will prevent this file from taking
-# effect in the subprocess. So we may need to use a different approach, perhaps even using
-# a different PEP 517 front end like pypa/build.
 
 import os
-from os.path import abspath, commonpath
 import sys
 
-from setuptools._distutils import ccompiler, dist, sysconfig, util
-
-
-# --no-clean currently has no effect when running `pip wheel`
-# (https://github.com/pypa/pip/issues/5661), so disable the clean command to prevent it
-# destroying the evidence after a build failure. Monkey-patching at this level also handles
-# packages overriding the `clean` command using `cmdclass.`
-run_command_original = dist.Distribution.run_command
-
-def run_command_override(self, command):
-    if command == "clean":
-        print("Chaquopy: clean command disabled")
-    else:
-        run_command_original(self, command)
-
-dist.Distribution.run_command = run_command_override
-
-
-# Remove include and library directories which are not in known safe locations.
-# Monkey-patching at this level handles both default paths added by distutils itself,
-# and paths added explicitly by package build scripts.
-# We must also allow include directories from env/ dir 
-# as they contain pybind11/pybind11.h headers needed to build some packages like scipy
-src_dir = os.environ["SRC_DIR"]
-valid_dirs = [abspath(path) for path in [src_dir, f"{src_dir}/../requirements", f"{src_dir}/../env"]]
-
-def filter_dirs(dir_type, dirs):
-    result = []
-    for dir in dirs:
-        if any(commonpath([vd, abspath(dir)]) == vd for vd in valid_dirs):
-            result.append(dir)
-        else:
-            print(f"Chaquopy: ignored invalid {dir_type} directory: {dir!r}")
-    return result
-
-
-gen_preprocess_options_original = ccompiler.gen_preprocess_options
-
-def gen_preprocess_options_override(macros, include_dirs):
-    return gen_preprocess_options_original(macros, filter_dirs("include", include_dirs))
-
-ccompiler.gen_preprocess_options = gen_preprocess_options_override
-
-
-gen_lib_options_original = ccompiler.gen_lib_options
-
-def gen_lib_options_override(compiler, library_dirs, runtime_library_dirs, libraries):
-    return gen_lib_options_original(
-        compiler, filter_dirs("library", library_dirs), runtime_library_dirs, libraries)
-
-ccompiler.gen_lib_options = gen_lib_options_override
+from setuptools._distutils import sysconfig, util
 
 
 # Override the CFLAGS from the build Python sysconfigdata file.

server/pypi/packages/scipy/meta.yaml

@@ -1,20 +1,11 @@
-{% if PY_VER == "3.8" %}
-    {% set numpy_version = "1.19.5" %}
-{% else %}
-    {% set numpy_version = "1.23.3" %}
-{% endif %}
-
 package:
   name: scipy
-  version: "1.4.1"
+  version: "1.6.3"
 
 requirements:
   build:
     - fortran
-    # setup.py has pybind11 in setup_requires, but this doesn't work because the headers aren't
-    # unpacked correctly (https://github.com/pybind/python_example/issues/16).
-    - pybind11 2.4.3
   host:
     - chaquopy-libgfortran 4.9
     - chaquopy-openblas 0.2.20
-    - numpy {{ numpy_version }}
+    - numpy 1.20.3

server/pypi/packages/scipy/patches/chaquopy.patch

@@ -1,16 +1,16 @@
---- src-original/scipy/__init__.py	2019-12-18 19:27:46.000000000 +0000
-+++ src/scipy/__init__.py	2020-01-29 16:07:52.673532778 +0000
-@@ -60,59 +60,60 @@
-
+diff -Naur src-original/scipy/__init__.py src/scipy/__init__.py
+--- src-original/scipy/__init__.py	2023-08-31 12:17:17.197651190 +0200
++++ src/scipy/__init__.py	2023-08-31 12:17:39.438111356 +0200
+@@ -58,50 +58,54 @@
+ """
  __all__ = ['test']
 
 -from numpy import show_config as show_numpy_config
 -if show_numpy_config is None:
 -    raise ImportError(
--        "Cannot import scipy when running from numpy source directory.")
-+# Chaquopy: removed `from numpy import show_config` test
- from numpy import __version__ as __numpy_version__
-
+-        "Cannot import SciPy when running from NumPy source directory.")
+-from numpy import __version__ as __numpy_version__
+-
 -# Import numpy symbols to scipy name space (DEPRECATED)
 -from ._lib.deprecation import _deprecated
 -import numpy as _num
@@ -28,14 +28,8 @@
 -        'use numpy.random.{0} instead')
 -rand = _deprecated(_msg.format('rand'))(rand)
 -randn = _deprecated(_msg.format('randn'))(randn)
--from numpy.fft import fft, ifft
--# fft is especially problematic, so we deprecate it with a shorter window
--fft_msg = ('Using scipy.fft as a function is deprecated and will be '
--           'removed in SciPy 1.5.0, use scipy.fft.fft instead.')
--# for wrapping in scipy.fft.__call__, so the stacklevel is one off from the
--# usual (2)
--_dep_fft = _deprecated(fft_msg, stacklevel=3)(fft)
--fft = _deprecated(fft_msg)(fft)
+-# fft is especially problematic, so was removed in SciPy 1.6.0
+-from numpy.fft import ifft
 -ifft = _deprecated('scipy.ifft is deprecated and will be removed in SciPy '
 -                   '2.0.0, use scipy.fft.ifft instead')(ifft)
 -import numpy.lib.scimath as _sci
@@ -47,21 +41,24 @@
 -        _fun = _deprecated(_msg.format(_key))(_fun)
 -    globals()[_key] = _fun
 -
--# Allow distributors to run custom init code
--from . import _distributor_init
--
 -__all__ += _num.__all__
--__all__ += ['randn', 'rand', 'fft', 'ifft']
+-__all__ += ['randn', 'rand', 'ifft']
 -
 -del _num
--# Remove the linalg imported from numpy so that the scipy.linalg package can be
+-# Remove the linalg imported from NumPy so that the scipy.linalg package can be
 -# imported.
 -del linalg
 -__all__.remove('linalg')
 +# Chaquopy: don't import numpy during build.
 +try:
 +    __SCIPY_SETUP__
 +except NameError:
++    from numpy import show_config as show_numpy_config
++    if show_numpy_config is None:
++        raise ImportError(
++            "Cannot import SciPy when running from NumPy source directory.")
++    from numpy import __version__ as __numpy_version__
++
 +    # Import numpy symbols to scipy name space (DEPRECATED)
 +    from ._lib.deprecation import _deprecated
 +    import numpy as _num
@@ -79,16 +76,10 @@
 +            'use numpy.random.{0} instead')
 +    rand = _deprecated(_msg.format('rand'))(rand)
 +    randn = _deprecated(_msg.format('randn'))(randn)
-+    from numpy.fft import fft, ifft
-+    # fft is especially problematic, so we deprecate it with a shorter window
-+    fft_msg = ('Using scipy.fft as a function is deprecated and will be '
-+               'removed in SciPy 1.5.0, use scipy.fft.fft instead.')
-+    # for wrapping in scipy.fft.__call__, so the stacklevel is one off from the
-+    # usual (2)
-+    _dep_fft = _deprecated(fft_msg, stacklevel=3)(fft)
-+    fft = _deprecated(fft_msg)(fft)
++    # fft is especially problematic, so was removed in SciPy 1.6.0
++    from numpy.fft import ifft
 +    ifft = _deprecated('scipy.ifft is deprecated and will be removed in SciPy '
-+                       '2.0.0, use scipy.fft.ifft instead')(ifft)
++                    '2.0.0, use scipy.fft.ifft instead')(ifft)
 +    import numpy.lib.scimath as _sci
 +    _msg = ('scipy.{0} is deprecated and will be removed in SciPy 2.0.0, '
 +            'use numpy.lib.scimath.{0} instead')
@@ -98,23 +89,20 @@
 +            _fun = _deprecated(_msg.format(_key))(_fun)
 +        globals()[_key] = _fun
 +
-+    # Allow distributors to run custom init code
-+    from . import _distributor_init
-+
 +    __all__ += _num.__all__
-+    __all__ += ['randn', 'rand', 'fft', 'ifft']
++    __all__ += ['randn', 'rand', 'ifft']
 +
 +    del _num
-+    # Remove the linalg imported from numpy so that the scipy.linalg package can be
++    # Remove the linalg imported from NumPy so that the scipy.linalg package can be
 +    # imported.
 +    del linalg
 +    __all__.remove('linalg')
 
- # We first need to detect if we're being called as part of the scipy
+ # We first need to detect if we're being called as part of the SciPy
  # setup procedure itself in a reliable manner.
-@@ -126,6 +127,11 @@
+@@ -115,6 +119,11 @@
      import sys as _sys
-     _sys.stderr.write('Running from scipy source directory.\n')
+     _sys.stderr.write('Running from SciPy source directory.\n')
      del _sys
 +
 +    # Chaquopy: the module namespace was originally empty in this mode, but the following
@@ -124,12 +112,11 @@
  else:
      try:
          from scipy.__config__ import show as show_config
-diff -ur src-original/setup.py src/setup.py
---- src-original/setup.py	2018-05-05 17:10:14.000000000 +0000
-+++ src/setup.py	2018-12-20 11:40:38.859887187 +0000
-@@ -34,6 +34,21 @@
- else:
-     import builtins
+--- src-original/setup.py	2021-04-18 14:28:08.000000000 +0000
++++ src/setup.py	2023-09-09 15:45:15.243083019 +0000
+@@ -33,6 +33,20 @@
+ import builtins
+
 
 +
 +# Chaquopy: an OpenBLAS test executable is built without using LDFLAGS, so we have to add
@@ -141,27 +128,18 @@ diff -ur src-original/setup.py src/setup.py
 +                    "extra_link_args = -Wl,-rpath-link,{reqs_dir}/lib\n"
 +                    .format(reqs_dir=os.path.abspath("../requirements/chaquopy")))
 +
-+# Chaquopy added
-+sys.path.insert(0, os.path.abspath("../requirements"))  # For numpy.distutils
-+builtins.__NUMPY_SETUP__ = True  # Prevent the rest of NumPy from being imported.
++# Chaquopy: prevent the compiled parts NumPy from being imported
++builtins.__NUMPY_SETUP__ = True
 +
 +
  CLASSIFIERS = """\
  Development Status :: 5 - Production/Stable
  Intended Audience :: Science/Research
-@@ -472,7 +487,7 @@
-                           else [])
-
-     install_requires = build_requires
--    setup_requires = build_requires + ['pybind11>=2.4.0']
-+    setup_requires = ['pybind11>=2.4.0']  # Chaquopy: removed build_requires
-
-     metadata = dict(
-         name='scipy',
---- src-original/scipy/special/utils/makenpz.py	2018-05-05 17:10:11.000000000 +0000
-+++ src/scipy/special/utils/makenpz.py	2018-12-15 13:47:45.525539590 +0000
-@@ -8,7 +8,8 @@
- from __future__ import division, print_function, absolute_import
+diff -Naur src-original/scipy/special/utils/makenpz.py src/scipy/special/utils/makenpz.py
+--- src-original/scipy/special/utils/makenpz.py	2023-08-31 12:17:17.365654666 +0200
++++ src/scipy/special/utils/makenpz.py	2023-08-31 12:17:39.442111438 +0200
+@@ -6,7 +6,8 @@
+ """
 
  import os
 -import numpy as np