[v1.9.x] [BUGFIX] Upgrade numpy to <1.20.0 to avoid security vulnerab…

…ilities affecting numpy<1.19.1 (#20940)

* Pin numpy==1.19.1 to demonstrate issues

* Relax min numpy version

* Make test_np_array_function_protocol delete() testing work with numpy>=1.19

* Fix test_np_delete to also work with numpy>=1.19

* Pip install python module 'packaging'

* More pip-install 'packaging'

* Update windows requirements.txt

* Allow numpy as advanced as 1.19.5

ci/docker/install/centos7_python.sh

@@ -29,5 +29,5 @@ yum -y install python36u
 # Install PIP
 curl "https://bootstrap.pypa.io/pip/3.6/get-pip.py" -o "get-pip.py"
 python3.6 get-pip.py
-# Restrict numpy version to < 1.19.0 due to https://github.com/apache/incubator-mxnet/issues/18600
-python3.6 -m pip install nose pylint 'numpy>1.16.0,<1.19.0' nose-timer requests 'h5py<3' scipy==1.2.3
+# Allow numpy version as advanced as 1.19.5 to avoid CVE-2021-41495 and CVE-2021-41496 affecting <1.19.1.
+python3.6 -m pip install nose pylint 'numpy>=1.16.0,<1.20.0' nose-timer requests 'h5py<3' scipy==1.2.3 packaging

ci/docker/install/requirements

@@ -26,10 +26,12 @@ h5py<3
 mock==2.0.0
 nose==1.3.7
 nose-timer==0.7.3
-numpy>1.16.0,<1.19.0  # Restrict numpy version to < 1.19.0 due to https://github.com/apache/incubator-mxnet/issues/18600
+# Allow numpy version as advanced as 1.19.5 to avoid CVE-2021-41495 and CVE-2021-41496 affecting <1.19.1.
+numpy>=1.16.0,<1.20.0
 pylint==2.3.1  # pylint and astroid need to be aligned
 astroid==2.3.3  # pylint and astroid need to be aligned
 requests<2.19.0,>=2.18.4
 scipy==1.2.1
 setuptools
 coverage
+packaging

ci/docker/install/requirements_aarch64

@@ -31,3 +31,4 @@ requests<2.19.0,>=2.18.4
 setuptools
 coverage
 wheel
+packaging

ci/docker/install/ubuntu_onnx.sh

@@ -18,7 +18,7 @@
 # under the License.
 
 ######################################################################
-# This script installs ONNX for Python along with all required dependencies 
+# This script installs ONNX for Python along with all required dependencies
 # on a Ubuntu Machine.
 # Tested on Ubuntu 16.04 distro.
 ######################################################################
@@ -30,4 +30,5 @@ echo "Installing libprotobuf-dev and protobuf-compiler ..."
 apt-get update || true
 apt-get install -y libprotobuf-dev protobuf-compiler
 
-pip3 install pytest==6.2.2 pytest-cov==2.11.1 pytest-xdist==2.2.1 protobuf==3.13.0 onnx==1.8.1 Pillow==5.0.0 tabulate==0.7.5 onnxruntime==1.7.0 'numpy>1.16.0,<1.19.0' gluonnlp==0.10.0 gluoncv==0.8.0
+# Allow numpy version as advanced as 1.19.5 to avoid CVE-2021-41495 and CVE-2021-41496 affecting <1.19.1.
+pip3 install pytest==6.2.2 pytest-cov==2.11.1 pytest-xdist==2.2.1 protobuf==3.13.0 onnx==1.8.1 Pillow==5.0.0 tabulate==0.7.5 onnxruntime==1.7.0 'numpy>=1.16.0,<1.20.0' gluonnlp==0.10.0 gluoncv==0.8.0 packaging

ci/docker/install/ubuntu_publish.sh

@@ -84,8 +84,8 @@ apt-get install -y python python-pip python3 python3-pip
 # Restrict pip version to <19 due to use of Python 3.4 on Ubuntu 14.04
 python3 -m pip install --upgrade 'pip<19'
 
-# Restrict numpy version to <1.18 due to use of Python 3.4 on Ubuntu 14.04
-python3 -m pip install --upgrade --ignore-installed nose cpplint==1.3.0 pylint==2.3.1 'numpy>1.16.0,<1.18' nose-timer 'requests<2.19.0,>=2.18.4' 'h5py<3' scipy==1.0.1 boto3
+# Allow numpy version as advanced as 1.19.5 to avoid CVE-2021-41495 and CVE-2021-41496 affecting <1.19.1.
+python3 -m pip install --upgrade --ignore-installed nose cpplint==1.3.0 pylint==2.3.1 'numpy>=1.16.0,<1.20.0' nose-timer 'requests<2.19.0,>=2.18.4' 'h5py<3' scipy==1.0.1 boto3 packaging
 
 # CMake 3.13.2+ is required
 mkdir /opt/cmake && cd /opt/cmake

ci/travis/install.sh

@@ -22,6 +22,6 @@ export HOMEBREW_NO_AUTO_UPDATE=1
 
 if [ ${TRAVIS_OS_NAME} == "osx" ]; then
     brew install opencv
-    # Restrict numpy version to < 1.19.0 due to https://github.com/apache/incubator-mxnet/issues/18600
-    python -m pip install --user nose 'numpy>1.16.0,<1.19.0' cython scipy requests mock nose-timer nose-exclude mxnet-to-coreml
+    # Allow numpy version as advanced as 1.19.5 to avoid CVE-2021-41495 and CVE-2021-41496 affecting <1.19.1.
+    python -m pip install --user nose 'numpy>=1.16.0,<1.20.0' cython scipy requests mock nose-timer nose-exclude mxnet-to-coreml packaging
 fi

tests/python/unittest/test_numpy_interoperability.py

@@ -1194,10 +1194,12 @@ def _add_workload_delete():
                 s = slice(start, stop, step)
                 OpArgMngr.add_workload('delete', a, s)
                 OpArgMngr.add_workload('delete', nd_a, s, axis=1)
-    OpArgMngr.add_workload('delete', a, np.array([]), axis=0)
+    # mxnet.numpy arrays, even 0-sized, have a float32 dtype.  Starting with numpy 1.19, the
+    # index array's of delete() must be of integer or boolean type, so we force that below.
+    OpArgMngr.add_workload('delete', a, np.array([], dtype='int32'), axis=0)
     OpArgMngr.add_workload('delete', a, 0)
-    OpArgMngr.add_workload('delete', a, np.array([]))
-    OpArgMngr.add_workload('delete', a, np.array([0, 1]))
+    OpArgMngr.add_workload('delete', a, np.array([], dtype='int32'))
+    OpArgMngr.add_workload('delete', a, np.array([0, 1], dtype='int32'))
     OpArgMngr.add_workload('delete', a, slice(1, 2))
     OpArgMngr.add_workload('delete', a, slice(1, -2))
     k = np.arange(10).reshape(2, 5)

tests/python/unittest/test_numpy_op.py

@@ -26,6 +26,7 @@
 import mxnet as mx
 import scipy.stats as ss
 import scipy.special as scipy_special
+from packaging.version import parse
 from nose.tools import assert_raises
 from mxnet import np, npx
 from mxnet.gluon import HybridBlock
@@ -3622,6 +3623,12 @@ def GetDimSize(shp, axis):
             if type(obj) == list:
                 obj_mxnp = np.array(obj, dtype=objtype)
                 obj_onp = _np.array(obj, dtype=objtype)
+                # To match mxnet.numpy's behavior of ignoring out-of-bounds indices,
+                # we may need to filter out indices that this numpy would not ignore.
+                onp_ignores_oob_indices = parse(_np.version.version) < parse('1.19')
+                if not onp_ignores_oob_indices:
+                    dim_size = GetDimSize(arr_shape,axis)
+                    obj_onp = obj_onp[((obj_onp>=0) & (obj_onp<dim_size))]
             elif type(obj) == slice:
                 obj_mxnp = obj
                 obj_onp = obj

tests/requirements.txt

@@ -4,6 +4,8 @@ mock
 nose
 nose-timer
 ipython
-numpy>1.16.0,<1.19.0  # Restrict numpy version to < 1.19.0 due to https://github.com/apache/incubator-mxnet/issues/18600
+# Allow numpy version as advanced as 1.19.5 to avoid CVE-2021-41495 and CVE-2021-41496 affecting <1.19.1.
+numpy>=1.16.0,<1.20.0
 scipy<1.7.0 # Restrict scipy version due to https://github.com/apache/incubator-mxnet/issues/20389
 onnxruntime
+packaging