ENH: Upgrade to Python 3.9 and package update (#910)

- Upgrade to Python 3.9 as a minimum - Upgrade pillow because of security alerts. This required some API changes and update of test result images - The behaviour of the `LogCaptureFixture` changed, requiring changes from `logging.info` to `logger.info` - Downgraded `black` from 23 to 22 because of version constraints - Changes in calls to `WorkflowParams` because of `mypy` issues that were not detected until now - `hi-ml-azure` now also has its own minimal Conda dev environment
microsoft · Nov 9, 2023 · 1cf19cc · 1cf19cc
1 parent 2d8a380
commit 1cf19cc
Show file tree

Hide file tree

Showing 67 changed files with 783 additions and 510 deletions.
diff --git a/.github/actions/cancel_azureml_jobs/azureml-env.yml b/.github/actions/cancel_azureml_jobs/azureml-env.yml
@@ -2,7 +2,7 @@ name: AzureML_SDK
 channels:
   - defaults
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - pip:
-      - azureml-sdk==1.36.0
+      - azureml-sdk==1.53.0
diff --git a/.github/workflows/hi-ml-pr.yml b/.github/workflows/hi-ml-pr.yml
@@ -17,7 +17,7 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
 
 env:
-  pythonVersion: 3.7
+  pythonVersion: 3.9
   HIML_TENANT_ID: ${{ secrets.HIML_TENANT_ID }}
   HIML_RESOURCE_GROUP: ${{ secrets.HIML_RESOURCE_GROUP }}
   HIML_SUBSCRIPTION_ID: ${{ secrets.HIML_SUBSCRIPTION_ID }}

diff --git a/docs/source/developers.md b/docs/source/developers.md
@@ -80,8 +80,8 @@ pip install --extra-index-url https://test.pypi.org/simple/ hi-ml==0.1.0.post165
 ```yaml
 name: foo
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - pip:
       - --extra-index-url https://test.pypi.org/simple/
       - hi-ml==0.1.0.post165

diff --git a/docs/source/examples/1/environment.yml b/docs/source/examples/1/environment.yml
@@ -1,4 +1,4 @@
 name: primes-env
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
diff --git a/docs/source/examples/10/environment.yml b/docs/source/examples/10/environment.yml
@@ -1,7 +1,7 @@
 name: iris-env
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - scikit-learn
   - pip:
     - hi-ml-azure==0.1.10
diff --git a/docs/source/examples/2/environment.yml b/docs/source/examples/2/environment.yml
@@ -1,6 +1,6 @@
 name: primes-env
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - pip:
     - hi-ml-azure>=0.1.0
diff --git a/docs/source/examples/3/environment.yml b/docs/source/examples/3/environment.yml
@@ -1,6 +1,6 @@
 name: primes-env
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - pip:
     - hi-ml-azure>=0.1.0
diff --git a/docs/source/examples/4/environment.yml b/docs/source/examples/4/environment.yml
@@ -1,6 +1,6 @@
 name: primes-env
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - pip:
     - hi-ml-azure>=0.1.0
diff --git a/docs/source/examples/5/environment.yml b/docs/source/examples/5/environment.yml
@@ -1,5 +1,5 @@
 name: iris-env
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - scikit-learn
diff --git a/docs/source/examples/6/environment.yml b/docs/source/examples/6/environment.yml
@@ -1,7 +1,7 @@
 name: iris-env
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - scikit-learn
   - pip:
     - hi-ml-azure>=0.1.0
diff --git a/docs/source/examples/7/environment.yml b/docs/source/examples/7/environment.yml
@@ -1,7 +1,7 @@
 name: iris-env
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - scikit-learn
   - pip:
     - hi-ml-azure>=0.1.0
diff --git a/docs/source/examples/8/environment.yml b/docs/source/examples/8/environment.yml
@@ -1,7 +1,7 @@
 name: iris-env
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - scikit-learn
   - pip:
     - hi-ml-azure>=0.1.0
diff --git a/docs/source/examples/9/tensorboard_env.yml b/docs/source/examples/9/tensorboard_env.yml
@@ -3,8 +3,8 @@ channels:
   - defaults
   - pytorch
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - pip=23.3
+  - python=3.9.18
   - pytorch=1.4.0
   - pip:
     - tensorboard==2.2.1
diff --git a/hi-ml-azure/.mypy.ini b/hi-ml-azure/.mypy.ini
@@ -1,5 +1,5 @@
 [mypy]
-python_version=3.7
+python_version=3.9
 scripts_are_modules=True
 namespace_packages=True
 show_traceback=True

diff --git a/hi-ml-azure/environment.yml b/hi-ml-azure/environment.yml
@@ -1,6 +1,232 @@
-# This environment definition contains all a basic environment definition that
-# is suitable for submitting scripts from the hi-ml-azure folder.
+# WARNING - DO NOT EDIT THIS FILE MANUALLY
+# To update, please modify 'primary_deps.yml' and then run the locking script 'create_and_lock_environment.sh'
 name: himl-azure
+channels:
+  - defaults
 dependencies:
-  - pip=20.1.1
-  - python=3.7.3
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - ca-certificates=2023.08.22=h06a4308_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.12=h7f8727e_0
+  - pip=23.3=py39h06a4308_0
+  - python=3.9.18
+  - readline=8.2=h5eee18b_0
+  - setuptools=68.0.0=py39h06a4308_0
+  - sqlite=3.41.2=h5eee18b_0
+  - tk=8.6.12=h1ccaba5_0
+  - xz=5.4.2=h5eee18b_0
+  - zlib=1.2.13=h5eee18b_0
+  - pip:
+      - absl-py==2.0.0
+      - adal==1.2.7
+      - alabaster==0.7.13
+      - alembic==1.12.1
+      - applicationinsights==0.11.10
+      - argcomplete==2.1.2
+      - astroid==2.15.8
+      - attrs==21.4.0
+      - azure-ai-ml==1.11.1
+      - azure-common==1.1.28
+      - azure-core==1.29.5
+      - azure-graphrbac==0.61.1
+      - azure-identity==1.15.0
+      - azure-mgmt-authorization==3.0.0
+      - azure-mgmt-containerregistry==10.2.0
+      - azure-mgmt-core==1.4.0
+      - azure-mgmt-keyvault==10.3.0
+      - azure-mgmt-network==21.0.1
+      - azure-mgmt-resource==22.0.0
+      - azure-mgmt-storage==21.0.0
+      - azure-storage-blob==12.10.0
+      - azure-storage-file-datalake==12.6.0
+      - azure-storage-file-share==12.15.0
+      - azureml-core==1.53.0
+      - azureml-dataprep==4.12.7
+      - azureml-dataprep-native==38.0.0
+      - azureml-dataprep-rslex==2.19.6
+      - azureml-dataset-runtime==1.53.0
+      - azureml-mlflow==1.53.0
+      - azureml-telemetry==1.53.0
+      - azureml-tensorboard==1.53.0
+      - azureml-train-core==1.53.0
+      - azureml-train-restclients-hyperdrive==1.53.0
+      - babel==2.13.1
+      - backports-tempfile==1.0
+      - backports-weakref==1.0.post1
+      - bcrypt==4.0.1
+      - black==23.1.0
+      - blinker==1.7.0
+      - cachetools==5.3.2
+      - certifi==2023.7.22
+      - cffi==1.16.0
+      - cfgv==3.4.0
+      - charset-normalizer==3.3.2
+      - click==8.1.7
+      - cloudpickle==2.2.1
+      - colorama==0.4.6
+      - conda-merge==0.2.0
+      - contextlib2==21.6.0
+      - contourpy==1.2.0
+      - coverage==7.1.0
+      - cryptography==41.0.5
+      - cycler==0.12.1
+      - databricks-cli==0.18.0
+      - dill==0.3.7
+      - distlib==0.3.7
+      - distro==1.8.0
+      - docker==6.1.3
+      - docutils==0.16
+      - dotnetcore2==3.1.23
+      - entrypoints==0.4
+      - exceptiongroup==1.1.3
+      - filelock==3.13.1
+      - flake8==6.1.0
+      - flake8-bugbear==23.3.12
+      - flask==3.0.0
+      - fonttools==4.44.0
+      - fusepy==3.0.1
+      - gitdb==4.0.11
+      - gitpython==3.1.40
+      - google-api-core==2.13.0
+      - google-auth==2.23.4
+      - google-auth-oauthlib==1.1.0
+      - googleapis-common-protos==1.61.0
+      - greenlet==3.0.1
+      - grpcio==1.59.2
+      - gunicorn==21.2.0
+      - humanfriendly==10.0
+      - identify==2.5.31
+      - idna==3.4
+      - imagesize==1.4.1
+      - importlib-metadata==6.8.0
+      - importlib-resources==6.1.1
+      - iniconfig==2.0.0
+      - isodate==0.6.1
+      - isort==5.12.0
+      - itsdangerous==2.1.2
+      - jaraco-classes==3.3.0
+      - jeepney==0.8.0
+      - jinja2==3.1.2
+      - jmespath==1.0.1
+      - joblib==1.3.2
+      - jsonpickle==3.0.2
+      - jsonschema==4.17.3
+      - keyring==24.2.0
+      - kiwisolver==1.4.5
+      - knack==0.10.1
+      - lazy-object-proxy==1.9.0
+      - lxml==4.9.3
+      - mako==1.2.4
+      - markdown==3.5.1
+      - markdown-it-py==1.1.0
+      - markupsafe==2.1.3
+      - marshmallow==3.20.1
+      - matplotlib==3.8.1
+      - mccabe==0.7.0
+      - mdit-py-plugins==0.2.8
+      - mlflow==2.8.0
+      - mlflow-skinny==2.8.0
+      - more-itertools==10.1.0
+      - msal==1.25.0
+      - msal-extensions==1.0.0
+      - msrest==0.7.1
+      - msrestazure==0.6.4
+      - mypy==1.0.1
+      - mypy-extensions==1.0.0
+      - myst-parser==0.15.2
+      - ndg-httpsclient==0.5.1
+      - nh3==0.2.14
+      - nodeenv==1.8.0
+      - numpy==1.23.5
+      - oauthlib==3.2.2
+      - opencensus==0.11.3
+      - opencensus-context==0.1.3
+      - opencensus-ext-azure==1.1.11
+      - packaging==23.0
+      - pandas==2.1.2
+      - param==1.13.0
+      - paramiko==3.3.1
+      - pathspec==0.11.2
+      - pillow==10.1.0
+      - pkginfo==1.9.6
+      - platformdirs==3.11.0
+      - pluggy==1.3.0
+      - portalocker==2.8.2
+      - pre-commit==2.21.0
+      - protobuf==3.20.3
+      - psutil==5.9.6
+      - pyarrow==11.0.0
+      - pyasn1==0.5.0
+      - pyasn1-modules==0.3.0
+      - pycobertura==3.0.0
+      - pycodestyle==2.11.1
+      - pycparser==2.21
+      - pydash==7.0.5
+      - pyflakes==3.1.0
+      - pygments==2.16.1
+      - pyjwt==2.8.0
+      - pylint==2.16.2
+      - pynacl==1.5.0
+      - pyopenssl==23.3.0
+      - pyparsing==3.1.1
+      - pyrsistent==0.20.0
+      - pysocks==1.7.1
+      - pytest==7.2.1
+      - pytest-cov==4.0.0
+      - pytest-rerunfailures==11.1.1
+      - pytest-timeout==2.1.0
+      - python-dateutil==2.8.2
+      - pytz==2023.3.post1
+      - pyyaml==6.0.1
+      - querystring-parser==1.2.4
+      - readme-renderer==42.0
+      - requests==2.31.0
+      - requests-oauthlib==1.3.1
+      - requests-toolbelt==1.0.0
+      - rfc3986==2.0.0
+      - rsa==4.9
+      - ruamel-yaml==0.18.5
+      - ruamel-yaml-clib==0.2.8
+      - scikit-learn==1.3.2
+      - scipy==1.11.3
+      - secretstorage==3.3.3
+      - six==1.16.0
+      - smmap==5.0.1
+      - snowballstemmer==2.2.0
+      - sphinx==4.1.2
+      - sphinx-argparse==0.3.1
+      - sphinx-autodoc-typehints==1.12.0
+      - sphinx-automodapi==0.13
+      - sphinx-rtd-theme==1.0.0
+      - sphinxcontrib-applehelp==1.0.4
+      - sphinxcontrib-devhelp==1.0.2
+      - sphinxcontrib-htmlhelp==2.0.1
+      - sphinxcontrib-jsmath==1.0.1
+      - sphinxcontrib-qthelp==1.0.3
+      - sphinxcontrib-serializinghtml==1.1.5
+      - sqlalchemy==2.0.23
+      - sqlparse==0.4.4
+      - strictyaml==1.7.3
+      - tabulate==0.9.0
+      - tensorboard==2.15.1
+      - tensorboard-data-server==0.7.2
+      - threadpoolctl==3.2.0
+      - tomli==2.0.1
+      - tomlkit==0.12.2
+      - tqdm==4.66.1
+      - twine==3.3.0
+      - typing-extensions==4.8.0
+      - tzdata==2023.3
+      - urllib3==1.26.18
+      - virtualenv==20.24.6
+      - websocket-client==1.6.4
+      - werkzeug==3.0.1
+      - wheel==0.38.1
+      - wrapt==1.15.0
+      - zipp==3.17.0
diff --git a/hi-ml-azure/primary_deps.yml b/hi-ml-azure/primary_deps.yml
@@ -0,0 +1,12 @@
+# This environment definition contains all packages to run hi-ml-azure development work, building and
+# testing
+name: himl-azure
+channels:
+  - defaults
+dependencies:
+  - pip=23.3
+  - python=3.9.18
+  - pip:
+      - -r run_requirements.txt
+      - -r ../build_requirements.txt
+      - -r ../test_requirements.txt
diff --git a/hi-ml-azure/run_requirements.txt b/hi-ml-azure/run_requirements.txt
@@ -8,7 +8,7 @@ azureml-train-core>=1.42.0
 conda-merge>=0.1.5
 mlflow>=1.29.0
 pandas>=1.3.4
-param>=1.12
+param>=1.12,<2.0
 protobuf<4.0
 pysocks>=1.5.8
 ruamel.yaml>=0.16.12

diff --git a/hi-ml-azure/setup.py b/hi-ml-azure/setup.py
@@ -77,7 +77,7 @@
         "Intended Audience :: Science/Research",
         "Topic :: Scientific/Engineering :: Medical Science Apps.",
         "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.9",
     ],
     keywords="Health Futures, Health Intelligence, AzureML",
     license="MIT License",