From 127af59bc6bd2ceb32c5f46378ccd8f706af400e Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Wed, 26 Apr 2023 20:57:32 +0100 Subject: [PATCH 01/11] Remove upper-bound on cupy version for conda package --- conda/recipes/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/meta.yaml b/conda/recipes/meta.yaml index f41494e1e2d..8220959e64f 100644 --- a/conda/recipes/meta.yaml +++ b/conda/recipes/meta.yaml @@ -34,7 +34,7 @@ requirements: - python - cudf>=21.10.* - dask-cudf>=21.10.* - - cupy>=7.2.0,<10.0.0a + - cupy>=7.2.0 - nvtx>=0.2.1 - pynvml From eada6b51b7a86260b5e01bd90bf270d0ed2b7853 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Wed, 26 Apr 2023 21:32:58 +0100 Subject: [PATCH 02/11] Update conda package publish to build for multiple python versions --- .github/workflows/cpu-packages.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/cpu-packages.yml index d0350d4e84e..f41ba8f353e 100644 --- a/.github/workflows/cpu-packages.yml +++ b/.github/workflows/cpu-packages.yml @@ -10,20 +10,15 @@ on: jobs: build-conda: name: Build Conda - runs-on: ${{ matrix.os }} - strategy: - matrix: - python-version: [3.8] - os: [ubuntu-latest] - + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.8 uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: 3.8 - name: Install Ubuntu packages run: | sudo apt-get update -y @@ -38,8 +33,8 @@ jobs: run: | conda update conda conda install conda-build pybind11 - conda build --python ${{ matrix.python-version }} . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages - export CONDA_PACKAGE=$(conda build --python ${{ matrix.python-version }} . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages --output) + conda build --python 3.8 --python 3.9 --python 3.10 --python 3.11 . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages + export CONDA_PACKAGE=$(conda build --python 3.8 --python 3.9 --python 3.10 --python 3.11 . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages --output) echo "conda_package : $CONDA_PACKAGE" echo "conda_package=$CONDA_PACKAGE" >> $GITHUB_OUTPUT - name: Upload conda artifacts to github From aa3e91c656e1c50f34053b890604e57371d6b941 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Wed, 26 Apr 2023 21:37:51 +0100 Subject: [PATCH 03/11] Use find command to get CONDA_PACKAGE output --- .github/workflows/cpu-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/cpu-packages.yml index f41ba8f353e..9992d12642f 100644 --- a/.github/workflows/cpu-packages.yml +++ b/.github/workflows/cpu-packages.yml @@ -34,7 +34,7 @@ jobs: conda update conda conda install conda-build pybind11 conda build --python 3.8 --python 3.9 --python 3.10 --python 3.11 . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages - export CONDA_PACKAGE=$(conda build --python 3.8 --python 3.9 --python 3.10 --python 3.11 . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages --output) + export CONDA_PACKAGE=$(find ./conda_packages/ -name nvtabular*.tar.bz2) echo "conda_package : $CONDA_PACKAGE" echo "conda_package=$CONDA_PACKAGE" >> $GITHUB_OUTPUT - name: Upload conda artifacts to github From 20dd1689c24394ad3d802144d13c4b3dab8b44c3 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Wed, 26 Apr 2023 21:50:30 +0100 Subject: [PATCH 04/11] Remove Python 3.11 conda build (rapids does not yet support this) --- .github/workflows/cpu-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/cpu-packages.yml index 9992d12642f..3c491bae56f 100644 --- a/.github/workflows/cpu-packages.yml +++ b/.github/workflows/cpu-packages.yml @@ -33,7 +33,7 @@ jobs: run: | conda update conda conda install conda-build pybind11 - conda build --python 3.8 --python 3.9 --python 3.10 --python 3.11 . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages + conda build --python 3.8 --python 3.9 --python 3.10 . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages export CONDA_PACKAGE=$(find ./conda_packages/ -name nvtabular*.tar.bz2) echo "conda_package : $CONDA_PACKAGE" echo "conda_package=$CONDA_PACKAGE" >> $GITHUB_OUTPUT From 1927d43a0cb6a65403539ff7aa874f9c5a064b55 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Fri, 28 Apr 2023 13:44:58 +0100 Subject: [PATCH 05/11] Use strategy matrix for python version builds --- .github/workflows/cpu-packages.yml | 41 ++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/cpu-packages.yml index 3c491bae56f..aba0a0b4502 100644 --- a/.github/workflows/cpu-packages.yml +++ b/.github/workflows/cpu-packages.yml @@ -6,26 +6,31 @@ on: branches: [main] tags: - v* + pull_request: + branches: [main] jobs: build-conda: name: Build Conda runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8, 3.9, 3.10] steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python 3.8 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: ${{ matrix.python-version }} - name: Install Ubuntu packages run: | sudo apt-get update -y sudo apt-get install -y protobuf-compiler - name: Install and upgrade python packages run: | - python -m pip install --upgrade pip setuptools==59.4.0 wheel tox pybind11 + python -m pip install --upgrade pip setuptools==59.4.0 wheel pybind11 python -m pip uninstall protobuf -y python -m pip install --no-binary=protobuf protobuf - name: Generate package for conda @@ -33,14 +38,15 @@ jobs: run: | conda update conda conda install conda-build pybind11 - conda build --python 3.8 --python 3.9 --python 3.10 . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages - export CONDA_PACKAGE=$(find ./conda_packages/ -name nvtabular*.tar.bz2) - echo "conda_package : $CONDA_PACKAGE" - echo "conda_package=$CONDA_PACKAGE" >> $GITHUB_OUTPUT + conda build --python ${{ matrix.python-version }} . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages + conda_package=$(find ./conda_packages/ -name "*.tar.bz2") + export CONDA_PACKAGE=$conda_package + echo "conda_package : $conda_package" + echo "conda_package=$conda_package" >> $GITHUB_OUTPUT - name: Upload conda artifacts to github uses: actions/upload-artifact@v3 with: - name: conda + name: conda-${{ matrix.python-version }} path: ${{ steps.conda_build.outputs.conda_package }} build-pypi: @@ -107,6 +113,25 @@ jobs: name: pr path: pr/ + check-conda: + name: Check Conda Artifacts + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8, 3.9, 3.10] + needs: [build-conda] + steps: + - uses: actions/setup-python@v4 + with: + python-version: 3.9 + - uses: actions/download-artifact@v2 + with: + name: conda-${{ matrix.python-version }} + path: conda + - name: check artifacts + run: | + find conda -name "*.tar.gz" + release-conda: name: Release Conda runs-on: ubuntu-latest From 90eca06628188c311c181c64b0ea2ada35142604 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Fri, 28 Apr 2023 13:46:34 +0100 Subject: [PATCH 06/11] quote python versions --- .github/workflows/cpu-packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/cpu-packages.yml index aba0a0b4502..b73c746d3bd 100644 --- a/.github/workflows/cpu-packages.yml +++ b/.github/workflows/cpu-packages.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, 3.10] + python-version: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v3 with: @@ -118,7 +118,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, 3.10] + python-version: ["3.8", "3.9", "3.10"] needs: [build-conda] steps: - uses: actions/setup-python@v4 From 7ed3e31b3155fc34e69cbd3adee234dbd4c1a371 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Fri, 28 Apr 2023 13:53:00 +0100 Subject: [PATCH 07/11] Set concurrency to cancel builds --- .github/workflows/cpu-packages.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/cpu-packages.yml index b73c746d3bd..7b4573c8909 100644 --- a/.github/workflows/cpu-packages.yml +++ b/.github/workflows/cpu-packages.yml @@ -9,6 +9,10 @@ on: pull_request: branches: [main] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build-conda: name: Build Conda From 16c8dc8dd0fd93d4f4e09952e070541597017967 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Fri, 28 Apr 2023 13:53:39 +0100 Subject: [PATCH 08/11] Use mambabuild to speed up conda build time --- .github/workflows/cpu-packages.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/cpu-packages.yml index 7b4573c8909..f1cf0b7eb9c 100644 --- a/.github/workflows/cpu-packages.yml +++ b/.github/workflows/cpu-packages.yml @@ -41,8 +41,9 @@ jobs: id: conda_build run: | conda update conda - conda install conda-build pybind11 - conda build --python ${{ matrix.python-version }} . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages + conda install -c conda-forge mamba + mamba install -c conda-forge conda-build boa conda-verify pybind11 + conda mambabuild --python ${{ matrix.python-version }} . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages conda_package=$(find ./conda_packages/ -name "*.tar.bz2") export CONDA_PACKAGE=$conda_package echo "conda_package : $conda_package" From 355dc7899794c35dfeea774e5c5c85710046efb1 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Fri, 28 Apr 2023 13:58:41 +0100 Subject: [PATCH 09/11] Add setup with conda-incubator --- .github/workflows/cpu-packages.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/cpu-packages.yml index f1cf0b7eb9c..9ba4ed08d49 100644 --- a/.github/workflows/cpu-packages.yml +++ b/.github/workflows/cpu-packages.yml @@ -37,9 +37,13 @@ jobs: python -m pip install --upgrade pip setuptools==59.4.0 wheel pybind11 python -m pip uninstall protobuf -y python -m pip install --no-binary=protobuf protobuf + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true - name: Generate package for conda id: conda_build run: | + echo "conda pkgs dir $CONDA_PKGS_DIRS" conda update conda conda install -c conda-forge mamba mamba install -c conda-forge conda-build boa conda-verify pybind11 From b3036ee98121ec10cebf190c7afe6214f63e060d Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Fri, 28 Apr 2023 15:20:49 +0100 Subject: [PATCH 10/11] remove pull_request workflow condition and add release branch push --- .github/workflows/cpu-packages.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/cpu-packages.yml index 9ba4ed08d49..22f5a8d0644 100644 --- a/.github/workflows/cpu-packages.yml +++ b/.github/workflows/cpu-packages.yml @@ -3,11 +3,11 @@ name: Packages on: workflow_dispatch: push: - branches: [main] + branches: + - main + - release* tags: - v* - pull_request: - branches: [main] concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 25d1b3de6b4d361096a2d00e01efeeccabc6209e Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Fri, 28 Apr 2023 15:21:23 +0100 Subject: [PATCH 11/11] Add Python version matrix for conda package publish --- .github/workflows/cpu-packages.yml | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/cpu-packages.yml index 22f5a8d0644..984211b01fa 100644 --- a/.github/workflows/cpu-packages.yml +++ b/.github/workflows/cpu-packages.yml @@ -122,37 +122,21 @@ jobs: name: pr path: pr/ - check-conda: - name: Check Conda Artifacts + release-conda: + name: Release Conda runs-on: ubuntu-latest strategy: matrix: python-version: ["3.8", "3.9", "3.10"] - needs: [build-conda] - steps: - - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - uses: actions/download-artifact@v2 - with: - name: conda-${{ matrix.python-version }} - path: conda - - name: check artifacts - run: | - find conda -name "*.tar.gz" - - release-conda: - name: Release Conda - runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/')" needs: [build-conda] steps: - uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: ${{ matrix.python-version }} - uses: actions/download-artifact@v2 with: - name: conda + name: conda-${{ matrix.python-version }} path: conda - uses: conda-incubator/setup-miniconda@v2 with: