From a5e43a5468f5c7385719ed774f0ad8228c05fbd7 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Thu, 7 Mar 2024 20:59:09 +0800 Subject: [PATCH 01/29] X --- .../workflows/multiplatform-build-test.yml | 972 +++++++++--------- 1 file changed, 503 insertions(+), 469 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index a4009cfc52..e0a03f2d4c 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -9,6 +9,9 @@ jobs: ubuntu-2004-build-test: name: ubuntu-20.04 runs-on: ubuntu-20.04 + defaults: + run: + shell: bash env: CC: gcc-11 CXX: g++-11 @@ -37,478 +40,509 @@ jobs: run: make release NUM_THREADS=$(nproc) - name: Test - run: make test NUM_THREADS=$(nproc) - - - name: C and C++ examples - run: make example NUM_THREADS=$(nproc) - - - name: Python test - run: make pytest NUM_THREADS=$(nproc) - - - name: Node.js test - run: make nodejstest NUM_THREADS=$(nproc) - - - name: Java test - run: make javatest NUM_THREADS=$(nproc) - - - name: Rust share build - run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - - - name: Rust set env - run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - - - name: Rust test - working-directory: tools/rust_api - run: | - cargo test --release --features arrow -- --test-threads=1 - - - name: Rust example - working-directory: examples/rust - run: cargo build --release --features arrow - - macos-build-test: - strategy: - matrix: - runner: [macos-12, macos-13] - fail-fast: false - name: ${{ matrix.runner }} - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Ensure Python dependencies - run: | - pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu - pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - - - name: Ensure Node.js dependencies - working-directory: tools/nodejs_api - run: npm install --include=dev - - - name: Build - run: make release NUM_THREADS=$(sysctl -n hw.physicalcpu) - - - name: Test - run: make test NUM_THREADS=$(sysctl -n hw.physicalcpu) - - - name: C and C++ examples - run: make example NUM_THREADS=$(sysctl -n hw.physicalcpu) - - - name: Python test - run: make pytest NUM_THREADS=$(sysctl -n hw.physicalcpu) - - - name: Node.js test - run: make nodejstest NUM_THREADS=$(sysctl -n hw.physicalcpu) - - - name: Java test - run: make javatest NUM_THREADS=$(sysctl -n hw.physicalcpu) - - - name: Rust share build - run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - - - name: Rust set env - run: echo "CARGO_BUILD_JOBS=$(sysctl -n hw.physicalcpu)" >> $GITHUB_ENV - - - name: Rust test - working-directory: tools/rust_api - run: | - cargo test --release --features arrow -- --test-threads=1 - - - name: Rust example - working-directory: examples/rust - run: cargo build --release --features arrow - - windows-build-test: - strategy: - matrix: - runner: [windows-2019, windows-2022] - fail-fast: false - name: ${{ matrix.runner }} - runs-on: ${{ matrix.runner }} - steps: - - name: Disable Windows Defender - shell: powershell - run: | - Set-MpPreference -DisableRealtimeMonitoring $true - Set-MpPreference -DisableBehaviorMonitoring $true - Set-MpPreference -DisableBlockAtFirstSeen $true - Set-MpPreference -DisableIOAVProtection $true - Set-MpPreference -DisablePrivacyMode $true - Set-MpPreference -SignatureDisableUpdateOnStartupWithoutEngine $true - Set-MpPreference -DisableArchiveScanning $true - Set-MpPreference -DisableIntrusionPreventionSystem $true - Set-MpPreference -DisableScriptScanning $true - Set-MpPreference -SubmitSamplesConsent NeverSend - - - uses: actions/checkout@v3 - - - uses: ilammy/msvc-dev-cmd@v1 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Ensure Python dependencies - shell: cmd - run: | - pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu - pip install --user -r tools\python_api\requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - - - name: Ensure Node.js dependencies - shell: cmd - working-directory: .\tools\nodejs_api - run: npm install --include=dev - - - name: Build - shell: cmd - run: make release NUM_THREADS=%NUMBER_OF_PROCESSORS% - - - name: Test - shell: cmd - run: make test NUM_THREADS=%NUMBER_OF_PROCESSORS% - - - name: C and C++ examples - shell: cmd - run: make example NUM_THREADS=%NUMBER_OF_PROCESSORS% - - - name: Python test - shell: cmd - run: make pytest NUM_THREADS=%NUMBER_OF_PROCESSORS% - - - name: Node.js test - shell: cmd - run: make nodejstest NUM_THREADS=%NUMBER_OF_PROCESSORS% - - - name: Java test - shell: cmd - run: make javatest NUM_THREADS=%NUMBER_OF_PROCESSORS% - - - name: Rust share build - shell: cmd - run: (echo [workspace]& echo members = ["tools/rust_api","examples/rust"]) > Cargo.toml - - - name: Rust test - shell: cmd - run: | - make clean - set OPENSSL_DIR=C:\Program Files\OpenSSL-Win64 - set CXXFLAGS=/std:c++20 - set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% - cargo test --release --features arrow -- --test-threads=1 - - - name: Rust example - shell: cmd - run: | - set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% - cargo build --release --features arrow - - debian-ubuntu-build-test: - strategy: - matrix: - image: ["ubuntu:23.04", "ubuntu:23.10", "debian:12", "debian:sid"] - fail-fast: false - name: ${{ matrix.image }} - runs-on: ubuntu-latest - container: - image: ${{ matrix.image }} - env: - CC: gcc - CXX: g++ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64 - steps: - - name: Install packages - run: | - apt-get update - apt-get install -y ca-certificates curl gnupg - mkdir -p /etc/apt/keyrings - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list - apt-get update - apt-get install -y git build-essential cmake gcc g++ python3 python3-dev python3-pip openjdk-17-jdk nodejs - - - uses: actions/checkout@v3 - - - name: Ensure Python dependencies - run: | - pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu ${{ matrix.image != 'debian:11' && '--break-system-packages' || '' }} - pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html ${{ matrix.image != 'debian:11' && '--break-system-packages' || '' }} - - - name: Ensure Node.js dependencies - working-directory: tools/nodejs_api - run: npm install --include=dev - - - name: Install Rust - run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - $HOME/.cargo/bin/rustup toolchain install 1.67 - - - name: Build - run: make release NUM_THREADS=$(nproc) - - - name: Test - run: make test NUM_THREADS=$(nproc) - - - name: C and C++ examples - run: make example NUM_THREADS=$(nproc) - - - name: Python test - run: make pytest NUM_THREADS=$(nproc) - - - name: Node.js test - run: make nodejstest NUM_THREADS=$(nproc) - - - name: Java test - run: make javatest NUM_THREADS=$(nproc) - - - name: Rust share build - run: echo '[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - - - name: Rust set env - run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - - - name: Rust test - working-directory: tools/rust_api - run: | - cargo test --release --features arrow -- --test-threads=1 - - - name: Rust example - working-directory: examples/rust - run: cargo build --release --features arrow - - rhel-fedora-build-test: - strategy: - matrix: - image: ["rockylinux:8", "rockylinux:9", "fedora:38"] - fail-fast: false - name: ${{ matrix.image }} - runs-on: ubuntu-latest - container: - image: ${{ matrix.image }} - env: - CC: gcc - CXX: g++ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk - HOME: /root - steps: - - name: Enable EPEL - if: ${{ matrix.image == 'rockylinux:8' || matrix.image == 'rockylinux:9' }} - run: | - dnf update -y - dnf install -y epel-release - dnf update -y - - - name: Enable SHA-1 on Rocky Linux 9 - if: matrix.image == 'rockylinux:9' - run: update-crypto-policies --set LEGACY - - - name: Install packages - run: | - curl -fsSL https://rpm.nodesource.com/setup_20.x | bash - - dnf install -y git cmake ${{ matrix.image == 'rockylinux:8' && 'gcc-toolset-12 python39 python39-devel' || 'gcc gcc-c++ python3-devel' }} java-17-openjdk-devel nodejs - - - name: Enable gcc-toolset-12 and python39 on Rocky Linux 8 - if: matrix.image == 'rockylinux:8' - run: | - alternatives --set python /usr/bin/python3.9 - alternatives --set python3 /usr/bin/python3.9 - echo "PYTHON_EXECUTABLE=/usr/bin/python3.9" >> $GITHUB_ENV - echo "PYBIND11_PYTHON_VERSION=3.9" >> $GITHUB_ENV - source /opt/rh/gcc-toolset-12/enable - echo $PATH >> $GITHUB_PATH - - - uses: actions/checkout@v3 - - - name: Ensure Python dependencies - run: | - python3 -m venv /opt/venv - source /opt/venv/bin/activate - pip3 install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu - pip3 install -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - - - name: Ensure Node.js dependencies - working-directory: tools/nodejs_api - run: npm install --include=dev - - - name: Install Rust - run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - $HOME/.cargo/bin/rustup toolchain install 1.67 - - - name: Build - run: make release NUM_THREADS=$(nproc) - - - name: Test - run: make test NUM_THREADS=$(nproc) - - - name: C and C++ examples - run: make example NUM_THREADS=$(nproc) - - - name: Python test - run: | - source /opt/venv/bin/activate - make pytest NUM_THREADS=$(nproc) - - - name: Node.js test - run: make nodejstest NUM_THREADS=$(nproc) - - - name: Java test - run: make javatest NUM_THREADS=$(nproc) - - - name: Rust share build - run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - - - name: Rust set env - run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - - - name: Rust test - working-directory: tools/rust_api - run: | - cargo test --release --features arrow -- --test-threads=1 - - - name: Rust example - working-directory: examples/rust - run: cargo build --release --features arrow - - centos-7-build-test: - name: centos-7 - runs-on: ubuntu-latest - container: - image: quay.io/pypa/manylinux2014_x86_64 - env: - CC: gcc - CXX: g++ - PYBIND11_PYTHON_VERSION: 3.10 - PYTHON_EXECUTABLE: /usr/local/bin/python3.10 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk - steps: - - name: Install packages - run: | - yum update -y - yum install -y npm java-11-openjdk-devel devtoolset-11 - - - name: Enable gcc-toolset-11 - run: | - source /opt/rh/devtoolset-11/enable - echo $PATH >> $GITHUB_PATH - - - uses: actions/checkout@v3 - - - name: Ensure Python dependencies - run: | - ln -s /usr/local/bin/python3.10 /usr/bin/python3 - python3 -m pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu - python3 -m pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - - - name: Ensure Node.js dependencies - working-directory: tools/nodejs_api - run: npm install --include=dev - - - name: Install Rust - run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - $HOME/.cargo/bin/rustup toolchain install 1.67 - - - name: Build - run: make release NUM_THREADS=$(nproc) - - - name: Test - run: make test NUM_THREADS=$(nproc) - - - name: C and C++ examples - run: make example NUM_THREADS=$(nproc) - - - name: Python test - run: make pytest NUM_THREADS=$(nproc) - - - name: Node.js test - run: make nodejstest NUM_THREADS=$(nproc) - - - name: Java test - run: make javatest NUM_THREADS=$(nproc) - - - name: Rust share build - run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - - - name: Rust set env - run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - - - name: Rust test - working-directory: tools/rust_api - run: | - cargo test --release --features arrow -- --test-threads=1 - - - name: Rust example - working-directory: examples/rust - run: cargo build --release --features arrow - - archlinux-build-test: - name: archlinux - runs-on: ubuntu-latest - container: - image: archlinux:latest - env: - CC: gcc - CXX: g++ - JAVA_HOME: /usr/lib/jvm/java-17-openjdk - steps: - - name: Install packages - run: | - pacman -Syu --noconfirm - pacman -S --needed --noconfirm git base-devel cmake gcc python python-pip npm jdk17-openjdk - - - uses: actions/checkout@v3 - - - name: Ensure Python dependencies - run: | - pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu --break-system-packages - pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html --break-system-packages - - - name: Ensure Node.js dependencies - working-directory: tools/nodejs_api - run: npm install --include=dev - - - name: Install Rust + continue-on-error: true run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - $HOME/.cargo/bin/rustup toolchain install 1.67 - - - name: Build - run: make release NUM_THREADS=$(nproc) + make test NUM_THREADS=$(nproc) + status=$? + echo "Test,$status" >> status.txt + + # - name: C and C++ examples + # continue-on-error: true + # run: | + # make example NUM_THREADS=$(nproc) + # status=$? + # echo "C and C++ examples,$status" >> status.txt + + # - name: Python test + # continue-on-error: true + # run: | + # make pytest NUM_THREADS=$(nproc) + # status=$? + # echo "Python test,$status" >> status.txt + + + # - name: Node.js test + # continue-on-error: true + # run: | + # make nodejstest NUM_THREADS=$(nproc) + # status=$? + # echo "Node.js test,$status" >> status.txt + + # - name: Java test + # continue-on-error: true + # run: | + # make javatest NUM_THREADS=$(nproc) + # status=$? + # echo "Java test,$status" >> status.txt + + # - name: Rust share build + # run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + + # - name: Rust set env + # run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV + + # - name: Rust test + # continue-on-error: true + # working-directory: tools/rust_api + # run: | + # cargo test --release --features arrow -- --test-threads=1 + # status=$? + # echo "Rust test,$status" >> status.txt + + # - name: Rust example + # continue-on-error: true + # working-directory: examples/rust + # run: | + # cargo build --release --features arrow + # status=$? + # echo "Rust example,$status" >> status.txt + + - name: Rename status.txt + run: mv status.txt ubuntu-20.04.csv + + # macos-build-test: + # strategy: + # matrix: + # runner: [macos-12, macos-13] + # fail-fast: false + # name: ${{ matrix.runner }} + # runs-on: ${{ matrix.runner }} + # steps: + # - uses: actions/checkout@v3 + + # - uses: actions/setup-python@v4 + # with: + # python-version: "3.10" + + # - name: Ensure Python dependencies + # run: | + # pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu + # pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html + + # - name: Ensure Node.js dependencies + # working-directory: tools/nodejs_api + # run: npm install --include=dev + + # - name: Build + # run: make release NUM_THREADS=$(sysctl -n hw.physicalcpu) + + # - name: Test + # run: make test NUM_THREADS=$(sysctl -n hw.physicalcpu) + + # - name: C and C++ examples + # run: make example NUM_THREADS=$(sysctl -n hw.physicalcpu) + + # - name: Python test + # run: make pytest NUM_THREADS=$(sysctl -n hw.physicalcpu) + + # - name: Node.js test + # run: make nodejstest NUM_THREADS=$(sysctl -n hw.physicalcpu) + + # - name: Java test + # run: make javatest NUM_THREADS=$(sysctl -n hw.physicalcpu) + + # - name: Rust share build + # run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + + # - name: Rust set env + # run: echo "CARGO_BUILD_JOBS=$(sysctl -n hw.physicalcpu)" >> $GITHUB_ENV + + # - name: Rust test + # working-directory: tools/rust_api + # run: | + # cargo test --release --features arrow -- --test-threads=1 + + # - name: Rust example + # working-directory: examples/rust + # run: cargo build --release --features arrow + + # windows-build-test: + # strategy: + # matrix: + # runner: [windows-2019, windows-2022] + # fail-fast: false + # name: ${{ matrix.runner }} + # runs-on: ${{ matrix.runner }} + # steps: + # - name: Disable Windows Defender + # shell: powershell + # run: | + # Set-MpPreference -DisableRealtimeMonitoring $true + # Set-MpPreference -DisableBehaviorMonitoring $true + # Set-MpPreference -DisableBlockAtFirstSeen $true + # Set-MpPreference -DisableIOAVProtection $true + # Set-MpPreference -DisablePrivacyMode $true + # Set-MpPreference -SignatureDisableUpdateOnStartupWithoutEngine $true + # Set-MpPreference -DisableArchiveScanning $true + # Set-MpPreference -DisableIntrusionPreventionSystem $true + # Set-MpPreference -DisableScriptScanning $true + # Set-MpPreference -SubmitSamplesConsent NeverSend + + # - uses: actions/checkout@v3 + + # - uses: ilammy/msvc-dev-cmd@v1 + + # - uses: actions/setup-python@v4 + # with: + # python-version: "3.10" + + # - name: Ensure Python dependencies + # shell: cmd + # run: | + # pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu + # pip install --user -r tools\python_api\requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html + + # - name: Ensure Node.js dependencies + # shell: cmd + # working-directory: .\tools\nodejs_api + # run: npm install --include=dev + + # - name: Build + # shell: cmd + # run: make release NUM_THREADS=%NUMBER_OF_PROCESSORS% + + # - name: Test + # shell: cmd + # run: make test NUM_THREADS=%NUMBER_OF_PROCESSORS% + + # - name: C and C++ examples + # shell: cmd + # run: make example NUM_THREADS=%NUMBER_OF_PROCESSORS% + + # - name: Python test + # shell: cmd + # run: make pytest NUM_THREADS=%NUMBER_OF_PROCESSORS% + + # - name: Node.js test + # shell: cmd + # run: make nodejstest NUM_THREADS=%NUMBER_OF_PROCESSORS% + + # - name: Java test + # shell: cmd + # run: make javatest NUM_THREADS=%NUMBER_OF_PROCESSORS% + + # - name: Rust share build + # shell: cmd + # run: (echo [workspace]& echo members = ["tools/rust_api","examples/rust"]) > Cargo.toml + + # - name: Rust test + # shell: cmd + # run: | + # make clean + # set OPENSSL_DIR=C:\Program Files\OpenSSL-Win64 + # set CXXFLAGS=/std:c++20 + # set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% + # cargo test --release --features arrow -- --test-threads=1 + + # - name: Rust example + # shell: cmd + # run: | + # set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% + # cargo build --release --features arrow + + # debian-ubuntu-build-test: + # strategy: + # matrix: + # image: ["ubuntu:23.04", "ubuntu:23.10", "debian:12", "debian:sid"] + # fail-fast: false + # name: ${{ matrix.image }} + # runs-on: ubuntu-latest + # container: + # image: ${{ matrix.image }} + # env: + # CC: gcc + # CXX: g++ + # JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64 + # steps: + # - name: Install packages + # run: | + # apt-get update + # apt-get install -y ca-certificates curl gnupg + # mkdir -p /etc/apt/keyrings + # curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg + # echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + # apt-get update + # apt-get install -y git build-essential cmake gcc g++ python3 python3-dev python3-pip openjdk-17-jdk nodejs + + # - uses: actions/checkout@v3 + + # - name: Ensure Python dependencies + # run: | + # pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu ${{ matrix.image != 'debian:11' && '--break-system-packages' || '' }} + # pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html ${{ matrix.image != 'debian:11' && '--break-system-packages' || '' }} + + # - name: Ensure Node.js dependencies + # working-directory: tools/nodejs_api + # run: npm install --include=dev + + # - name: Install Rust + # run: | + # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain + # echo "$HOME/.cargo/bin" >> $GITHUB_PATH + # $HOME/.cargo/bin/rustup toolchain install 1.67 + + # - name: Build + # run: make release NUM_THREADS=$(nproc) + + # - name: Test + # run: make test NUM_THREADS=$(nproc) + + # - name: C and C++ examples + # run: make example NUM_THREADS=$(nproc) + + # - name: Python test + # run: make pytest NUM_THREADS=$(nproc) + + # - name: Node.js test + # run: make nodejstest NUM_THREADS=$(nproc) + + # - name: Java test + # run: make javatest NUM_THREADS=$(nproc) + + # - name: Rust share build + # run: echo '[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + + # - name: Rust set env + # run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV + + # - name: Rust test + # working-directory: tools/rust_api + # run: | + # cargo test --release --features arrow -- --test-threads=1 + + # - name: Rust example + # working-directory: examples/rust + # run: cargo build --release --features arrow + + # rhel-fedora-build-test: + # strategy: + # matrix: + # image: ["rockylinux:8", "rockylinux:9", "fedora:38"] + # fail-fast: false + # name: ${{ matrix.image }} + # runs-on: ubuntu-latest + # container: + # image: ${{ matrix.image }} + # env: + # CC: gcc + # CXX: g++ + # JAVA_HOME: /usr/lib/jvm/java-17-openjdk + # HOME: /root + # steps: + # - name: Enable EPEL + # if: ${{ matrix.image == 'rockylinux:8' || matrix.image == 'rockylinux:9' }} + # run: | + # dnf update -y + # dnf install -y epel-release + # dnf update -y + + # - name: Enable SHA-1 on Rocky Linux 9 + # if: matrix.image == 'rockylinux:9' + # run: update-crypto-policies --set LEGACY + + # - name: Install packages + # run: | + # curl -fsSL https://rpm.nodesource.com/setup_20.x | bash - + # dnf install -y git cmake ${{ matrix.image == 'rockylinux:8' && 'gcc-toolset-12 python39 python39-devel' || 'gcc gcc-c++ python3-devel' }} java-17-openjdk-devel nodejs + + # - name: Enable gcc-toolset-12 and python39 on Rocky Linux 8 + # if: matrix.image == 'rockylinux:8' + # run: | + # alternatives --set python /usr/bin/python3.9 + # alternatives --set python3 /usr/bin/python3.9 + # echo "PYTHON_EXECUTABLE=/usr/bin/python3.9" >> $GITHUB_ENV + # echo "PYBIND11_PYTHON_VERSION=3.9" >> $GITHUB_ENV + # source /opt/rh/gcc-toolset-12/enable + # echo $PATH >> $GITHUB_PATH + + # - uses: actions/checkout@v3 + + # - name: Ensure Python dependencies + # run: | + # python3 -m venv /opt/venv + # source /opt/venv/bin/activate + # pip3 install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu + # pip3 install -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html + + # - name: Ensure Node.js dependencies + # working-directory: tools/nodejs_api + # run: npm install --include=dev + + # - name: Install Rust + # run: | + # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain + # echo "$HOME/.cargo/bin" >> $GITHUB_PATH + # $HOME/.cargo/bin/rustup toolchain install 1.67 + + # - name: Build + # run: make release NUM_THREADS=$(nproc) + + # - name: Test + # run: make test NUM_THREADS=$(nproc) + + # - name: C and C++ examples + # run: make example NUM_THREADS=$(nproc) + + # - name: Python test + # run: | + # source /opt/venv/bin/activate + # make pytest NUM_THREADS=$(nproc) + + # - name: Node.js test + # run: make nodejstest NUM_THREADS=$(nproc) + + # - name: Java test + # run: make javatest NUM_THREADS=$(nproc) + + # - name: Rust share build + # run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + + # - name: Rust set env + # run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV + + # - name: Rust test + # working-directory: tools/rust_api + # run: | + # cargo test --release --features arrow -- --test-threads=1 + + # - name: Rust example + # working-directory: examples/rust + # run: cargo build --release --features arrow + + # centos-7-build-test: + # name: centos-7 + # runs-on: ubuntu-latest + # container: + # image: quay.io/pypa/manylinux2014_x86_64 + # env: + # CC: gcc + # CXX: g++ + # PYBIND11_PYTHON_VERSION: 3.10 + # PYTHON_EXECUTABLE: /usr/local/bin/python3.10 + # JAVA_HOME: /usr/lib/jvm/java-11-openjdk + # steps: + # - name: Install packages + # run: | + # yum update -y + # yum install -y npm java-11-openjdk-devel devtoolset-11 + + # - name: Enable gcc-toolset-11 + # run: | + # source /opt/rh/devtoolset-11/enable + # echo $PATH >> $GITHUB_PATH + + # - uses: actions/checkout@v3 + + # - name: Ensure Python dependencies + # run: | + # ln -s /usr/local/bin/python3.10 /usr/bin/python3 + # python3 -m pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu + # python3 -m pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html + + # - name: Ensure Node.js dependencies + # working-directory: tools/nodejs_api + # run: npm install --include=dev + + # - name: Install Rust + # run: | + # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain + # echo "$HOME/.cargo/bin" >> $GITHUB_PATH + # $HOME/.cargo/bin/rustup toolchain install 1.67 + + # - name: Build + # run: make release NUM_THREADS=$(nproc) + + # - name: Test + # run: make test NUM_THREADS=$(nproc) + + # - name: C and C++ examples + # run: make example NUM_THREADS=$(nproc) + + # - name: Python test + # run: make pytest NUM_THREADS=$(nproc) + + # - name: Node.js test + # run: make nodejstest NUM_THREADS=$(nproc) + + # - name: Java test + # run: make javatest NUM_THREADS=$(nproc) + + # - name: Rust share build + # run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + + # - name: Rust set env + # run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV + + # - name: Rust test + # working-directory: tools/rust_api + # run: | + # cargo test --release --features arrow -- --test-threads=1 + + # - name: Rust example + # working-directory: examples/rust + # run: cargo build --release --features arrow + + # archlinux-build-test: + # name: archlinux + # runs-on: ubuntu-latest + # container: + # image: archlinux:latest + # env: + # CC: gcc + # CXX: g++ + # JAVA_HOME: /usr/lib/jvm/java-17-openjdk + # steps: + # - name: Install packages + # run: | + # pacman -Syu --noconfirm + # pacman -S --needed --noconfirm git base-devel cmake gcc python python-pip npm jdk17-openjdk + + # - uses: actions/checkout@v3 + + # - name: Ensure Python dependencies + # run: | + # pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu --break-system-packages + # pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html --break-system-packages + + # - name: Ensure Node.js dependencies + # working-directory: tools/nodejs_api + # run: npm install --include=dev + + # - name: Install Rust + # run: | + # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain + # echo "$HOME/.cargo/bin" >> $GITHUB_PATH + # $HOME/.cargo/bin/rustup toolchain install 1.67 + + # - name: Build + # run: make release NUM_THREADS=$(nproc) + + # - name: Test + # run: make test NUM_THREADS=$(nproc) + + # - name: C and C++ examples + # run: make example NUM_THREADS=$(nproc) + + # - name: Python test + # run: make pytest NUM_THREADS=$(nproc) - - name: Test - run: make test NUM_THREADS=$(nproc) + # - name: Node.js test + # run: make nodejstest NUM_THREADS=$(nproc) - - name: C and C++ examples - run: make example NUM_THREADS=$(nproc) + # - name: Java test + # run: make javatest NUM_THREADS=$(nproc) - - name: Python test - run: make pytest NUM_THREADS=$(nproc) + # - name: Rust share build + # run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + + # - name: Rust set env + # run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - - name: Node.js test - run: make nodejstest NUM_THREADS=$(nproc) - - - name: Java test - run: make javatest NUM_THREADS=$(nproc) - - - name: Rust share build - run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - - - name: Rust set env - run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - - - name: Rust test - working-directory: tools/rust_api - run: | - cargo test --release --features arrow -- --test-threads=1 + # - name: Rust test + # working-directory: tools/rust_api + # run: | + # cargo test --release --features arrow -- --test-threads=1 - - name: Rust example - working-directory: examples/rust - run: cargo build --release --features arrow + # - name: Rust example + # working-directory: examples/rust + # run: cargo build --release --features arrow From ab8d91532ff6e2d6520c6dcaba1ec49d27366ce5 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Thu, 7 Mar 2024 21:03:17 +0800 Subject: [PATCH 02/29] X --- .github/workflows/multiplatform-build-test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index e0a03f2d4c..f2122fd55b 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -99,6 +99,11 @@ jobs: - name: Rename status.txt run: mv status.txt ubuntu-20.04.csv + + - uses: actions/upload-artifact@v4 + with: + name: ubuntu-20.04 + path: ubuntu-20.04.csv # macos-build-test: # strategy: From 7de4776d80ea66af1b2634a2ce78c489e9c2233d Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Thu, 7 Mar 2024 21:13:27 +0800 Subject: [PATCH 03/29] X --- .github/workflows/multiplatform-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index f2122fd55b..35fad541cf 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -28,7 +28,7 @@ jobs: - name: Ensure Python dependencies run: | - pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu + pip install torch==2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html pip install --user backports.zoneinfo From 702729a5ad3dcce6658abdaf463031db0dd387f7 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Thu, 7 Mar 2024 21:21:24 +0800 Subject: [PATCH 04/29] X --- .github/workflows/multiplatform-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index 35fad541cf..8db59d2245 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -24,7 +24,7 @@ jobs: sudo apt-get install -y build-essential manpages-dev software-properties-common sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt-get update - sudo apt-get install -y gcc-11 g++-11 python3.9 + sudo apt-get install -y gcc-11 g++-11 python3.10 - name: Ensure Python dependencies run: | From 23b0117a9655992c3de455fa49de9b71c3a4cd41 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Thu, 7 Mar 2024 21:23:22 +0800 Subject: [PATCH 05/29] X --- .github/workflows/multiplatform-build-test.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index 8db59d2245..3eb7a785a9 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -24,7 +24,11 @@ jobs: sudo apt-get install -y build-essential manpages-dev software-properties-common sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt-get update - sudo apt-get install -y gcc-11 g++-11 python3.10 + sudo apt-get install -y gcc-11 g++-11 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" - name: Ensure Python dependencies run: | @@ -60,7 +64,6 @@ jobs: # status=$? # echo "Python test,$status" >> status.txt - # - name: Node.js test # continue-on-error: true # run: | @@ -96,10 +99,10 @@ jobs: # cargo build --release --features arrow # status=$? # echo "Rust example,$status" >> status.txt - + - name: Rename status.txt run: mv status.txt ubuntu-20.04.csv - + - uses: actions/upload-artifact@v4 with: name: ubuntu-20.04 From 0c958c1eb6ab48911536961d71c546bac26d6c60 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Thu, 7 Mar 2024 21:23:35 +0800 Subject: [PATCH 06/29] X --- .github/workflows/multiplatform-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index 3eb7a785a9..e01b592ce2 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -32,7 +32,7 @@ jobs: - name: Ensure Python dependencies run: | - pip install torch==2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu + pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html pip install --user backports.zoneinfo From cd57b03c86c50d573acdfd84a4bb207ac91ef3a2 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Thu, 7 Mar 2024 21:27:15 +0800 Subject: [PATCH 07/29] X --- .github/workflows/multiplatform-build-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index e01b592ce2..7c465b378f 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -34,7 +34,6 @@ jobs: run: | pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - pip install --user backports.zoneinfo - name: Ensure Node.js dependencies working-directory: tools/nodejs_api From 09be21c88803142cfc9d4fbc6fa909f7ffad3f58 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Thu, 7 Mar 2024 21:33:49 +0800 Subject: [PATCH 08/29] X --- .../workflows/multiplatform-build-test.yml | 117 +++++++++++------- 1 file changed, 75 insertions(+), 42 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index 7c465b378f..b3c9416683 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -107,62 +107,95 @@ jobs: name: ubuntu-20.04 path: ubuntu-20.04.csv - # macos-build-test: - # strategy: - # matrix: - # runner: [macos-12, macos-13] - # fail-fast: false - # name: ${{ matrix.runner }} - # runs-on: ${{ matrix.runner }} - # steps: - # - uses: actions/checkout@v3 + macos-build-test: + strategy: + matrix: + runner: [macos-13, macos-14] + fail-fast: false + name: ${{ matrix.runner }} + runs-on: ${{ matrix.runner }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v3 - # - uses: actions/setup-python@v4 - # with: - # python-version: "3.10" + - uses: actions/setup-python@v4 + with: + python-version: "3.10" - # - name: Ensure Python dependencies - # run: | - # pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu - # pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html + - name: Ensure Python dependencies + run: | + pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu + pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - # - name: Ensure Node.js dependencies - # working-directory: tools/nodejs_api - # run: npm install --include=dev + - name: Ensure Node.js dependencies + working-directory: tools/nodejs_api + run: npm install --include=dev - # - name: Build - # run: make release NUM_THREADS=$(sysctl -n hw.physicalcpu) + - name: Build + run: make release NUM_THREADS=$(sysctl -n hw.physicalcpu) - # - name: Test - # run: make test NUM_THREADS=$(sysctl -n hw.physicalcpu) + - name: Test + continue-on-error: true + run: | + make test NUM_THREADS=$(sysctl -n hw.physicalcpu) + $status=$? + echo "Test,$status" >> status.txt - # - name: C and C++ examples - # run: make example NUM_THREADS=$(sysctl -n hw.physicalcpu) + - name: C and C++ examples + continue-on-error: true + run: | + make example NUM_THREADS=$(sysctl -n hw.physicalcpu) + $status=$? + echo "C and C++ examples,$status" >> status.txt - # - name: Python test - # run: make pytest NUM_THREADS=$(sysctl -n hw.physicalcpu) + - name: Python test + continue-on-error: true + run: | + make pytest NUM_THREADS=$(sysctl -n hw.physicalcpu) + $status=$? + echo "Python test,$status" >> status.txt - # - name: Node.js test - # run: make nodejstest NUM_THREADS=$(sysctl -n hw.physicalcpu) + - name: Node.js test + continue-on-error: true + run: | + make nodejstest NUM_THREADS=$(sysctl -n hw.physicalcpu) + $status=$? + echo "Node.js test,$status" >> status.txt - # - name: Java test - # run: make javatest NUM_THREADS=$(sysctl -n hw.physicalcpu) + - name: Java test + continue-on-error: true + run: | + make javatest NUM_THREADS=$(sysctl -n hw.physicalcpu) + $status=$? + echo "Java test,$status" >> status.txt - # - name: Rust share build - # run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + - name: Rust share build + run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - # - name: Rust set env - # run: echo "CARGO_BUILD_JOBS=$(sysctl -n hw.physicalcpu)" >> $GITHUB_ENV + - name: Rust set env + run: echo "CARGO_BUILD_JOBS=$(sysctl -n hw.physicalcpu)" >> $GITHUB_ENV - # - name: Rust test - # working-directory: tools/rust_api - # run: | - # cargo test --release --features arrow -- --test-threads=1 + - name: Rust test + continue-on-error: true + working-directory: tools/rust_api + run: | + cargo test --release --features arrow -- --test-threads=1 - # - name: Rust example - # working-directory: examples/rust - # run: cargo build --release --features arrow + - name: Rust example + continue-on-error: true + working-directory: examples/rust + run: cargo build --release --features arrow + - name: Rename status.txt + run: mv status.txt ${{ matrix.runner }}.csv + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.runner }} + path: ${{ matrix.runner }}.csv + # windows-build-test: # strategy: # matrix: From f2cb375263b2012521283f4c12f449295c10f2e8 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Thu, 7 Mar 2024 22:14:12 +0800 Subject: [PATCH 09/29] X --- .../workflows/multiplatform-build-test.yml | 123 +++++++++--------- 1 file changed, 62 insertions(+), 61 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index b3c9416683..b506cac9c2 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -49,55 +49,55 @@ jobs: status=$? echo "Test,$status" >> status.txt - # - name: C and C++ examples - # continue-on-error: true - # run: | - # make example NUM_THREADS=$(nproc) - # status=$? - # echo "C and C++ examples,$status" >> status.txt - - # - name: Python test - # continue-on-error: true - # run: | - # make pytest NUM_THREADS=$(nproc) - # status=$? - # echo "Python test,$status" >> status.txt - - # - name: Node.js test - # continue-on-error: true - # run: | - # make nodejstest NUM_THREADS=$(nproc) - # status=$? - # echo "Node.js test,$status" >> status.txt - - # - name: Java test - # continue-on-error: true - # run: | - # make javatest NUM_THREADS=$(nproc) - # status=$? - # echo "Java test,$status" >> status.txt - - # - name: Rust share build - # run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - - # - name: Rust set env - # run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - - # - name: Rust test - # continue-on-error: true - # working-directory: tools/rust_api - # run: | - # cargo test --release --features arrow -- --test-threads=1 - # status=$? - # echo "Rust test,$status" >> status.txt - - # - name: Rust example - # continue-on-error: true - # working-directory: examples/rust - # run: | - # cargo build --release --features arrow - # status=$? - # echo "Rust example,$status" >> status.txt + - name: C and C++ examples + continue-on-error: true + run: | + make example NUM_THREADS=$(nproc) + status=$? + echo "C and C++ examples,$status" >> status.txt + + - name: Python test + continue-on-error: true + run: | + make pytest NUM_THREADS=$(nproc) + status=$? + echo "Python test,$status" >> status.txt + + - name: Node.js test + continue-on-error: true + run: | + make nodejstest NUM_THREADS=$(nproc) + status=$? + echo "Node.js test,$status" >> status.txt + + - name: Java test + continue-on-error: true + run: | + make javatest NUM_THREADS=$(nproc) + status=$? + echo "Java test,$status" >> status.txt + + - name: Rust share build + run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + + - name: Rust set env + run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV + + - name: Rust test + continue-on-error: true + working-directory: tools/rust_api + run: | + cargo test --release --features arrow -- --test-threads=1 + status=$? + echo "Rust test,$status" >> status.txt + + - name: Rust example + continue-on-error: true + working-directory: examples/rust + run: | + cargo build --release --features arrow + status=$? + echo "Rust example,$status" >> status.txt - name: Rename status.txt run: mv status.txt ubuntu-20.04.csv @@ -123,6 +123,9 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.10" + + - run: | + echo ${{ matrix.runner }} - name: Ensure Python dependencies run: | @@ -140,36 +143,31 @@ jobs: continue-on-error: true run: | make test NUM_THREADS=$(sysctl -n hw.physicalcpu) - $status=$? - echo "Test,$status" >> status.txt + echo "Test,$?" >> status.txt - name: C and C++ examples continue-on-error: true run: | make example NUM_THREADS=$(sysctl -n hw.physicalcpu) - $status=$? - echo "C and C++ examples,$status" >> status.txt + echo "C and C++ examples,$?" >> status.txt - name: Python test continue-on-error: true run: | make pytest NUM_THREADS=$(sysctl -n hw.physicalcpu) - $status=$? - echo "Python test,$status" >> status.txt + echo "Python test,$?" >> status.txt - name: Node.js test continue-on-error: true run: | make nodejstest NUM_THREADS=$(sysctl -n hw.physicalcpu) - $status=$? - echo "Node.js test,$status" >> status.txt + echo "Node.js test,$?" >> status.txt - name: Java test continue-on-error: true run: | make javatest NUM_THREADS=$(sysctl -n hw.physicalcpu) - $status=$? - echo "Java test,$status" >> status.txt + echo "Java test,$?" >> status.txt - name: Rust share build run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml @@ -182,11 +180,14 @@ jobs: working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 + echo "Rust test,$?" >> status.txt - name: Rust example continue-on-error: true working-directory: examples/rust - run: cargo build --release --features arrow + run: | + cargo build --release --features arrow + echo "Rust example,$?" >> status.txt - name: Rename status.txt run: mv status.txt ${{ matrix.runner }}.csv @@ -195,7 +196,7 @@ jobs: with: name: ${{ matrix.runner }} path: ${{ matrix.runner }}.csv - + # windows-build-test: # strategy: # matrix: From ce36e27c5bd35200bcc220608a4db6e56905bf3a Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 17:32:12 +0800 Subject: [PATCH 10/29] X --- .../workflows/multiplatform-build-test.yml | 181 ++++++++++-------- 1 file changed, 104 insertions(+), 77 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index b506cac9c2..8d722d1c2f 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -40,42 +40,39 @@ jobs: run: npm install --include=dev - name: Build - run: make release NUM_THREADS=$(nproc) + run: | + make release NUM_THREADS=$(nproc) + echo "Build,$?" > status.txt - name: Test continue-on-error: true run: | make test NUM_THREADS=$(nproc) - status=$? - echo "Test,$status" >> status.txt + echo "Test,$?" >> status.txt - name: C and C++ examples continue-on-error: true run: | make example NUM_THREADS=$(nproc) - status=$? - echo "C and C++ examples,$status" >> status.txt + echo "C and C++ examples,$?" >> status.txt - name: Python test continue-on-error: true run: | make pytest NUM_THREADS=$(nproc) - status=$? - echo "Python test,$status" >> status.txt + echo "Python test,$?" >> status.txt - name: Node.js test continue-on-error: true run: | make nodejstest NUM_THREADS=$(nproc) - status=$? - echo "Node.js test,$status" >> status.txt + echo "Node.js test,$?" >> status.txt - name: Java test continue-on-error: true run: | make javatest NUM_THREADS=$(nproc) - status=$? - echo "Java test,$status" >> status.txt + echo "Java test,$?" >> status.txt - name: Rust share build run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml @@ -88,16 +85,14 @@ jobs: working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 - status=$? - echo "Rust test,$status" >> status.txt + echo "Rust test,$?" >> ../../status.txt - name: Rust example continue-on-error: true working-directory: examples/rust run: | cargo build --release --features arrow - status=$? - echo "Rust example,$status" >> status.txt + echo "Rust example,$?" >> ../../status.txt - name: Rename status.txt run: mv status.txt ubuntu-20.04.csv @@ -137,7 +132,9 @@ jobs: run: npm install --include=dev - name: Build - run: make release NUM_THREADS=$(sysctl -n hw.physicalcpu) + run: | + make release NUM_THREADS=$(sysctl -n hw.physicalcpu) + echo "Build,$?" > status.txt - name: Test continue-on-error: true @@ -180,14 +177,14 @@ jobs: working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 - echo "Rust test,$?" >> status.txt + echo "Rust test,$?" >> ../../status.txt - name: Rust example continue-on-error: true working-directory: examples/rust run: | cargo build --release --features arrow - echo "Rust example,$?" >> status.txt + echo "Rust example,$?" >> ../../status.txt - name: Rename status.txt run: mv status.txt ${{ matrix.runner }}.csv @@ -281,79 +278,109 @@ jobs: # set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% # cargo build --release --features arrow - # debian-ubuntu-build-test: - # strategy: - # matrix: - # image: ["ubuntu:23.04", "ubuntu:23.10", "debian:12", "debian:sid"] - # fail-fast: false - # name: ${{ matrix.image }} - # runs-on: ubuntu-latest - # container: - # image: ${{ matrix.image }} - # env: - # CC: gcc - # CXX: g++ - # JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64 - # steps: - # - name: Install packages - # run: | - # apt-get update - # apt-get install -y ca-certificates curl gnupg - # mkdir -p /etc/apt/keyrings - # curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg - # echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list - # apt-get update - # apt-get install -y git build-essential cmake gcc g++ python3 python3-dev python3-pip openjdk-17-jdk nodejs + debian-ubuntu-build-test: + strategy: + matrix: + image: ["ubuntu:23.04", "ubuntu:23.10", "debian:12", "debian:sid"] + fail-fast: false + name: ${{ matrix.image }} + runs-on: ubuntu-latest + container: + image: ${{ matrix.image }} + env: + CC: gcc + CXX: g++ + JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64 + steps: + - name: Install packages + run: | + apt-get update + apt-get install -y ca-certificates curl gnupg + mkdir -p /etc/apt/keyrings + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg + echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + apt-get update + apt-get install -y git build-essential cmake gcc g++ python3 python3-dev python3-pip openjdk-17-jdk nodejs - # - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - # - name: Ensure Python dependencies - # run: | - # pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu ${{ matrix.image != 'debian:11' && '--break-system-packages' || '' }} - # pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html ${{ matrix.image != 'debian:11' && '--break-system-packages' || '' }} + - name: Ensure Python dependencies + run: | + pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu ${{ matrix.image != 'debian:11' && '--break-system-packages' || '' }} + pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html ${{ matrix.image != 'debian:11' && '--break-system-packages' || '' }} - # - name: Ensure Node.js dependencies - # working-directory: tools/nodejs_api - # run: npm install --include=dev + - name: Ensure Node.js dependencies + working-directory: tools/nodejs_api + run: npm install --include=dev - # - name: Install Rust - # run: | - # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain - # echo "$HOME/.cargo/bin" >> $GITHUB_PATH - # $HOME/.cargo/bin/rustup toolchain install 1.67 + - name: Install Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + $HOME/.cargo/bin/rustup toolchain install 1.67 - # - name: Build - # run: make release NUM_THREADS=$(nproc) + - name: Build + run: | + make release NUM_THREADS=$(nproc) + echo "Build,$?" > status.txt - # - name: Test - # run: make test NUM_THREADS=$(nproc) + - name: Test + continue-on-error: true + run: | + make test NUM_THREADS=$(nproc) + echo "Test,$?" >> status.txt - # - name: C and C++ examples - # run: make example NUM_THREADS=$(nproc) + - name: C and C++ examples + continue-on-error: true + run: | + make example NUM_THREADS=$(nproc) + echo "C and C++ examples,$?" >> status.txt - # - name: Python test - # run: make pytest NUM_THREADS=$(nproc) + - name: Python test + continue-on-error: true + run: | + make pytest NUM_THREADS=$(nproc) + echo "Python test,$?" >> status.txt - # - name: Node.js test - # run: make nodejstest NUM_THREADS=$(nproc) + - name: Node.js test + continue-on-error: true + run: | + make nodejstest NUM_THREADS=$(nproc) + echo "Node.js test,$?" >> status.txt - # - name: Java test - # run: make javatest NUM_THREADS=$(nproc) + - name: Java test + continue-on-error: true + run: | + make javatest NUM_THREADS=$(nproc) + echo "Java test,$?" >> status.txt - # - name: Rust share build - # run: echo '[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + - name: Rust share build + run: echo '[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - # - name: Rust set env - # run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV + - name: Rust set env + run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - # - name: Rust test - # working-directory: tools/rust_api - # run: | - # cargo test --release --features arrow -- --test-threads=1 + - name: Rust test + continue-on-error: true + working-directory: tools/rust_api + run: | + cargo test --release --features arrow -- --test-threads=1 + echo "Rust test,$?" >> ../../status.txt - # - name: Rust example - # working-directory: examples/rust - # run: cargo build --release --features arrow + - name: Rust example + continue-on-error: true + working-directory: examples/rust + run: | + cargo build --release --features arrow + echo "Rust example,$?" >> ../../status.txt + + - name: Rename status.txt + run: mv status.txt ${{ matrix.image }}.csv + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.image }} + path: ${{ matrix.image }}.csv # rhel-fedora-build-test: # strategy: From 6519a09da39238c24888eeae7a5f63bb47834f59 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 17:39:21 +0800 Subject: [PATCH 11/29] X --- tools/python_api/test/test_exception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/python_api/test/test_exception.py b/tools/python_api/test/test_exception.py index 46f493f4ae..1d3dd14868 100644 --- a/tools/python_api/test/test_exception.py +++ b/tools/python_api/test/test_exception.py @@ -14,13 +14,13 @@ def test_exception(establish_connection): def test_db_path_exception(): - path = '/:* /? " < > |' + path = "" if sys.platform == "win32": error_message = 'Failed to create directory' else: error_message = 'filesystem error' with pytest.raises(RuntimeError, match=error_message): - kuzu.Database(str(path)) + kuzu.Database(path) def test_read_only_exception(establish_connection): # TODO: Enable this test on Windows when the read-only mode is implemented. From a8c8ca75ab369022bc7289eb8d0c791b4fa1743d Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 18:42:52 +0800 Subject: [PATCH 12/29] X --- .../workflows/multiplatform-build-test.yml | 473 +++++++++++------- 1 file changed, 282 insertions(+), 191 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index 8d722d1c2f..20cd526246 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -118,7 +118,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.10" - + - run: | echo ${{ matrix.runner }} @@ -373,244 +373,335 @@ jobs: run: | cargo build --release --features arrow echo "Rust example,$?" >> ../../status.txt - + - name: Rename status.txt - run: mv status.txt ${{ matrix.image }}.csv + run: | + PLATFORM=$(echo ${{ matrix.image }} | tr -d ':') + echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV + mv status.txt $PLATFORM.csv - uses: actions/upload-artifact@v4 with: - name: ${{ matrix.image }} - path: ${{ matrix.image }}.csv + name: ${{env.PLATFORM}} + path: ${{env.PLATFORM}}.csv - # rhel-fedora-build-test: - # strategy: - # matrix: - # image: ["rockylinux:8", "rockylinux:9", "fedora:38"] - # fail-fast: false - # name: ${{ matrix.image }} - # runs-on: ubuntu-latest - # container: - # image: ${{ matrix.image }} - # env: - # CC: gcc - # CXX: g++ - # JAVA_HOME: /usr/lib/jvm/java-17-openjdk - # HOME: /root - # steps: - # - name: Enable EPEL - # if: ${{ matrix.image == 'rockylinux:8' || matrix.image == 'rockylinux:9' }} - # run: | - # dnf update -y - # dnf install -y epel-release - # dnf update -y + rhel-fedora-build-test: + strategy: + matrix: + image: ["rockylinux:8", "rockylinux:9", "fedora:38"] + fail-fast: false + name: ${{ matrix.image }} + runs-on: ubuntu-latest + container: + image: ${{ matrix.image }} + env: + CC: gcc + CXX: g++ + JAVA_HOME: /usr/lib/jvm/java-17-openjdk + HOME: /root + steps: + - name: Enable EPEL + if: ${{ matrix.image == 'rockylinux:8' || matrix.image == 'rockylinux:9' }} + run: | + dnf update -y + dnf install -y epel-release + dnf update -y - # - name: Enable SHA-1 on Rocky Linux 9 - # if: matrix.image == 'rockylinux:9' - # run: update-crypto-policies --set LEGACY + - name: Enable SHA-1 on Rocky Linux 9 + if: matrix.image == 'rockylinux:9' + run: update-crypto-policies --set LEGACY - # - name: Install packages - # run: | - # curl -fsSL https://rpm.nodesource.com/setup_20.x | bash - - # dnf install -y git cmake ${{ matrix.image == 'rockylinux:8' && 'gcc-toolset-12 python39 python39-devel' || 'gcc gcc-c++ python3-devel' }} java-17-openjdk-devel nodejs + - name: Install packages + run: | + curl -fsSL https://rpm.nodesource.com/setup_20.x | bash - + dnf install -y git cmake ${{ matrix.image == 'rockylinux:8' && 'gcc-toolset-12 python3.11 python3.11-devel' || 'gcc gcc-c++ python3-devel' }} java-17-openjdk-devel nodejs - # - name: Enable gcc-toolset-12 and python39 on Rocky Linux 8 - # if: matrix.image == 'rockylinux:8' - # run: | - # alternatives --set python /usr/bin/python3.9 - # alternatives --set python3 /usr/bin/python3.9 - # echo "PYTHON_EXECUTABLE=/usr/bin/python3.9" >> $GITHUB_ENV - # echo "PYBIND11_PYTHON_VERSION=3.9" >> $GITHUB_ENV - # source /opt/rh/gcc-toolset-12/enable - # echo $PATH >> $GITHUB_PATH + - name: Enable gcc-toolset-12 and python3.11 on Rocky Linux 8 + if: matrix.image == 'rockylinux:8' + run: | + alternatives --set python /usr/bin/python3.11 + alternatives --set python3 /usr/bin/python3.11 + echo "PYTHON_EXECUTABLE=/usr/bin/python3.11" >> $GITHUB_ENV + echo "PYBIND11_PYTHON_VERSION=3.11" >> $GITHUB_ENV + source /opt/rh/gcc-toolset-12/enable + echo $PATH >> $GITHUB_PATH - # - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - # - name: Ensure Python dependencies - # run: | - # python3 -m venv /opt/venv - # source /opt/venv/bin/activate - # pip3 install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu - # pip3 install -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html + - name: Ensure Python dependencies + run: | + python3 -m venv /opt/venv + source /opt/venv/bin/activate + pip3 install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu + pip3 install -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - # - name: Ensure Node.js dependencies - # working-directory: tools/nodejs_api - # run: npm install --include=dev + - name: Ensure Node.js dependencies + working-directory: tools/nodejs_api + run: npm install --include=dev - # - name: Install Rust - # run: | - # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain - # echo "$HOME/.cargo/bin" >> $GITHUB_PATH - # $HOME/.cargo/bin/rustup toolchain install 1.67 + - name: Install Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + $HOME/.cargo/bin/rustup toolchain install 1.67 - # - name: Build - # run: make release NUM_THREADS=$(nproc) + - name: Build + run: | + make release NUM_THREADS=$(nproc) + echo "Build,$?" > status.txt - # - name: Test - # run: make test NUM_THREADS=$(nproc) + - name: Test + run: | + make test NUM_THREADS=$(nproc) + echo "Test,$?" >> status.txt - # - name: C and C++ examples - # run: make example NUM_THREADS=$(nproc) + - name: C and C++ examples + run: | + make example NUM_THREADS=$(nproc) + echo "C and C++ examples,$?" >> status.txt - # - name: Python test - # run: | - # source /opt/venv/bin/activate - # make pytest NUM_THREADS=$(nproc) + - name: Python test + run: | + source /opt/venv/bin/activate + make pytest NUM_THREADS=$(nproc) + echo "Python test,$?" >> status.txt - # - name: Node.js test - # run: make nodejstest NUM_THREADS=$(nproc) + - name: Node.js test + run: | + make nodejstest NUM_THREADS=$(nproc) + echo "Node.js test,$?" >> status.txt - # - name: Java test - # run: make javatest NUM_THREADS=$(nproc) + - name: Java test + run: | + make javatest NUM_THREADS=$(nproc) + echo "Java test,$?" >> status.txt - # - name: Rust share build - # run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + - name: Rust share build + run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - # - name: Rust set env - # run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV + - name: Rust set env + run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - # - name: Rust test - # working-directory: tools/rust_api - # run: | - # cargo test --release --features arrow -- --test-threads=1 + - name: Rust test + working-directory: tools/rust_api + run: | + cargo test --release --features arrow -- --test-threads=1 + echo "Rust test,$?" >> ../../status.txt - # - name: Rust example - # working-directory: examples/rust - # run: cargo build --release --features arrow - - # centos-7-build-test: - # name: centos-7 - # runs-on: ubuntu-latest - # container: - # image: quay.io/pypa/manylinux2014_x86_64 - # env: - # CC: gcc - # CXX: g++ - # PYBIND11_PYTHON_VERSION: 3.10 - # PYTHON_EXECUTABLE: /usr/local/bin/python3.10 - # JAVA_HOME: /usr/lib/jvm/java-11-openjdk - # steps: - # - name: Install packages - # run: | - # yum update -y - # yum install -y npm java-11-openjdk-devel devtoolset-11 + - name: Rust example + working-directory: examples/rust + run: | + cargo build --release --features arrow + echo "Rust example,$?" >> ../../status.txt - # - name: Enable gcc-toolset-11 - # run: | - # source /opt/rh/devtoolset-11/enable - # echo $PATH >> $GITHUB_PATH + - name: Rename status.txt + run: | + PLATFORM=$(echo ${{ matrix.image }} | tr -d ':') + echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV + mv status.txt $PLATFORM.csv - # - uses: actions/checkout@v3 + - uses: actions/upload-artifact@v4 + with: + name: ${{env.PLATFORM}} + path: ${{env.PLATFORM}}.csv - # - name: Ensure Python dependencies - # run: | - # ln -s /usr/local/bin/python3.10 /usr/bin/python3 - # python3 -m pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu - # python3 -m pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html + centos-7-build-test: + name: centos-7 + runs-on: ubuntu-latest + container: + image: quay.io/pypa/manylinux2014_x86_64 + env: + CC: gcc + CXX: g++ + PYBIND11_PYTHON_VERSION: 3.10 + PYTHON_EXECUTABLE: /usr/local/bin/python3.10 + JAVA_HOME: /usr/lib/jvm/java-11-openjdk + steps: + - name: Install packages + run: | + yum update -y + yum install -y npm java-11-openjdk-devel devtoolset-11 - # - name: Ensure Node.js dependencies - # working-directory: tools/nodejs_api - # run: npm install --include=dev + - name: Enable gcc-toolset-11 + run: | + source /opt/rh/devtoolset-11/enable + echo $PATH >> $GITHUB_PATH - # - name: Install Rust - # run: | - # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain - # echo "$HOME/.cargo/bin" >> $GITHUB_PATH - # $HOME/.cargo/bin/rustup toolchain install 1.67 + - uses: actions/checkout@v3 - # - name: Build - # run: make release NUM_THREADS=$(nproc) + - name: Ensure Python dependencies + run: | + ln -s /usr/local/bin/python3.10 /usr/bin/python3 + python3 -m pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu + python3 -m pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - # - name: Test - # run: make test NUM_THREADS=$(nproc) + - name: Ensure Node.js dependencies + working-directory: tools/nodejs_api + run: npm install --include=dev - # - name: C and C++ examples - # run: make example NUM_THREADS=$(nproc) + - name: Install Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + $HOME/.cargo/bin/rustup toolchain install 1.67 - # - name: Python test - # run: make pytest NUM_THREADS=$(nproc) + - name: Build + run: | + make release NUM_THREADS=$(nproc) + echo "Build,$?" > status.txt - # - name: Node.js test - # run: make nodejstest NUM_THREADS=$(nproc) + - name: Test + continue-on-error: true + run: | + make test NUM_THREADS=$(nproc) + echo "Test,$?" >> status.txt - # - name: Java test - # run: make javatest NUM_THREADS=$(nproc) + - name: C and C++ examples + continue-on-error: true + run: | + make example NUM_THREADS=$(nproc) + echo "C and C++ examples,$?" >> status.txt - # - name: Rust share build - # run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + - name: Python test + continue-on-error: true + run: | + make pytest NUM_THREADS=$(nproc) + echo "Python test,$?" >> status.txt - # - name: Rust set env - # run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV + - name: Node.js test + continue-on-error: true + run: | + make nodejstest NUM_THREADS=$(nproc) + echo "Node.js test,$?" >> status.txt - # - name: Rust test - # working-directory: tools/rust_api - # run: | - # cargo test --release --features arrow -- --test-threads=1 + - name: Java test + continue-on-error: true + run: | + make javatest NUM_THREADS=$(nproc) + echo "Java test,$?" >> status.txt - # - name: Rust example - # working-directory: examples/rust - # run: cargo build --release --features arrow - - # archlinux-build-test: - # name: archlinux - # runs-on: ubuntu-latest - # container: - # image: archlinux:latest - # env: - # CC: gcc - # CXX: g++ - # JAVA_HOME: /usr/lib/jvm/java-17-openjdk - # steps: - # - name: Install packages - # run: | - # pacman -Syu --noconfirm - # pacman -S --needed --noconfirm git base-devel cmake gcc python python-pip npm jdk17-openjdk + - name: Rust share build + run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - # - uses: actions/checkout@v3 + - name: Rust set env + run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - # - name: Ensure Python dependencies - # run: | - # pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu --break-system-packages - # pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html --break-system-packages + - name: Rust test + continue-on-error: true + working-directory: tools/rust_api + run: | + cargo test --release --features arrow -- --test-threads=1 + echo "Rust test,$?" >> ../status.txt - # - name: Ensure Node.js dependencies - # working-directory: tools/nodejs_api - # run: npm install --include=dev + - name: Rust example + continue-on-error: true + working-directory: examples/rust + run: | + cargo build --release --features arrow + echo "Rust example,$?" >> ../status.txt - # - name: Install Rust - # run: | - # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain - # echo "$HOME/.cargo/bin" >> $GITHUB_PATH - # $HOME/.cargo/bin/rustup toolchain install 1.67 + - name: Rename status.txt + run: mv status.txt centos-7.csv - # - name: Build - # run: make release NUM_THREADS=$(nproc) + - uses: actions/upload-artifact@v4 + with: + name: centos-7 + path: centos-7.csv - # - name: Test - # run: make test NUM_THREADS=$(nproc) + archlinux-build-test: + name: archlinux + runs-on: ubuntu-latest + container: + image: archlinux:latest + env: + CC: gcc + CXX: g++ + JAVA_HOME: /usr/lib/jvm/java-17-openjdk + steps: + - name: Install packages + run: | + pacman -Syu --noconfirm + pacman -S --needed --noconfirm git base-devel cmake gcc python python-pip npm jdk17-openjdk - # - name: C and C++ examples - # run: make example NUM_THREADS=$(nproc) + - uses: actions/checkout@v3 - # - name: Python test - # run: make pytest NUM_THREADS=$(nproc) + - name: Ensure Python dependencies + run: | + pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu --break-system-packages + pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html --break-system-packages - # - name: Node.js test - # run: make nodejstest NUM_THREADS=$(nproc) + - name: Ensure Node.js dependencies + working-directory: tools/nodejs_api + run: npm install --include=dev - # - name: Java test - # run: make javatest NUM_THREADS=$(nproc) + - name: Install Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + $HOME/.cargo/bin/rustup toolchain install 1.67 - # - name: Rust share build - # run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + - name: Build + run: | + make release NUM_THREADS=$(nproc) + echo "Build,$?" > status.txt - # - name: Rust set env - # run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV + - name: Test + continue-on-error: true + run: | + make test NUM_THREADS=$(nproc) + echo "Test,$?" >> status.txt - # - name: Rust test - # working-directory: tools/rust_api - # run: | - # cargo test --release --features arrow -- --test-threads=1 + - name: C and C++ examples + continue-on-error: true + run: | + make example NUM_THREADS=$(nproc) + echo "C and C++ examples,$?" >> status.txt - # - name: Rust example - # working-directory: examples/rust - # run: cargo build --release --features arrow + - name: Python test + continue-on-error: true + run: | + make pytest NUM_THREADS=$(nproc) + echo "Python test,$?" >> status.txt + + - name: Node.js test + continue-on-error: true + run: | + make nodejstest NUM_THREADS=$(nproc) + echo "Node.js test,$?" >> status.txt + + - name: Java test + continue-on-error: true + run: | + make javatest NUM_THREADS=$(nproc) + echo "Java test,$?" >> status.txt + + - name: Rust share build + run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml + + - name: Rust set env + run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV + + - name: Rust test + continue-on-error: true + working-directory: tools/rust_api + run: | + cargo test --release --features arrow -- --test-threads=1 + echo "Rust test,$?" >> ../../status.txt + + - name: Rust example + continue-on-error: true + working-directory: examples/rust + run: | + cargo build --release --features arrow + echo "Rust example,$?" >> ../../status.txt + + - name: Rename status.txt + run: | + PLATFORM=$(echo ${{ matrix.image }} | tr -d ':') + echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV + mv status.txt $PLATFORM.csv + + - uses: actions/upload-artifact@v4 + with: + name: ${{env.PLATFORM}} + path: ${{env.PLATFORM}}.csv From 708447cabd384ffeda73f3d1b5b8c4c9d30403d7 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 19:04:20 +0800 Subject: [PATCH 13/29] X --- .../workflows/multiplatform-build-test.yml | 197 ++++++++++-------- 1 file changed, 114 insertions(+), 83 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index 20cd526246..896b6a7f43 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -194,89 +194,120 @@ jobs: name: ${{ matrix.runner }} path: ${{ matrix.runner }}.csv - # windows-build-test: - # strategy: - # matrix: - # runner: [windows-2019, windows-2022] - # fail-fast: false - # name: ${{ matrix.runner }} - # runs-on: ${{ matrix.runner }} - # steps: - # - name: Disable Windows Defender - # shell: powershell - # run: | - # Set-MpPreference -DisableRealtimeMonitoring $true - # Set-MpPreference -DisableBehaviorMonitoring $true - # Set-MpPreference -DisableBlockAtFirstSeen $true - # Set-MpPreference -DisableIOAVProtection $true - # Set-MpPreference -DisablePrivacyMode $true - # Set-MpPreference -SignatureDisableUpdateOnStartupWithoutEngine $true - # Set-MpPreference -DisableArchiveScanning $true - # Set-MpPreference -DisableIntrusionPreventionSystem $true - # Set-MpPreference -DisableScriptScanning $true - # Set-MpPreference -SubmitSamplesConsent NeverSend - - # - uses: actions/checkout@v3 - - # - uses: ilammy/msvc-dev-cmd@v1 - - # - uses: actions/setup-python@v4 - # with: - # python-version: "3.10" - - # - name: Ensure Python dependencies - # shell: cmd - # run: | - # pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu - # pip install --user -r tools\python_api\requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - - # - name: Ensure Node.js dependencies - # shell: cmd - # working-directory: .\tools\nodejs_api - # run: npm install --include=dev - - # - name: Build - # shell: cmd - # run: make release NUM_THREADS=%NUMBER_OF_PROCESSORS% - - # - name: Test - # shell: cmd - # run: make test NUM_THREADS=%NUMBER_OF_PROCESSORS% - - # - name: C and C++ examples - # shell: cmd - # run: make example NUM_THREADS=%NUMBER_OF_PROCESSORS% - - # - name: Python test - # shell: cmd - # run: make pytest NUM_THREADS=%NUMBER_OF_PROCESSORS% - - # - name: Node.js test - # shell: cmd - # run: make nodejstest NUM_THREADS=%NUMBER_OF_PROCESSORS% - - # - name: Java test - # shell: cmd - # run: make javatest NUM_THREADS=%NUMBER_OF_PROCESSORS% - - # - name: Rust share build - # shell: cmd - # run: (echo [workspace]& echo members = ["tools/rust_api","examples/rust"]) > Cargo.toml - - # - name: Rust test - # shell: cmd - # run: | - # make clean - # set OPENSSL_DIR=C:\Program Files\OpenSSL-Win64 - # set CXXFLAGS=/std:c++20 - # set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% - # cargo test --release --features arrow -- --test-threads=1 - - # - name: Rust example - # shell: cmd - # run: | - # set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% - # cargo build --release --features arrow + windows-build-test: + strategy: + matrix: + runner: [windows-2019, windows-2022] + fail-fast: false + name: ${{ matrix.runner }} + runs-on: ${{ matrix.runner }} + steps: + - name: Disable Windows Defender + shell: powershell + run: | + Set-MpPreference -DisableRealtimeMonitoring $true + Set-MpPreference -DisableBehaviorMonitoring $true + Set-MpPreference -DisableBlockAtFirstSeen $true + Set-MpPreference -DisableIOAVProtection $true + Set-MpPreference -DisablePrivacyMode $true + Set-MpPreference -SignatureDisableUpdateOnStartupWithoutEngine $true + Set-MpPreference -DisableArchiveScanning $true + Set-MpPreference -DisableIntrusionPreventionSystem $true + Set-MpPreference -DisableScriptScanning $true + Set-MpPreference -SubmitSamplesConsent NeverSend + + - uses: actions/checkout@v3 + + - uses: ilammy/msvc-dev-cmd@v1 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Ensure Python dependencies + shell: cmd + run: | + pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu + pip install --user -r tools\python_api\requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html + + - name: Ensure Node.js dependencies + shell: cmd + working-directory: .\tools\nodejs_api + run: npm install --include=dev + + - name: Build + shell: cmd + run: | + make release NUM_THREADS=%NUMBER_OF_PROCESSORS% + echo Build,%ERRORLEVEL% > status.txt + + - name: Test + shell: cmd + continue-on-error: true + run: | + make test NUM_THREADS=%NUMBER_OF_PROCESSORS% + echo Test,%ERRORLEVEL% >> status.txt + + - name: C and C++ examples + shell: cmd + continue-on-error: true + run: | + make example NUM_THREADS=%NUMBER_OF_PROCESSORS% + echo C and C++ examples,%ERRORLEVEL% >> status.txt + + - name: Python test + shell: cmd + continue-on-error: true + run: | + make pytest NUM_THREADS=%NUMBER_OF_PROCESSORS% + echo Python test,%ERRORLEVEL% >> status.txt + + - name: Node.js test + shell: cmd + continue-on-error: true + run: | + make nodejstest NUM_THREADS=%NUMBER_OF_PROCESSORS% + echo Node.js test,%ERRORLEVEL% >> status.txt + + - name: Java test + shell: cmd + continue-on-error: true + run: | + make javatest NUM_THREADS=%NUMBER_OF_PROCESSORS% + echo Java test,%ERRORLEVEL% >> status.txt + + - name: Rust share build + shell: cmd + run: (echo [workspace]& echo members = ["tools/rust_api","examples/rust"]) > Cargo.toml + + - name: Rust test + shell: cmd + continue-on-error: true + run: | + make clean + set OPENSSL_DIR=C:\Program Files\OpenSSL-Win64 + set CXXFLAGS=/std:c++20 + set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% + cargo test --release --features arrow -- --test-threads=1 + echo Rust test,%ERRORLEVEL% >> status.txt + + - name: Rust example + shell: cmd + continue-on-error: true + run: | + set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% + cargo build --release --features arrow + echo Rust example,%ERRORLEVEL% >> status.txt + + - name: Rename status.txt + shell: cmd + run: | + rename status.txt ${{ matrix.runner }}.csv + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.runner }} + path: ${{ matrix.runner }}.csv debian-ubuntu-build-test: strategy: From 0968f2d720bf811187d7e6035e3bec214ef45655 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 19:11:22 +0800 Subject: [PATCH 14/29] X --- .github/workflows/multiplatform-build-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index 896b6a7f43..ceb259b09f 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -348,7 +348,7 @@ jobs: run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain echo "$HOME/.cargo/bin" >> $GITHUB_PATH - $HOME/.cargo/bin/rustup toolchain install 1.67 + $HOME/.cargo/bin/rustup toolchain install 1.76 - name: Build run: | @@ -474,7 +474,7 @@ jobs: run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain echo "$HOME/.cargo/bin" >> $GITHUB_PATH - $HOME/.cargo/bin/rustup toolchain install 1.67 + $HOME/.cargo/bin/rustup toolchain install 1.76 - name: Build run: | @@ -574,7 +574,7 @@ jobs: run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain echo "$HOME/.cargo/bin" >> $GITHUB_PATH - $HOME/.cargo/bin/rustup toolchain install 1.67 + $HOME/.cargo/bin/rustup toolchain install 1.76 - name: Build run: | @@ -669,7 +669,7 @@ jobs: run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain echo "$HOME/.cargo/bin" >> $GITHUB_PATH - $HOME/.cargo/bin/rustup toolchain install 1.67 + $HOME/.cargo/bin/rustup toolchain install 1.76 - name: Build run: | From ce407ba834e0cb81c6f27158791f69511ec6cac1 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 19:14:58 +0800 Subject: [PATCH 15/29] X --- .../workflows/multiplatform-build-test.yml | 105 +----------------- 1 file changed, 1 insertion(+), 104 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index ceb259b09f..b21ce20174 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -419,7 +419,7 @@ jobs: rhel-fedora-build-test: strategy: matrix: - image: ["rockylinux:8", "rockylinux:9", "fedora:38"] + image: ["rockylinux:8", "rockylinux:9", "fedora:38", "fedora:39"] fail-fast: false name: ${{ matrix.image }} runs-on: ubuntu-latest @@ -536,109 +536,6 @@ jobs: name: ${{env.PLATFORM}} path: ${{env.PLATFORM}}.csv - centos-7-build-test: - name: centos-7 - runs-on: ubuntu-latest - container: - image: quay.io/pypa/manylinux2014_x86_64 - env: - CC: gcc - CXX: g++ - PYBIND11_PYTHON_VERSION: 3.10 - PYTHON_EXECUTABLE: /usr/local/bin/python3.10 - JAVA_HOME: /usr/lib/jvm/java-11-openjdk - steps: - - name: Install packages - run: | - yum update -y - yum install -y npm java-11-openjdk-devel devtoolset-11 - - - name: Enable gcc-toolset-11 - run: | - source /opt/rh/devtoolset-11/enable - echo $PATH >> $GITHUB_PATH - - - uses: actions/checkout@v3 - - - name: Ensure Python dependencies - run: | - ln -s /usr/local/bin/python3.10 /usr/bin/python3 - python3 -m pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu - python3 -m pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - - - name: Ensure Node.js dependencies - working-directory: tools/nodejs_api - run: npm install --include=dev - - - name: Install Rust - run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - $HOME/.cargo/bin/rustup toolchain install 1.76 - - - name: Build - run: | - make release NUM_THREADS=$(nproc) - echo "Build,$?" > status.txt - - - name: Test - continue-on-error: true - run: | - make test NUM_THREADS=$(nproc) - echo "Test,$?" >> status.txt - - - name: C and C++ examples - continue-on-error: true - run: | - make example NUM_THREADS=$(nproc) - echo "C and C++ examples,$?" >> status.txt - - - name: Python test - continue-on-error: true - run: | - make pytest NUM_THREADS=$(nproc) - echo "Python test,$?" >> status.txt - - - name: Node.js test - continue-on-error: true - run: | - make nodejstest NUM_THREADS=$(nproc) - echo "Node.js test,$?" >> status.txt - - - name: Java test - continue-on-error: true - run: | - make javatest NUM_THREADS=$(nproc) - echo "Java test,$?" >> status.txt - - - name: Rust share build - run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - - - name: Rust set env - run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - - - name: Rust test - continue-on-error: true - working-directory: tools/rust_api - run: | - cargo test --release --features arrow -- --test-threads=1 - echo "Rust test,$?" >> ../status.txt - - - name: Rust example - continue-on-error: true - working-directory: examples/rust - run: | - cargo build --release --features arrow - echo "Rust example,$?" >> ../status.txt - - - name: Rename status.txt - run: mv status.txt centos-7.csv - - - uses: actions/upload-artifact@v4 - with: - name: centos-7 - path: centos-7.csv - archlinux-build-test: name: archlinux runs-on: ubuntu-latest From 5cfbcc02d8e2f438949648dc7fb03583cc7da8af Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 19:17:25 +0800 Subject: [PATCH 16/29] X --- .github/workflows/multiplatform-build-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index b21ce20174..a8b09b385f 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -407,7 +407,7 @@ jobs: - name: Rename status.txt run: | - PLATFORM=$(echo ${{ matrix.image }} | tr -d ':') + PLATFORM=$(echo ${{ matrix.image }} | tr ':' '-') echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV mv status.txt $PLATFORM.csv @@ -527,7 +527,7 @@ jobs: - name: Rename status.txt run: | - PLATFORM=$(echo ${{ matrix.image }} | tr -d ':') + PLATFORM=$(echo ${{ matrix.image }} | tr ':' '-') echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV mv status.txt $PLATFORM.csv @@ -625,7 +625,7 @@ jobs: - name: Rename status.txt run: | - PLATFORM=$(echo ${{ matrix.image }} | tr -d ':') + PLATFORM=$(echo ${{ matrix.image }} | tr ':' '-') echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV mv status.txt $PLATFORM.csv From d5064566e6b9ba06f4745868f3b606b9bc230509 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 19:27:29 +0800 Subject: [PATCH 17/29] X --- .github/workflows/multiplatform-build-test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index a8b09b385f..c069e8c4b7 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -437,6 +437,10 @@ jobs: dnf update -y dnf install -y epel-release dnf update -y + + - name: Cleanup + run: | + rm -rf /root/.docker - name: Enable SHA-1 on Rocky Linux 9 if: matrix.image == 'rockylinux:9' From db0f95649481964673a22b1cd7ef262727025644 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 19:34:42 +0800 Subject: [PATCH 18/29] X --- .github/workflows/multiplatform-build-test.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index c069e8c4b7..afef11bf9f 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -437,10 +437,6 @@ jobs: dnf update -y dnf install -y epel-release dnf update -y - - - name: Cleanup - run: | - rm -rf /root/.docker - name: Enable SHA-1 on Rocky Linux 9 if: matrix.image == 'rockylinux:9' @@ -467,8 +463,8 @@ jobs: run: | python3 -m venv /opt/venv source /opt/venv/bin/activate - pip3 install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu - pip3 install -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html + pip3 install torch~=2.2.1 --extra-index-url https://download.pytorch.org/whl/cpu + pip3 install -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.2.1+cpu.html - name: Ensure Node.js dependencies working-directory: tools/nodejs_api From 2f1202a36469f4c64d9ced079032fe39978a56e7 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 19:42:21 +0800 Subject: [PATCH 19/29] X --- .github/workflows/multiplatform-build-test.yml | 12 ++++++++++++ scripts/multiplatform-test-helper/collect-results.py | 0 2 files changed, 12 insertions(+) create mode 100644 scripts/multiplatform-test-helper/collect-results.py diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index afef11bf9f..bbb81adcbf 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -633,3 +633,15 @@ jobs: with: name: ${{env.PLATFORM}} path: ${{env.PLATFORM}}.csv + + collect-results: + runs-on: ubuntu-latest + needs: [ubuntu-2004-build-test, macos-build-test, windows-build-test, debian-ubuntu-build-test, rhel-fedora-build-test, archlinux-build-test] + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: results + + - name: List artifacts + run: ls -R results diff --git a/scripts/multiplatform-test-helper/collect-results.py b/scripts/multiplatform-test-helper/collect-results.py new file mode 100644 index 0000000000..e69de29bb2 From 702db9b5d815485bdaec31923a15ee8022d2da9f Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 19:42:24 +0800 Subject: [PATCH 20/29] X --- scripts/multiplatform-test-helper/collect-results.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/multiplatform-test-helper/collect-results.py b/scripts/multiplatform-test-helper/collect-results.py index e69de29bb2..63f77b6be1 100644 --- a/scripts/multiplatform-test-helper/collect-results.py +++ b/scripts/multiplatform-test-helper/collect-results.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 + From ebaefe0bc8d9451fbe79bc85aa025f816d62f157 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 20:03:12 +0800 Subject: [PATCH 21/29] X --- .../workflows/multiplatform-build-test.yml | 30 ++++++++++++++----- .../collect-results.py | 16 ++++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index bbb81adcbf..4d90094f18 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -625,23 +625,37 @@ jobs: - name: Rename status.txt run: | - PLATFORM=$(echo ${{ matrix.image }} | tr ':' '-') - echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV - mv status.txt $PLATFORM.csv + mv status.txt archlinux.csv - uses: actions/upload-artifact@v4 with: - name: ${{env.PLATFORM}} - path: ${{env.PLATFORM}}.csv + name: archlinux + path: archlinux.csv collect-results: runs-on: ubuntu-latest - needs: [ubuntu-2004-build-test, macos-build-test, windows-build-test, debian-ubuntu-build-test, rhel-fedora-build-test, archlinux-build-test] + needs: + [ + ubuntu-2004-build-test, + macos-build-test, + windows-build-test, + debian-ubuntu-build-test, + rhel-fedora-build-test, + archlinux-build-test, + ] steps: + - uses: actions/checkout@v3 + - name: Download artifacts uses: actions/download-artifact@v4 - with: + with: path: results - + - name: List artifacts run: ls -R results + + - name: Create summary + run: | + $GITHUB_URL=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} + pip3 install pandas + python3 scripts/multiplatform-test-helper/collect-results.py results diff --git a/scripts/multiplatform-test-helper/collect-results.py b/scripts/multiplatform-test-helper/collect-results.py index 63f77b6be1..57bdf25f99 100644 --- a/scripts/multiplatform-test-helper/collect-results.py +++ b/scripts/multiplatform-test-helper/collect-results.py @@ -1,2 +1,18 @@ #!/usr/bin/env python3 +import os +import sys +import pandas as pd + +def main(): + if len(sys.argv) != 2: + print("Usage: collect-results.py ") + sys.exit(1) + + results_dir = sys.argv[1] + results = {} + + for root, dirs, files in os.walk(results_dir): + for file in files: + if file.endswith(".csv"): + results[file] = pd.read_csv(os.path.join(root, file)) From a725de7316edafe58b117a6e36d3e213c5e8120f Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 20:36:56 +0800 Subject: [PATCH 22/29] X --- .../workflows/multiplatform-build-test.yml | 41 +++---------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index 4d90094f18..f74f73716d 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -7,6 +7,7 @@ on: jobs: ubuntu-2004-build-test: + continue-on-error: true name: ubuntu-20.04 runs-on: ubuntu-20.04 defaults: @@ -45,31 +46,26 @@ jobs: echo "Build,$?" > status.txt - name: Test - continue-on-error: true run: | make test NUM_THREADS=$(nproc) echo "Test,$?" >> status.txt - name: C and C++ examples - continue-on-error: true run: | make example NUM_THREADS=$(nproc) echo "C and C++ examples,$?" >> status.txt - name: Python test - continue-on-error: true run: | make pytest NUM_THREADS=$(nproc) echo "Python test,$?" >> status.txt - name: Node.js test - continue-on-error: true run: | make nodejstest NUM_THREADS=$(nproc) echo "Node.js test,$?" >> status.txt - name: Java test - continue-on-error: true run: | make javatest NUM_THREADS=$(nproc) echo "Java test,$?" >> status.txt @@ -81,14 +77,12 @@ jobs: run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - name: Rust test - continue-on-error: true working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 echo "Rust test,$?" >> ../../status.txt - name: Rust example - continue-on-error: true working-directory: examples/rust run: | cargo build --release --features arrow @@ -103,6 +97,7 @@ jobs: path: ubuntu-20.04.csv macos-build-test: + continue-on-error: true strategy: matrix: runner: [macos-13, macos-14] @@ -137,31 +132,26 @@ jobs: echo "Build,$?" > status.txt - name: Test - continue-on-error: true run: | make test NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "Test,$?" >> status.txt - name: C and C++ examples - continue-on-error: true run: | make example NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "C and C++ examples,$?" >> status.txt - name: Python test - continue-on-error: true run: | make pytest NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "Python test,$?" >> status.txt - name: Node.js test - continue-on-error: true run: | make nodejstest NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "Node.js test,$?" >> status.txt - name: Java test - continue-on-error: true run: | make javatest NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "Java test,$?" >> status.txt @@ -173,14 +163,12 @@ jobs: run: echo "CARGO_BUILD_JOBS=$(sysctl -n hw.physicalcpu)" >> $GITHUB_ENV - name: Rust test - continue-on-error: true working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 echo "Rust test,$?" >> ../../status.txt - name: Rust example - continue-on-error: true working-directory: examples/rust run: | cargo build --release --features arrow @@ -195,6 +183,7 @@ jobs: path: ${{ matrix.runner }}.csv windows-build-test: + continue-on-error: true strategy: matrix: runner: [windows-2019, windows-2022] @@ -243,35 +232,30 @@ jobs: - name: Test shell: cmd - continue-on-error: true run: | make test NUM_THREADS=%NUMBER_OF_PROCESSORS% echo Test,%ERRORLEVEL% >> status.txt - name: C and C++ examples shell: cmd - continue-on-error: true run: | make example NUM_THREADS=%NUMBER_OF_PROCESSORS% echo C and C++ examples,%ERRORLEVEL% >> status.txt - name: Python test shell: cmd - continue-on-error: true run: | make pytest NUM_THREADS=%NUMBER_OF_PROCESSORS% echo Python test,%ERRORLEVEL% >> status.txt - name: Node.js test shell: cmd - continue-on-error: true run: | make nodejstest NUM_THREADS=%NUMBER_OF_PROCESSORS% echo Node.js test,%ERRORLEVEL% >> status.txt - name: Java test shell: cmd - continue-on-error: true run: | make javatest NUM_THREADS=%NUMBER_OF_PROCESSORS% echo Java test,%ERRORLEVEL% >> status.txt @@ -282,7 +266,6 @@ jobs: - name: Rust test shell: cmd - continue-on-error: true run: | make clean set OPENSSL_DIR=C:\Program Files\OpenSSL-Win64 @@ -293,7 +276,6 @@ jobs: - name: Rust example shell: cmd - continue-on-error: true run: | set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% cargo build --release --features arrow @@ -310,6 +292,7 @@ jobs: path: ${{ matrix.runner }}.csv debian-ubuntu-build-test: + continue-on-error: true strategy: matrix: image: ["ubuntu:23.04", "ubuntu:23.10", "debian:12", "debian:sid"] @@ -356,31 +339,26 @@ jobs: echo "Build,$?" > status.txt - name: Test - continue-on-error: true run: | make test NUM_THREADS=$(nproc) echo "Test,$?" >> status.txt - name: C and C++ examples - continue-on-error: true run: | make example NUM_THREADS=$(nproc) echo "C and C++ examples,$?" >> status.txt - name: Python test - continue-on-error: true run: | make pytest NUM_THREADS=$(nproc) echo "Python test,$?" >> status.txt - name: Node.js test - continue-on-error: true run: | make nodejstest NUM_THREADS=$(nproc) echo "Node.js test,$?" >> status.txt - name: Java test - continue-on-error: true run: | make javatest NUM_THREADS=$(nproc) echo "Java test,$?" >> status.txt @@ -392,14 +370,12 @@ jobs: run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - name: Rust test - continue-on-error: true working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 echo "Rust test,$?" >> ../../status.txt - name: Rust example - continue-on-error: true working-directory: examples/rust run: | cargo build --release --features arrow @@ -417,6 +393,7 @@ jobs: path: ${{env.PLATFORM}}.csv rhel-fedora-build-test: + continue-on-error: true strategy: matrix: image: ["rockylinux:8", "rockylinux:9", "fedora:38", "fedora:39"] @@ -537,6 +514,7 @@ jobs: path: ${{env.PLATFORM}}.csv archlinux-build-test: + continue-on-error: true name: archlinux runs-on: ubuntu-latest container: @@ -574,31 +552,26 @@ jobs: echo "Build,$?" > status.txt - name: Test - continue-on-error: true run: | make test NUM_THREADS=$(nproc) echo "Test,$?" >> status.txt - name: C and C++ examples - continue-on-error: true run: | make example NUM_THREADS=$(nproc) echo "C and C++ examples,$?" >> status.txt - name: Python test - continue-on-error: true run: | make pytest NUM_THREADS=$(nproc) echo "Python test,$?" >> status.txt - name: Node.js test - continue-on-error: true run: | make nodejstest NUM_THREADS=$(nproc) echo "Node.js test,$?" >> status.txt - name: Java test - continue-on-error: true run: | make javatest NUM_THREADS=$(nproc) echo "Java test,$?" >> status.txt @@ -610,14 +583,12 @@ jobs: run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - name: Rust test - continue-on-error: true working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 echo "Rust test,$?" >> ../../status.txt - name: Rust example - continue-on-error: true working-directory: examples/rust run: | cargo build --release --features arrow From cbb4e90b1592cc31df618ed8cc5cd78a9af502aa Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 21:05:48 +0800 Subject: [PATCH 23/29] X --- .../workflows/multiplatform-build-test.yml | 8 ++- .../collect-results.py | 54 +++++++++++++++---- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index f74f73716d..37801699bb 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -628,5 +628,11 @@ jobs: - name: Create summary run: | $GITHUB_URL=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} - pip3 install pandas + pip3 install pandas tabulate python3 scripts/multiplatform-test-helper/collect-results.py results + + - name: Upload summary + uses: actions/upload-artifact@v4 + with: + name: results + path: results.md diff --git a/scripts/multiplatform-test-helper/collect-results.py b/scripts/multiplatform-test-helper/collect-results.py index 57bdf25f99..9d3ebfa93d 100644 --- a/scripts/multiplatform-test-helper/collect-results.py +++ b/scripts/multiplatform-test-helper/collect-results.py @@ -4,15 +4,51 @@ import sys import pandas as pd -def main(): - if len(sys.argv) != 2: - print("Usage: collect-results.py ") - sys.exit(1) +def main(): results_dir = sys.argv[1] - results = {} + results_df_hash = {} + results_exit_codes_hash = {} + results_summary = {} + stages = [] + for root, _, files in os.walk(results_dir): + for csv_file in files: + if not csv_file.endswith(".csv"): + continue + platform = csv_file.split(".")[0] + df = pd.read_csv(os.path.join(root, csv_file), header=None) + df.columns = ["stage", "exit_code"] + results_df_hash[platform] = df + + for platform, df in results_df_hash.items(): + for stage, exit_code in df.values: + if stage not in stages: + stages.append(stage) + if platform not in results_exit_codes_hash: + results_exit_codes_hash[platform] = {} + results_exit_codes_hash[platform][stage] = int(exit_code) + + for platform in results_df_hash.keys(): + results_summary[platform] = [] + for stage in stages: + status = ( + "✅" + if stage in results_exit_codes_hash[platform] + and results_exit_codes_hash[platform][stage] == 0 + else "❌" + ) + results_summary[platform].append({"stage": stage, "status": status}) + + summary_df = {"stage": stages} + for platform, summary in results_summary.items(): + df = pd.DataFrame(summary) + status = df["status"] + summary_df[platform] = status + summary_df = pd.DataFrame(summary_df) + summary_df.index = summary_df["stage"] + del summary_df["stage"] + summary_df.index.name = None - for root, dirs, files in os.walk(results_dir): - for file in files: - if file.endswith(".csv"): - results[file] = pd.read_csv(os.path.join(root, file)) + markdown = summary_df.to_markdown() + with open("results.md", "w") as f: + f.write(markdown) From 7fd8de5a625449fe8479ff930bf1f7dc50492ef5 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 21:09:29 +0800 Subject: [PATCH 24/29] X --- scripts/multiplatform-test-helper/collect-results.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/multiplatform-test-helper/collect-results.py b/scripts/multiplatform-test-helper/collect-results.py index 9d3ebfa93d..2130beb00d 100644 --- a/scripts/multiplatform-test-helper/collect-results.py +++ b/scripts/multiplatform-test-helper/collect-results.py @@ -6,6 +6,12 @@ def main(): + if len(sys.argv) != 2: + print("Usage: collect-results.py ") + sys.exit(1) + if not os.path.isdir(sys.argv[1]): + print(f"Error: {sys.argv[1]} is not a directory") + sys.exit(1) results_dir = sys.argv[1] results_df_hash = {} results_exit_codes_hash = {} @@ -52,3 +58,7 @@ def main(): markdown = summary_df.to_markdown() with open("results.md", "w") as f: f.write(markdown) + + +if __name__ == "__main__": + main() From cc8141e55d36092c46a7e2978409d3c3776fba94 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 21:16:15 +0800 Subject: [PATCH 25/29] X --- scripts/multiplatform-test-helper/collect-results.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/multiplatform-test-helper/collect-results.py b/scripts/multiplatform-test-helper/collect-results.py index 2130beb00d..968ed4a6b8 100644 --- a/scripts/multiplatform-test-helper/collect-results.py +++ b/scripts/multiplatform-test-helper/collect-results.py @@ -3,6 +3,7 @@ import os import sys import pandas as pd +import json def main(): @@ -59,6 +60,9 @@ def main(): with open("results.md", "w") as f: f.write(markdown) + with open("results.json", "w") as f: + json.dump(results_summary, f, indent=4) + if __name__ == "__main__": main() From 455741bc33dfb19acf3eeb56927b342ee6390e67 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Mon, 11 Mar 2024 21:54:33 +0800 Subject: [PATCH 26/29] X --- .../workflows/multiplatform-build-test.yml | 6 ++- .../notify-discord.py | 44 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 scripts/multiplatform-test-helper/notify-discord.py diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index 37801699bb..77e068410e 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -605,6 +605,9 @@ jobs: collect-results: runs-on: ubuntu-latest + env: + DISCORD_CHANNEL_ID: ${{ secrets.DISCORD_CHANNEL_ID }} + DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} needs: [ ubuntu-2004-build-test, @@ -628,8 +631,9 @@ jobs: - name: Create summary run: | $GITHUB_URL=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} - pip3 install pandas tabulate + pip3 install pandas tabulate discord.py python3 scripts/multiplatform-test-helper/collect-results.py results + python3 scripts/multiplatform-test-helper/notify-discord.py results.json - name: Upload summary uses: actions/upload-artifact@v4 diff --git a/scripts/multiplatform-test-helper/notify-discord.py b/scripts/multiplatform-test-helper/notify-discord.py new file mode 100644 index 0000000000..63befce987 --- /dev/null +++ b/scripts/multiplatform-test-helper/notify-discord.py @@ -0,0 +1,44 @@ +import discord +import os +import json +import sys + +TOKEN = os.getenv("DISCORD_BOT_TOKEN") +CHANNEL_ID = os.getenv("DISCORD_CHANNEL_ID") +GITHUB_URL = os.getenv("GITHUB_URL") + +message = "" + + +if __name__ == "__main__": + if not len(sys.argv) == 2: + print("Usage: python send-dicord-notification.py ") + sys.exit(1) + if not os.path.isfile(sys.argv[1]): + print(f"Error: {sys.argv[1]} is not a file") + sys.exit(1) + if not TOKEN: + print("Error: DISCORD_BOT_TOKEN is not set") + sys.exit(1) + + if not CHANNEL_ID: + print("Error: DISCORD_CHANNEL_ID is not set") + sys.exit(1) + client = discord.Client(intents=discord.Intents.default()) + + @client.event + async def on_ready(): + channel = client.get_channel(int(CHANNEL_ID)) + await channel.send(message) + await client.close() + + message += "## Multiplatform test result:\n" + with open(sys.argv[1], "r") as f: + result = json.load(f) + for platform in result: + message += f"- **{platform}**:\n" + for r in result[platform]: + message += f" - {r['stage']}: {r['status']}\n" + if GITHUB_URL: + message += f" [Github]({GITHUB_URL})" + client.run(TOKEN) From 9fede76d9931921e2aa5661dd3ebbb95aaf842a8 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Tue, 12 Mar 2024 00:23:32 +0800 Subject: [PATCH 27/29] X --- .../workflows/multiplatform-build-test.yml | 115 ++++++++++++++++-- 1 file changed, 105 insertions(+), 10 deletions(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index 77e068410e..d9dbe9e249 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -7,7 +7,6 @@ on: jobs: ubuntu-2004-build-test: - continue-on-error: true name: ubuntu-20.04 runs-on: ubuntu-20.04 defaults: @@ -18,8 +17,10 @@ jobs: CXX: g++-11 steps: - uses: actions/checkout@v3 + continue-on-error: true - name: Install packages + continue-on-error: true run: | sudo apt-get update sudo apt-get install -y build-essential manpages-dev software-properties-common @@ -28,76 +29,90 @@ jobs: sudo apt-get install -y gcc-11 g++-11 - uses: actions/setup-python@v5 + continue-on-error: true with: python-version: "3.11" - name: Ensure Python dependencies + continue-on-error: true run: | pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - name: Ensure Node.js dependencies + continue-on-error: true working-directory: tools/nodejs_api run: npm install --include=dev - name: Build + continue-on-error: true run: | make release NUM_THREADS=$(nproc) echo "Build,$?" > status.txt - name: Test + continue-on-error: true run: | make test NUM_THREADS=$(nproc) echo "Test,$?" >> status.txt - name: C and C++ examples + continue-on-error: true run: | make example NUM_THREADS=$(nproc) echo "C and C++ examples,$?" >> status.txt - name: Python test + continue-on-error: true run: | make pytest NUM_THREADS=$(nproc) echo "Python test,$?" >> status.txt - name: Node.js test + continue-on-error: true run: | make nodejstest NUM_THREADS=$(nproc) echo "Node.js test,$?" >> status.txt - name: Java test + continue-on-error: true run: | make javatest NUM_THREADS=$(nproc) echo "Java test,$?" >> status.txt - name: Rust share build + continue-on-error: true run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - name: Rust set env + continue-on-error: true run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - name: Rust test + continue-on-error: true working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 echo "Rust test,$?" >> ../../status.txt - name: Rust example + continue-on-error: true working-directory: examples/rust run: | cargo build --release --features arrow echo "Rust example,$?" >> ../../status.txt - name: Rename status.txt + continue-on-error: true run: mv status.txt ubuntu-20.04.csv - uses: actions/upload-artifact@v4 + continue-on-error: true with: name: ubuntu-20.04 path: ubuntu-20.04.csv macos-build-test: - continue-on-error: true strategy: matrix: runner: [macos-13, macos-14] @@ -109,81 +124,93 @@ jobs: shell: bash steps: - uses: actions/checkout@v3 + continue-on-error: true - uses: actions/setup-python@v4 + continue-on-error: true with: python-version: "3.10" - - run: | - echo ${{ matrix.runner }} - - name: Ensure Python dependencies + continue-on-error: true run: | pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - name: Ensure Node.js dependencies + continue-on-error: true working-directory: tools/nodejs_api run: npm install --include=dev - name: Build + continue-on-error: true run: | make release NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "Build,$?" > status.txt - name: Test + continue-on-error: true run: | make test NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "Test,$?" >> status.txt - name: C and C++ examples + continue-on-error: true run: | make example NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "C and C++ examples,$?" >> status.txt - name: Python test + continue-on-error: true run: | make pytest NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "Python test,$?" >> status.txt - name: Node.js test + continue-on-error: true run: | make nodejstest NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "Node.js test,$?" >> status.txt - name: Java test + continue-on-error: true run: | make javatest NUM_THREADS=$(sysctl -n hw.physicalcpu) echo "Java test,$?" >> status.txt - name: Rust share build + continue-on-error: true run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - name: Rust set env + continue-on-error: true run: echo "CARGO_BUILD_JOBS=$(sysctl -n hw.physicalcpu)" >> $GITHUB_ENV - name: Rust test + continue-on-error: true working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 echo "Rust test,$?" >> ../../status.txt - name: Rust example + continue-on-error: true working-directory: examples/rust run: | cargo build --release --features arrow echo "Rust example,$?" >> ../../status.txt - name: Rename status.txt + continue-on-error: true run: mv status.txt ${{ matrix.runner }}.csv - uses: actions/upload-artifact@v4 + continue-on-error: true with: name: ${{ matrix.runner }} path: ${{ matrix.runner }}.csv windows-build-test: - continue-on-error: true strategy: matrix: runner: [windows-2019, windows-2022] @@ -192,6 +219,7 @@ jobs: runs-on: ${{ matrix.runner }} steps: - name: Disable Windows Defender + continue-on-error: true shell: powershell run: | Set-MpPreference -DisableRealtimeMonitoring $true @@ -206,65 +234,78 @@ jobs: Set-MpPreference -SubmitSamplesConsent NeverSend - uses: actions/checkout@v3 + continue-on-error: true - uses: ilammy/msvc-dev-cmd@v1 + continue-on-error: true - uses: actions/setup-python@v4 + continue-on-error: true with: python-version: "3.10" - name: Ensure Python dependencies + continue-on-error: true shell: cmd run: | pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu pip install --user -r tools\python_api\requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html - name: Ensure Node.js dependencies + continue-on-error: true shell: cmd working-directory: .\tools\nodejs_api run: npm install --include=dev - name: Build + continue-on-error: true shell: cmd run: | make release NUM_THREADS=%NUMBER_OF_PROCESSORS% echo Build,%ERRORLEVEL% > status.txt - name: Test + continue-on-error: true shell: cmd run: | make test NUM_THREADS=%NUMBER_OF_PROCESSORS% echo Test,%ERRORLEVEL% >> status.txt - name: C and C++ examples + continue-on-error: true shell: cmd run: | make example NUM_THREADS=%NUMBER_OF_PROCESSORS% echo C and C++ examples,%ERRORLEVEL% >> status.txt - name: Python test + continue-on-error: true shell: cmd run: | make pytest NUM_THREADS=%NUMBER_OF_PROCESSORS% echo Python test,%ERRORLEVEL% >> status.txt - name: Node.js test + continue-on-error: true shell: cmd run: | make nodejstest NUM_THREADS=%NUMBER_OF_PROCESSORS% echo Node.js test,%ERRORLEVEL% >> status.txt - name: Java test + continue-on-error: true shell: cmd run: | make javatest NUM_THREADS=%NUMBER_OF_PROCESSORS% echo Java test,%ERRORLEVEL% >> status.txt - name: Rust share build + continue-on-error: true shell: cmd run: (echo [workspace]& echo members = ["tools/rust_api","examples/rust"]) > Cargo.toml - name: Rust test + continue-on-error: true shell: cmd run: | make clean @@ -275,6 +316,7 @@ jobs: echo Rust test,%ERRORLEVEL% >> status.txt - name: Rust example + continue-on-error: true shell: cmd run: | set CARGO_BUILD_JOBS=%NUMBER_OF_PROCESSORS% @@ -282,17 +324,18 @@ jobs: echo Rust example,%ERRORLEVEL% >> status.txt - name: Rename status.txt + continue-on-error: true shell: cmd run: | rename status.txt ${{ matrix.runner }}.csv - uses: actions/upload-artifact@v4 + continue-on-error: true with: name: ${{ matrix.runner }} path: ${{ matrix.runner }}.csv debian-ubuntu-build-test: - continue-on-error: true strategy: matrix: image: ["ubuntu:23.04", "ubuntu:23.10", "debian:12", "debian:sid"] @@ -307,6 +350,7 @@ jobs: JAVA_HOME: /usr/lib/jvm/java-17-openjdk-amd64 steps: - name: Install packages + continue-on-error: true run: | apt-get update apt-get install -y ca-certificates curl gnupg @@ -317,83 +361,98 @@ jobs: apt-get install -y git build-essential cmake gcc g++ python3 python3-dev python3-pip openjdk-17-jdk nodejs - uses: actions/checkout@v3 + continue-on-error: true - name: Ensure Python dependencies + continue-on-error: true run: | pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu ${{ matrix.image != 'debian:11' && '--break-system-packages' || '' }} pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html ${{ matrix.image != 'debian:11' && '--break-system-packages' || '' }} - name: Ensure Node.js dependencies working-directory: tools/nodejs_api + continue-on-error: true run: npm install --include=dev - name: Install Rust + continue-on-error: true run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain echo "$HOME/.cargo/bin" >> $GITHUB_PATH $HOME/.cargo/bin/rustup toolchain install 1.76 - name: Build + continue-on-error: true run: | make release NUM_THREADS=$(nproc) echo "Build,$?" > status.txt - name: Test + continue-on-error: true run: | make test NUM_THREADS=$(nproc) echo "Test,$?" >> status.txt - name: C and C++ examples + continue-on-error: true run: | make example NUM_THREADS=$(nproc) echo "C and C++ examples,$?" >> status.txt - name: Python test + continue-on-error: true run: | make pytest NUM_THREADS=$(nproc) echo "Python test,$?" >> status.txt - name: Node.js test + continue-on-error: true run: | make nodejstest NUM_THREADS=$(nproc) echo "Node.js test,$?" >> status.txt - name: Java test + continue-on-error: true run: | make javatest NUM_THREADS=$(nproc) echo "Java test,$?" >> status.txt - name: Rust share build + continue-on-error: true run: echo '[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - name: Rust set env + continue-on-error: true run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - name: Rust test + continue-on-error: true working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 echo "Rust test,$?" >> ../../status.txt - name: Rust example + continue-on-error: true working-directory: examples/rust run: | cargo build --release --features arrow echo "Rust example,$?" >> ../../status.txt - name: Rename status.txt + continue-on-error: true run: | PLATFORM=$(echo ${{ matrix.image }} | tr ':' '-') echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV mv status.txt $PLATFORM.csv - uses: actions/upload-artifact@v4 + continue-on-error: true with: name: ${{env.PLATFORM}} path: ${{env.PLATFORM}}.csv - rhel-fedora-build-test: - continue-on-error: true + rhel-fedora-build-test: strategy: matrix: image: ["rockylinux:8", "rockylinux:9", "fedora:38", "fedora:39"] @@ -409,6 +468,7 @@ jobs: HOME: /root steps: - name: Enable EPEL + continue-on-error: true if: ${{ matrix.image == 'rockylinux:8' || matrix.image == 'rockylinux:9' }} run: | dnf update -y @@ -416,15 +476,18 @@ jobs: dnf update -y - name: Enable SHA-1 on Rocky Linux 9 + continue-on-error: true if: matrix.image == 'rockylinux:9' run: update-crypto-policies --set LEGACY - name: Install packages + continue-on-error: true run: | curl -fsSL https://rpm.nodesource.com/setup_20.x | bash - dnf install -y git cmake ${{ matrix.image == 'rockylinux:8' && 'gcc-toolset-12 python3.11 python3.11-devel' || 'gcc gcc-c++ python3-devel' }} java-17-openjdk-devel nodejs - name: Enable gcc-toolset-12 and python3.11 on Rocky Linux 8 + continue-on-error: true if: matrix.image == 'rockylinux:8' run: | alternatives --set python /usr/bin/python3.11 @@ -435,8 +498,10 @@ jobs: echo $PATH >> $GITHUB_PATH - uses: actions/checkout@v3 + continue-on-error: true - name: Ensure Python dependencies + continue-on-error: true run: | python3 -m venv /opt/venv source /opt/venv/bin/activate @@ -444,77 +509,90 @@ jobs: pip3 install -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.2.1+cpu.html - name: Ensure Node.js dependencies + continue-on-error: true working-directory: tools/nodejs_api run: npm install --include=dev - name: Install Rust + continue-on-error: true run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain echo "$HOME/.cargo/bin" >> $GITHUB_PATH $HOME/.cargo/bin/rustup toolchain install 1.76 - name: Build + continue-on-error: true run: | make release NUM_THREADS=$(nproc) echo "Build,$?" > status.txt - name: Test + continue-on-error: true run: | make test NUM_THREADS=$(nproc) echo "Test,$?" >> status.txt - name: C and C++ examples + continue-on-error: true run: | make example NUM_THREADS=$(nproc) echo "C and C++ examples,$?" >> status.txt - name: Python test + continue-on-error: true run: | source /opt/venv/bin/activate make pytest NUM_THREADS=$(nproc) echo "Python test,$?" >> status.txt - name: Node.js test + continue-on-error: true run: | make nodejstest NUM_THREADS=$(nproc) echo "Node.js test,$?" >> status.txt - name: Java test + continue-on-error: true run: | make javatest NUM_THREADS=$(nproc) echo "Java test,$?" >> status.txt - name: Rust share build + continue-on-error: true run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - name: Rust set env + continue-on-error: true run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - name: Rust test + continue-on-error: true working-directory: tools/rust_api run: | cargo test --release --features arrow -- --test-threads=1 echo "Rust test,$?" >> ../../status.txt - name: Rust example + continue-on-error: true working-directory: examples/rust run: | cargo build --release --features arrow echo "Rust example,$?" >> ../../status.txt - name: Rename status.txt + continue-on-error: true run: | PLATFORM=$(echo ${{ matrix.image }} | tr ':' '-') echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV mv status.txt $PLATFORM.csv - uses: actions/upload-artifact@v4 + continue-on-error: true with: name: ${{env.PLATFORM}} path: ${{env.PLATFORM}}.csv archlinux-build-test: - continue-on-error: true name: archlinux runs-on: ubuntu-latest container: @@ -525,80 +603,97 @@ jobs: JAVA_HOME: /usr/lib/jvm/java-17-openjdk steps: - name: Install packages + continue-on-error: true run: | pacman -Syu --noconfirm pacman -S --needed --noconfirm git base-devel cmake gcc python python-pip npm jdk17-openjdk - uses: actions/checkout@v3 + continue-on-error: true - name: Ensure Python dependencies + continue-on-error: true run: | pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu --break-system-packages pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-2.0.0+cpu.html --break-system-packages - name: Ensure Node.js dependencies + continue-on-error: true working-directory: tools/nodejs_api run: npm install --include=dev - name: Install Rust + continue-on-error: true run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-update-default-toolchain echo "$HOME/.cargo/bin" >> $GITHUB_PATH $HOME/.cargo/bin/rustup toolchain install 1.76 - name: Build + continue-on-error: true run: | make release NUM_THREADS=$(nproc) echo "Build,$?" > status.txt - name: Test + continue-on-error: true run: | make test NUM_THREADS=$(nproc) echo "Test,$?" >> status.txt - name: C and C++ examples + continue-on-error: true run: | make example NUM_THREADS=$(nproc) echo "C and C++ examples,$?" >> status.txt - name: Python test + continue-on-error: true run: | make pytest NUM_THREADS=$(nproc) echo "Python test,$?" >> status.txt - name: Node.js test + continue-on-error: true run: | make nodejstest NUM_THREADS=$(nproc) echo "Node.js test,$?" >> status.txt - name: Java test + continue-on-error: true run: | make javatest NUM_THREADS=$(nproc) echo "Java test,$?" >> status.txt - name: Rust share build + continue-on-error: true run: echo $'[workspace]\nmembers = ["tools/rust_api","examples/rust"]' > Cargo.toml - name: Rust set env + continue-on-error: true run: echo "CARGO_BUILD_JOBS=$(nproc)" >> $GITHUB_ENV - name: Rust test working-directory: tools/rust_api + continue-on-error: true run: | cargo test --release --features arrow -- --test-threads=1 echo "Rust test,$?" >> ../../status.txt - name: Rust example working-directory: examples/rust + continue-on-error: true run: | cargo build --release --features arrow echo "Rust example,$?" >> ../../status.txt - name: Rename status.txt + continue-on-error: true run: | mv status.txt archlinux.csv - uses: actions/upload-artifact@v4 + continue-on-error: true with: name: archlinux path: archlinux.csv From 908aa432e2f826f8bedf03e85850a6548cc38da4 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Tue, 12 Mar 2024 10:59:18 +0800 Subject: [PATCH 28/29] X --- .github/workflows/multiplatform-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/multiplatform-build-test.yml b/.github/workflows/multiplatform-build-test.yml index d9dbe9e249..b730bd021d 100644 --- a/.github/workflows/multiplatform-build-test.yml +++ b/.github/workflows/multiplatform-build-test.yml @@ -703,6 +703,7 @@ jobs: env: DISCORD_CHANNEL_ID: ${{ secrets.DISCORD_CHANNEL_ID }} DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }} + GITHUB_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} needs: [ ubuntu-2004-build-test, @@ -725,7 +726,6 @@ jobs: - name: Create summary run: | - $GITHUB_URL=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} pip3 install pandas tabulate discord.py python3 scripts/multiplatform-test-helper/collect-results.py results python3 scripts/multiplatform-test-helper/notify-discord.py results.json From b74bd0a84d0baa5db16578ba4f66ec89f42ca410 Mon Sep 17 00:00:00 2001 From: Chang Liu Date: Tue, 12 Mar 2024 15:17:41 +0800 Subject: [PATCH 29/29] X --- .../multiplatform-test-helper/notify-discord.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/multiplatform-test-helper/notify-discord.py b/scripts/multiplatform-test-helper/notify-discord.py index 63befce987..a5046b044e 100644 --- a/scripts/multiplatform-test-helper/notify-discord.py +++ b/scripts/multiplatform-test-helper/notify-discord.py @@ -7,7 +7,7 @@ CHANNEL_ID = os.getenv("DISCORD_CHANNEL_ID") GITHUB_URL = os.getenv("GITHUB_URL") -message = "" +messages = [] if __name__ == "__main__": @@ -29,16 +29,27 @@ @client.event async def on_ready(): channel = client.get_channel(int(CHANNEL_ID)) - await channel.send(message) + for message in messages: + try: + await channel.send(message) + except Exception as e: + print(f"Error: {e}") + sys.exit(1) await client.close() + message = "" message += "## Multiplatform test result:\n" with open(sys.argv[1], "r") as f: result = json.load(f) for platform in result: + if len(message) >= 1500: + messages.append(message) + message = "" message += f"- **{platform}**:\n" for r in result[platform]: message += f" - {r['stage']}: {r['status']}\n" if GITHUB_URL: message += f" [Github]({GITHUB_URL})" + if message: + messages.append(message) client.run(TOKEN)