Use ccache for PyTorch builds with CUDA on GitHub Actions

.github/actions/deploy-centos/action.yml

@@ -6,6 +6,10 @@ runs:
       shell: bash
       run: |
         cd /root
+        mkdir -p .ccache
+        echo "max_size = 2.0G"                                                                        > .ccache/ccache.conf
+        echo "hash_dir = false"                                                                      >> .ccache/ccache.conf
+        echo "sloppiness = file_macro,include_file_ctime,include_file_mtime,pch_defines,time_macros" >> .ccache/ccache.conf
 
         SCL_ENABLE="devtoolset-9"
         CENTOS_VERSION=$(rpm --eval '%{centos_ver}')
@@ -90,6 +94,10 @@ runs:
           cp /usr/local/cuda/lib64/stubs/libcuda.so /usr/lib64/libcuda.so.1
           cp /usr/local/cuda/lib64/stubs/libnvidia-ml.so /usr/lib64/libnvidia-ml.so
           cp /usr/local/cuda/lib64/stubs/libnvidia-ml.so /usr/lib64/libnvidia-ml.so.1
+          sed -i /block_merge_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh
+          sed -i /device_merge_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh
+          sed -i /device_segmented_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh
+          sed -i /warp_merge_sort.cuh/d /usr/local/cuda/include/cub/cub.cuh
 
           # Remove downloaded archives and unused libraries to avoid running out of disk space
           rm -f *.rpm *.tgz *.txz *.tar.*
@@ -171,6 +179,7 @@ runs:
         mvn -version
         gpg --version
         python3 --version
+        ccache -s --version
         df -h
 
         export MAKEJ=$(getconf _NPROCESSORS_ONLN)

.github/actions/deploy-macosx/action.yml

@@ -6,6 +6,10 @@ runs:
       shell: bash
       run: |
         cd $HOME
+        mkdir -p .ccache
+        echo "max_size = 2.0G"                                                                        > .ccache/ccache.conf
+        echo "hash_dir = false"                                                                      >> .ccache/ccache.conf
+        echo "sloppiness = file_macro,include_file_ctime,include_file_mtime,pch_defines,time_macros" >> .ccache/ccache.conf
 
         export MACOSX_DEPLOYMENT_TARGET=10.15
         echo "MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET" >> $GITHUB_ENV
@@ -17,14 +21,9 @@ runs:
 
         brew install boost ccache gcc swig autoconf-archive automake cmake libomp libtool libusb ant maven nasm xz pkg-config sdl2 gpg1 bison flex perl ragel binutils gradle gmp isl libmpc mpfr geoip pcre ssdeep yajl
 
+        brew uninstall --force gcc gcc@7 gcc@8 gcc@9 gcc@11 gcc@12
         if [[ -n ${CI_DEPLOY_NEED_GCC:-} ]]; then
           echo Installing an older less buggy version of GCC
-          brew uninstall --force gcc
-          brew uninstall --force gcc@7
-          brew uninstall --force gcc@8
-          brew uninstall --force gcc@9
-          brew uninstall --force gcc@11
-          brew uninstall --force gcc@12
           brew install gcc@10
           brew link --overwrite gcc@10
 
@@ -43,6 +42,8 @@ runs:
           sudo install_name_tool -change /usr/local/lib/gcc/10/libgcc_s.1.dylib @rpath/libgcc_s.1.dylib /usr/local/lib/gcc/10/libgfortran.5.dylib
           sudo install_name_tool -change /usr/local/lib/gcc/10/libgcc_s.1.dylib @rpath/libgcc_s.1.dylib /usr/local/lib/gcc/10/libquadmath.0.dylib
           otool -L /usr/local/lib/gcc/10/*.dylib
+        else
+          brew uninstall --force gcc@10
         fi
 
         sudo install_name_tool -add_rpath @loader_path/. -id @rpath/libomp.dylib /usr/local/opt/libomp/lib/libomp.dylib
@@ -85,12 +86,13 @@ runs:
       run: |
         git --version
         clang --version
-        /usr/local/bin/gcc-* --version
+        /usr/local/bin/gcc-* --version || true
         cmake --version
         gradle --version
         mvn -version
         gpg --version
         python3 --version
+        ccache -s --version
         df -h
 
         export MAKEJ=$(getconf _NPROCESSORS_ONLN)

.github/actions/deploy-ubuntu/action.yml

@@ -6,6 +6,10 @@ runs:
       shell: bash
       run: |
         cd /root
+        mkdir -p .ccache
+        echo "max_size = 2.0G"                                                                        > .ccache/ccache.conf
+        echo "hash_dir = false"                                                                      >> .ccache/ccache.conf
+        echo "sloppiness = file_macro,include_file_ctime,include_file_mtime,pch_defines,time_macros" >> .ccache/ccache.conf
 
         if [[ "$CI_DEPLOY_PLATFORM" == "linux-armhf" ]]; then
           export ARCH=armhf
@@ -175,6 +179,7 @@ runs:
         mvn -version
         gpg --version
         python3 --version
+        ccache -s --version
         df -h
 
         export MAKEJ=$(getconf _NPROCESSORS_ONLN)

.github/actions/deploy-windows/action.yml

@@ -18,7 +18,7 @@ runs:
         sdksetup.exe /features OptionId.WindowsDesktopSoftwareDevelopmentKit OptionId.NetFxSoftwareDevelopmentKit /quiet
 
         echo Removing broken stuff from WSL and MSYS2
-        rm "C:/WINDOWS/system32/bash.EXE" "C:/msys64/usr/bin/link.exe" "C:/msys64/usr/bin/python.exe" "C:/msys64/usr/bin/python3.exe"
+        rm "C:/WINDOWS/system32/bash.EXE" "C:/msys64/usr/bin/link.exe" "C:/msys64/usr/bin/timeout.exe" "C:/msys64/usr/bin/python.exe" "C:/msys64/usr/bin/python3.exe"
         rm "C:/ProgramData/chocolatey/bin/gfortran.exe" "C:/msys64/mingw32/bin/gfortran.exe" "C:/msys64/mingw32/bin/python.exe" "C:/msys64/mingw32/bin/python3.exe"
         rm "C:/Strawberry/c/bin/gfortran.exe" "C:/msys64/mingw64/bin/gfortran.exe" "C:/msys64/mingw64/bin/python.exe" "C:/msys64/mingw64/bin/python3.exe"
         rm "C:/msys64/mingw32/bin/clang-cl.exe" "C:/msys64/mingw64/bin/clang-cl.exe" "C:/msys64/mingw32/bin/cmake.exe" "C:/msys64/mingw64/bin/cmake.exe"
@@ -34,6 +34,16 @@ runs:
           dir /s /a "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\14.*"
         )
 
+        if not "%CI_DEPLOY_NEED_CACHE%"=="" (
+          echo Installing ccache
+          curl -LO https://github.com/ccache/ccache/releases/download/v4.6/ccache-4.6-windows-64.zip
+          unzip -j ccache-4.6-windows-64.zip -d C:/msys64/mingw64/bin/
+          mkdir ccache
+          echo max_size = 3.0G                                                                        > ccache\ccache.conf
+          echo hash_dir = false                                                                      >> ccache\ccache.conf
+          echo sloppiness = file_macro,include_file_ctime,include_file_mtime,pch_defines,time_macros >> ccache\ccache.conf
+        )
+
         if not "%CI_DEPLOY_NEED_GCC%"=="" (
           echo Installing an older less buggy version of GCC
           curl -LO http://repo.msys2.org/mingw/i686/mingw-w64-i686-gdb-10.1-1-any.pkg.tar.zst
@@ -86,7 +96,10 @@ runs:
           echo Applying hotfix to Visual Studio 2019 for CUDA
           curl -LO https://github.com/raw/microsoft/STL/main/stl/inc/cmath
           bash -c "find 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/' -name cmath -exec cp -v cmath {} \;"
+          bash -c "sed -i '/block_merge_sort.cuh/d' 'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.6/include/cub/cub.cuh'"
+          bash -c "sed -i '/device_merge_sort.cuh/d' 'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.6/include/cub/cub.cuh'"
           bash -c "sed -i '/device_segmented_sort.cuh/d' 'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.6/include/cub/cub.cuh'"
+          bash -c "sed -i '/warp_merge_sort.cuh/d' 'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.6/include/cub/cub.cuh'"
         )
 
         if "%CI_DEPLOY_MODULE%"=="nvcodec" (
@@ -188,6 +201,7 @@ runs:
           set "PATH=%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v11.6\bin;%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v11.6\libnvvp;%PATH%"
           echo CUDA Version 11.6.1>"%ProgramFiles%\NVIDIA GPU Computing Toolkit\CUDA\v11.6\version.txt"
         )
+        set "CCACHE_DIR=%USERPROFILE%\ccache"
         set "PATH=C:\msys64\%MSYSTEM%\bin;C:\msys64\usr\bin;%PATH%"
 
         where bash
@@ -201,6 +215,7 @@ runs:
         where python
         where python3
         where clang-cl
+        where ccache
 
         bash --version
         git --version
@@ -212,6 +227,7 @@ runs:
         gpg --version
         python --version
         clang-cl --version
+        ccache --version -sv
         df -h
         wmic pagefile list /format:list
 
@@ -266,6 +282,7 @@ runs:
           exit /b !STATUS!
         )
         :done3
+        ccache --version -sv
         df -h
         wmic pagefile list /format:list
 

.github/actions/redeploy/action.yml

@@ -66,7 +66,7 @@ runs:
 
         rm -f dependencies.txt
         for POM in $(find $CI_DEPLOY_MODULE/platform/ -iname pom.xml); do
-            if [[ $POM =~ pytorch/platform/gpu ]] || [[ $POM =~ tensorflow/platform/python ]]; then
+            if [[ $POM =~ tensorflow/platform/python ]]; then
                 continue # Skip over builds taking more than 6 hours on GitHub Actions
             fi
             for i in {1..5}; do

.github/workflows/pytorch.yml

@@ -8,6 +8,7 @@ on:
       - pytorch/**
   workflow_dispatch:
 env:
+  CI_DEPLOY_NEED_CACHE: 1
   CI_DEPLOY_NEED_CUDA: 1
   CI_DEPLOY_MODULE: ${{ github.workflow }}
   CI_DEPLOY_PLATFORM: ${{ github.job }}
@@ -19,11 +20,17 @@ jobs:
   linux-x86_64:
     runs-on: ubuntu-18.04
     container: centos:7
-#    strategy:
-#      matrix:
-#        ext: ["", -gpu]
+    strategy:
+      matrix:
+        ext: ["", -gpu]
     steps:
+      - uses: bytedeco/cache@always
+        with:
+          key: ${{ github.workflow }}-${{ github.job }}${{ matrix.ext }}@${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: ${{ github.workflow }}-${{ github.job }}${{ matrix.ext }}@
+          path: /root/.ccache
       - uses: bytedeco/javacpp-presets/.github/actions/deploy-centos@actions
+        timeout-minutes: 350
   macosx-x86_64:
     runs-on: macos-10.15
 #    strategy:
@@ -33,9 +40,9 @@ jobs:
       - uses: bytedeco/javacpp-presets/.github/actions/deploy-macosx@actions
   windows-x86_64:
     runs-on: windows-2019
-#    strategy:
-#      matrix:
-#        ext: ["", -gpu]
+    strategy:
+      matrix:
+        ext: ["", -gpu]
     env:
         BUILD_DIR: C:\build
         CI_DEPLOY_OPTIONS: -Djavacpp.parser.skip -Dmaven.javadoc.skip
@@ -45,7 +52,13 @@ jobs:
           minimum-size: 16GB
           maximum-size: 32GB
           disk-root: "C:"
+      - uses: bytedeco/cache@always
+        with:
+          key: ${{ github.workflow }}-${{ github.job }}${{ matrix.ext }}@${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: ${{ github.workflow }}-${{ github.job }}${{ matrix.ext }}@
+          path: ~/ccache/?
       - uses: bytedeco/javacpp-presets/.github/actions/deploy-windows@actions
+        timeout-minutes: 350
   redeploy:
     needs: [linux-x86_64, macosx-x86_64, windows-x86_64]
     runs-on: ubuntu-18.04

pytorch/cppbuild.sh

@@ -101,8 +101,15 @@ case $PLATFORM in
         export CXX="clang++"
         ;;
     windows-x86_64)
-        export CC="cl.exe"
-        export CXX="cl.exe"
+        if which ccache.exe; then
+            export CC="ccache.exe cl.exe"
+            export CXX="ccache.exe cl.exe"
+            export CUDAHOSTCC="cl.exe"
+            export CUDAHOSTCXX="cl.exe"
+        else
+            export CC="cl.exe"
+            export CXX="cl.exe"
+        fi
         export CFLAGS="-I$CPYTHON_PATH/include/ -I$PYTHON_LIB_PATH/include/python/"
         ;;
     *)
@@ -111,9 +118,11 @@ case $PLATFORM in
         ;;
 esac
 
+# work around issues with the build system
 sedinplace '/Werror/d' CMakeLists.txt
 sedinplace 's/build_python=True/build_python=False/g' setup.py
 sedinplace 's/    build_deps()/    build_deps(); sys.exit()/g' setup.py
+sedinplace 's/AND NOT DEFINED ENV{CUDAHOSTCXX}//g' cmake/public/cuda.cmake
 
 # work around some compiler bugs
 sedinplace 's/!defined(__INTEL_COMPILER))/!defined(__INTEL_COMPILER) \&\& (__GNUC__ < 11))/g' third_party/XNNPACK/src/xnnpack/intrinsics-polyfill.h