Optimize CI workflows (#268)

* Reduce size of unit tests artifacts * Optimize Dockerfile build * Update build_dockerfile.sh * Add reusable workflow to build deps with Conan * Update ubuntu-ci.yml to use build-conan-deps.yml * Bugfix * Update macos-ci.yml to use build-conan-deps.yml * Update codecov.yml to use build-conan-deps.yml * Update run-clang-tidy.yml to use build-conan-deps.yml * Update cppstd.yml to use build-conan-deps.yml * Update fuzzy-testing.yml to use build-conan-deps.yml * Update packaging.yml to use build-conan-deps.yml * Update windows-ci.yml to use build-conan-deps.yml * Trigger CI * Bugfix * Bugfix * Make GHA caching of Ccache folder more resilient * Trigger CI * Bugfix * Explicitly disable Ccache inside the Dockerfile * Bugfix * Bugfix * Do not bother exporting Ubuntu Ccache cache when running the full-ci * Optimize build-dockerfile.yml
paulsengroup · Sep 30, 2024 · 08f37d5 · 08f37d5
1 parent 8c9584a
commit 08f37d5
Show file tree

Hide file tree

Showing 13 changed files with 538 additions and 676 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -13,6 +13,6 @@
 # Allowed files
 !CMakeLists.txt
 !LICENSE
-!conanfile.*
+!conanfile.Dockerfile.py
 !test/CMakeLists.txt
 !test/data/hictk_test_data.tar.zst
diff --git a/.github/workflows/build-dockerfile.yml b/.github/workflows/build-dockerfile.yml
@@ -16,7 +16,7 @@ on:
       - ".dockerignore"
       - "CMakeLists.txt"
       - "Dockerfile"
-      - "conanfile.py"
+      - "conanfile.Dockerfile.py"
     tags:
       - "v*.*.*"
 
@@ -31,7 +31,7 @@ on:
       - ".dockerignore"
       - "CMakeLists.txt"
       - "Dockerfile"
-      - "conanfile.py"
+      - "conanfile.Dockerfile.py"
 
 # https://stackoverflow.com/a/72408109
 concurrency:
@@ -97,6 +97,8 @@ jobs:
             VERSION="$GIT_TAG"
           fi
 
+          CACHE_REGISTRY='ghcr.io/${{ github.repository }}:buildcache'
+
           echo "C_COMPILER=$C_COMPILER" | tee -a "$GITHUB_OUTPUT"
           echo "CXX_COMPILER=$CXX_COMPILER" | tee -a "$GITHUB_OUTPUT"
           echo "FINAL_BASE_IMAGE=$FINAL_BASE_IMAGE" | tee -a "$GITHUB_OUTPUT"
@@ -109,6 +111,8 @@ jobs:
           echo "CREATION_DATE=$CREATION_DATE" | tee -a "$GITHUB_OUTPUT"
           echo "GIT_TAG=$GIT_TAG" | tee -a "$GITHUB_OUTPUT"
           echo "VERSION=$VERSION" | tee -a "$GITHUB_OUTPUT"
+          echo "CACHE_REGISTRY_X86=$CACHE_REGISTRY-x86" | tee -a "$GITHUB_OUTPUT"
+          echo "CACHE_REGISTRY_ARM64=$CACHE_REGISTRY-arm64" | tee -a "$GITHUB_OUTPUT"
 
       - name: Docker meta
         id: meta
@@ -154,9 +158,9 @@ jobs:
           context: ${{ github.workspace }}
           load: true
           push: false
-          cache-from: type=gha,scope=build-dockerfile
-          cache-to: type=gha,mode=max,scope=build-dockerfile
-          tags: hictk:test
+          cache-from: type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_X86 }}
+          cache-to: type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_X86 }},mode=max,compression=zstd
+          tags: hictk:x86
           platforms: linux/amd64
           build-args: |
             C_COMPILER=${{ steps.build-args.outputs.C_COMPILER }}
@@ -174,19 +178,17 @@ jobs:
             VERSION=${{ steps.build-args.outputs.VERSION }}
 
       - name: Test Docker image (x86)
-        run: |
-          utils/devel/test_docker_image.sh hictk:test
+        run: utils/devel/test_docker_image.sh hictk:x86
 
       - name: Build Docker image (arm64)
         if: github.event_name != 'pull_request'
         uses: docker/build-push-action@v6
         with:
           context: ${{ github.workspace }}
-          load: true
           push: false
-          cache-from: type=gha,scope=build-dockerfile
-          cache-to: type=gha,mode=max,scope=build-dockerfile
-          tags: hictk:test
+          cache-from: type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_ARM64 }}
+          cache-to: type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_ARM64 }},mode=max,compression=zstd
+          tags: hictk:arm64
           platforms: linux/arm64
           build-args: |
             C_COMPILER=${{ steps.build-args.outputs.C_COMPILER }}
@@ -209,8 +211,9 @@ jobs:
         with:
           context: ${{ github.workspace }}
           push: true
-          cache-from: type=gha,scope=build-dockerfile
-          cache-to: type=gha,mode=max,scope=build-dockerfile
+          cache-from: |
+            type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_X86 }}
+            type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_ARM64 }}
           tags: ${{ steps.meta.outputs.tags }}
           platforms: linux/amd64,linux/arm64
           build-args: |
@@ -237,6 +240,5 @@ jobs:
 
     steps:
       - name: Collect job results
-        if: |
-          needs.build-dockerfile.result != 'success'
+        if: needs.build-dockerfile.result != 'success'
         run: exit 1
diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml
@@ -8,21 +8,25 @@ on:
     branches: [main]
     paths:
       - ".github/codecov.yml"
+      - ".github/workflows/build-conan-deps.yml"
+      - ".github/workflows/cache-test-dataset.yml"
       - ".github/workflows/codecov.yml"
+      - ".github/workflows/evict-gha-cache.yml"
       - "cmake/**"
       - "examples/**"
       - "src/**"
       - "test/integration/**"
       - "test/units/**"
       - "CMakeLists.txt"
       - "conanfile.py"
-    tags:
-      - "v*.*.*"
 
   pull_request:
     paths:
       - ".github/codecov.yml"
+      - ".github/workflows/build-conan-deps.yml"
+      - ".github/workflows/cache-test-dataset.yml"
       - ".github/workflows/codecov.yml"
+      - ".github/workflows/evict-gha-cache.yml"
       - "cmake/**"
       - "examples/**"
       - "src/**"
@@ -42,11 +46,18 @@ defaults:
 
 jobs:
   cache-test-dataset:
-    uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main
     name: Cache test dataset
+    uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main
+
+  build-conan-deps:
+    name: Build Conan deps
+    uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main
+    with:
+      os: ubuntu-20.04
 
   build-project:
     name: Build project
+    needs: build-conan-deps
     runs-on: ubuntu-latest
     container:
       image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-gcc-14
@@ -79,62 +90,33 @@ jobs:
 
           ccache_key_prefix="codecov-ccache-$conanfile_hash"
 
-          echo "conan-key=codecov-$conanfile_hash" | tee -a "$GITHUB_OUTPUT"
-          echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a "$GITHUB_OUTPUT"
-          echo "ccache-restore-key=$ccache_key_prefix" | tee -a "$GITHUB_OUTPUT"
+          echo "ccache-key=${ccache_key_prefix}-$GITHUB_REF-${current_date}" | tee -a "$GITHUB_OUTPUT"
+          echo "ccache-restore-key-1=$ccache_key_prefix-$GITHUB_REF" | tee -a "$GITHUB_OUTPUT"
+          echo "ccache-restore-key-2=$ccache_key_prefix" | tee -a "$GITHUB_OUTPUT"
 
       - name: Restore Conan cache
-        id: cache-conan
         uses: actions/cache/restore@v4
         with:
-          key: conan-${{ steps.cache-key.outputs.conan-key }}
+          key: ${{ needs.build-conan-deps.outputs.conan-key }}
           path: ${{ env.CONAN_HOME }}/p
+          fail-on-cache-miss: true
 
-      - name: Clean Conan cache (pre-build)
-        if: steps.cache-conan.outputs.cache-hit != 'true'
-        run: |
-          conan cache clean "*" --build
-          conan cache clean "*" --download
-          conan cache clean "*" --source
-          conan remove --confirm "*"
-
-      - name: Copy Conan settings
-        run: |
-          if [ ! -f "$CONAN_HOME/settings.yml" ]; then
-            cp "/root/.conan2/settings.yml" "$CONAN_HOME"
-          fi
+      - name: Restore CMake configs
+        uses: actions/cache/restore@v4
+        with:
+          key: ${{ needs.build-conan-deps.outputs.cmake-prefix-debug-key }}
+          path: /tmp/cmake-prefix-dbg.tar
+          fail-on-cache-miss: true
 
-      - name: Install build dependencies
+      - name: Extract CMake configs
         run: |
-          conan install .                        \
-             --build=missing                     \
-             -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \
-             -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \
-             -s build_type=Debug                 \
-             -s compiler.libcxx=libstdc++11      \
-             -s compiler.cppstd=17               \
-             --output-folder=build
-
-      - name: Clean Conan cache (post-build)
-        if: steps.cache-conan.outputs.cache-hit != 'true'
-        run: |
-          conan cache clean "*" --build
-          conan cache clean "*" --download
-          conan cache clean "*" --source
-
-      - name: Save Conan cache
-        uses: actions/cache/save@v4
-        if: steps.cache-conan.outputs.cache-hit != 'true'
-        with:
-          key: conan-${{ steps.cache-key.outputs.conan-key }}
-          path: ${{ env.CONAN_HOME }}/p
-        env:
-          ZSTD_CLEVEL: 19
+          mkdir conan-env
+          tar -xf /tmp/cmake-prefix-dbg.tar -C conan-env/ --strip-components=1
 
       - name: Configure project
         run: |
           cmake -DCMAKE_BUILD_TYPE=Debug                      \
-                -DCMAKE_PREFIX_PATH="$PWD/build"              \
+                -DCMAKE_PREFIX_PATH="$PWD/conan-env"          \
                 -DENABLE_DEVELOPER_MODE=ON                    \
                 -DOPT_ENABLE_COVERAGE=ON                      \
                 -DOPT_ENABLE_SANITIZER_ADDRESS=OFF            \
@@ -151,7 +133,8 @@ jobs:
         id: cache-ccache
         uses: actions/cache/restore@v4
         with:
-          key: ${{ steps.cache-key.outputs.ccache-restore-key }}
+          key: ${{ steps.cache-key.outputs.ccache-restore-key-1 }}
+          restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key-2 }}
           path: ${{ env.CCACHE_DIR }}
 
       - name: Reset Ccache stats
@@ -161,7 +144,7 @@ jobs:
         run: cmake --build build -j $(nproc)
 
       - name: Package build folder
-        run: tar -cf - build/ | zstd -T0 -13 -o build.tar.zst
+        run: tar --exclude='*.o' -cf - build/ | zstd -T0 -13 -o build.tar.zst
 
       - name: Upload build folder
         uses: actions/upload-artifact@v4
@@ -468,15 +451,13 @@ jobs:
     if: ${{ always() }}
     runs-on: ubuntu-latest
     needs:
-      - cache-test-dataset
       - build-project
       - run-unit-tests
       - run-integration-tests
 
     steps:
       - name: Collect job results
         if: |
-          needs.cache-test-dataset.result != 'success' ||
           needs.build-project.result != 'success'      ||
           needs.run-unit-tests.result != 'success'     ||
           needs.run-integration-tests.result != 'success'