diff --git a/.dockerignore b/.dockerignore index 1b91a24d..4d49c8ab 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,6 +13,6 @@ # Allowed files !CMakeLists.txt !LICENSE -!conanfile.* +!conanfile.Dockerfile.py !test/CMakeLists.txt !test/data/hictk_test_data.tar.zst diff --git a/.github/workflows/build-dockerfile.yml b/.github/workflows/build-dockerfile.yml index 382c4de8..a9ff171f 100644 --- a/.github/workflows/build-dockerfile.yml +++ b/.github/workflows/build-dockerfile.yml @@ -16,7 +16,7 @@ on: - ".dockerignore" - "CMakeLists.txt" - "Dockerfile" - - "conanfile.py" + - "conanfile.Dockerfile.py" tags: - "v*.*.*" @@ -31,7 +31,7 @@ on: - ".dockerignore" - "CMakeLists.txt" - "Dockerfile" - - "conanfile.py" + - "conanfile.Dockerfile.py" # https://stackoverflow.com/a/72408109 concurrency: @@ -97,6 +97,8 @@ jobs: VERSION="$GIT_TAG" fi + CACHE_REGISTRY='ghcr.io/${{ github.repository }}:buildcache' + echo "C_COMPILER=$C_COMPILER" | tee -a "$GITHUB_OUTPUT" echo "CXX_COMPILER=$CXX_COMPILER" | tee -a "$GITHUB_OUTPUT" echo "FINAL_BASE_IMAGE=$FINAL_BASE_IMAGE" | tee -a "$GITHUB_OUTPUT" @@ -109,6 +111,8 @@ jobs: echo "CREATION_DATE=$CREATION_DATE" | tee -a "$GITHUB_OUTPUT" echo "GIT_TAG=$GIT_TAG" | tee -a "$GITHUB_OUTPUT" echo "VERSION=$VERSION" | tee -a "$GITHUB_OUTPUT" + echo "CACHE_REGISTRY_X86=$CACHE_REGISTRY-x86" | tee -a "$GITHUB_OUTPUT" + echo "CACHE_REGISTRY_ARM64=$CACHE_REGISTRY-arm64" | tee -a "$GITHUB_OUTPUT" - name: Docker meta id: meta @@ -154,9 +158,9 @@ jobs: context: ${{ github.workspace }} load: true push: false - cache-from: type=gha,scope=build-dockerfile - cache-to: type=gha,mode=max,scope=build-dockerfile - tags: hictk:test + cache-from: type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_X86 }} + cache-to: type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_X86 }},mode=max,compression=zstd + tags: hictk:x86 platforms: linux/amd64 build-args: | C_COMPILER=${{ steps.build-args.outputs.C_COMPILER }} @@ -174,19 +178,17 @@ jobs: VERSION=${{ steps.build-args.outputs.VERSION }} - name: Test Docker image (x86) - run: | - utils/devel/test_docker_image.sh hictk:test + run: utils/devel/test_docker_image.sh hictk:x86 - name: Build Docker image (arm64) if: github.event_name != 'pull_request' uses: docker/build-push-action@v6 with: context: ${{ github.workspace }} - load: true push: false - cache-from: type=gha,scope=build-dockerfile - cache-to: type=gha,mode=max,scope=build-dockerfile - tags: hictk:test + cache-from: type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_ARM64 }} + cache-to: type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_ARM64 }},mode=max,compression=zstd + tags: hictk:arm64 platforms: linux/arm64 build-args: | C_COMPILER=${{ steps.build-args.outputs.C_COMPILER }} @@ -209,8 +211,9 @@ jobs: with: context: ${{ github.workspace }} push: true - cache-from: type=gha,scope=build-dockerfile - cache-to: type=gha,mode=max,scope=build-dockerfile + cache-from: | + type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_X86 }} + type=registry,ref=${{ steps.build-args.outputs.CACHE_REGISTRY_ARM64 }} tags: ${{ steps.meta.outputs.tags }} platforms: linux/amd64,linux/arm64 build-args: | @@ -237,6 +240,5 @@ jobs: steps: - name: Collect job results - if: | - needs.build-dockerfile.result != 'success' + if: needs.build-dockerfile.result != 'success' run: exit 1 diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index a2aeed23..5a6a19d2 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -8,7 +8,10 @@ on: branches: [main] paths: - ".github/codecov.yml" + - ".github/workflows/build-conan-deps.yml" + - ".github/workflows/cache-test-dataset.yml" - ".github/workflows/codecov.yml" + - ".github/workflows/evict-gha-cache.yml" - "cmake/**" - "examples/**" - "src/**" @@ -16,13 +19,14 @@ on: - "test/units/**" - "CMakeLists.txt" - "conanfile.py" - tags: - - "v*.*.*" pull_request: paths: - ".github/codecov.yml" + - ".github/workflows/build-conan-deps.yml" + - ".github/workflows/cache-test-dataset.yml" - ".github/workflows/codecov.yml" + - ".github/workflows/evict-gha-cache.yml" - "cmake/**" - "examples/**" - "src/**" @@ -42,11 +46,18 @@ defaults: jobs: cache-test-dataset: - uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main name: Cache test dataset + uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main + + build-conan-deps: + name: Build Conan deps + uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main + with: + os: ubuntu-20.04 build-project: name: Build project + needs: build-conan-deps runs-on: ubuntu-latest container: image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-gcc-14 @@ -79,62 +90,33 @@ jobs: ccache_key_prefix="codecov-ccache-$conanfile_hash" - echo "conan-key=codecov-$conanfile_hash" | tee -a "$GITHUB_OUTPUT" - echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a "$GITHUB_OUTPUT" - echo "ccache-restore-key=$ccache_key_prefix" | tee -a "$GITHUB_OUTPUT" + echo "ccache-key=${ccache_key_prefix}-$GITHUB_REF-${current_date}" | tee -a "$GITHUB_OUTPUT" + echo "ccache-restore-key-1=$ccache_key_prefix-$GITHUB_REF" | tee -a "$GITHUB_OUTPUT" + echo "ccache-restore-key-2=$ccache_key_prefix" | tee -a "$GITHUB_OUTPUT" - name: Restore Conan cache - id: cache-conan uses: actions/cache/restore@v4 with: - key: conan-${{ steps.cache-key.outputs.conan-key }} + key: ${{ needs.build-conan-deps.outputs.conan-key }} path: ${{ env.CONAN_HOME }}/p + fail-on-cache-miss: true - - name: Clean Conan cache (pre-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - conan remove --confirm "*" - - - name: Copy Conan settings - run: | - if [ ! -f "$CONAN_HOME/settings.yml" ]; then - cp "/root/.conan2/settings.yml" "$CONAN_HOME" - fi + - name: Restore CMake configs + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps.outputs.cmake-prefix-debug-key }} + path: /tmp/cmake-prefix-dbg.tar + fail-on-cache-miss: true - - name: Install build dependencies + - name: Extract CMake configs run: | - conan install . \ - --build=missing \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ - -s build_type=Debug \ - -s compiler.libcxx=libstdc++11 \ - -s compiler.cppstd=17 \ - --output-folder=build - - - name: Clean Conan cache (post-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - - - name: Save Conan cache - uses: actions/cache/save@v4 - if: steps.cache-conan.outputs.cache-hit != 'true' - with: - key: conan-${{ steps.cache-key.outputs.conan-key }} - path: ${{ env.CONAN_HOME }}/p - env: - ZSTD_CLEVEL: 19 + mkdir conan-env + tar -xf /tmp/cmake-prefix-dbg.tar -C conan-env/ --strip-components=1 - name: Configure project run: | cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_PREFIX_PATH="$PWD/build" \ + -DCMAKE_PREFIX_PATH="$PWD/conan-env" \ -DENABLE_DEVELOPER_MODE=ON \ -DOPT_ENABLE_COVERAGE=ON \ -DOPT_ENABLE_SANITIZER_ADDRESS=OFF \ @@ -151,7 +133,8 @@ jobs: id: cache-ccache uses: actions/cache/restore@v4 with: - key: ${{ steps.cache-key.outputs.ccache-restore-key }} + key: ${{ steps.cache-key.outputs.ccache-restore-key-1 }} + restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key-2 }} path: ${{ env.CCACHE_DIR }} - name: Reset Ccache stats @@ -161,7 +144,7 @@ jobs: run: cmake --build build -j $(nproc) - name: Package build folder - run: tar -cf - build/ | zstd -T0 -13 -o build.tar.zst + run: tar --exclude='*.o' -cf - build/ | zstd -T0 -13 -o build.tar.zst - name: Upload build folder uses: actions/upload-artifact@v4 @@ -468,7 +451,6 @@ jobs: if: ${{ always() }} runs-on: ubuntu-latest needs: - - cache-test-dataset - build-project - run-unit-tests - run-integration-tests @@ -476,7 +458,6 @@ jobs: steps: - name: Collect job results if: | - needs.cache-test-dataset.result != 'success' || needs.build-project.result != 'success' || needs.run-unit-tests.result != 'success' || needs.run-integration-tests.result != 'success' diff --git a/.github/workflows/cppstd.yml b/.github/workflows/cppstd.yml index afe8e6c4..7debf952 100644 --- a/.github/workflows/cppstd.yml +++ b/.github/workflows/cppstd.yml @@ -7,7 +7,9 @@ on: push: branches: [main] paths: + - ".github/workflows/build-conan-deps.yml" - ".github/workflows/cppstd.yml" + - ".github/workflows/evict-gha-cache.yml" - "cmake/**" - "examples/**" - "src/**" @@ -15,12 +17,12 @@ on: - "test/units/**" - "CMakeLists.txt" - "conanfile.py" - tags: - - "v*.*.*" pull_request: paths: + - ".github/workflows/build-conan-deps.yml" - ".github/workflows/cppstd.yml" + - ".github/workflows/evict-gha-cache.yml" - "cmake/**" - "examples/**" - "src/**" @@ -39,96 +41,26 @@ defaults: shell: bash jobs: - build-deps: - runs-on: ubuntu-latest - name: Build dependencies - container: - image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-clang-19 - options: "--user=root" - - outputs: - cache-key: ${{ steps.generate-cache-key.outputs.conan-key }} - - env: - CONAN_HOME: "/opt/conan/" - - steps: - - uses: actions/checkout@v4 - - - name: Fix permissions - run: | - chown -R $(id -u):$(id -g) $PWD - - - name: Generate cache key - id: generate-cache-key - run: | - hash="${{ hashFiles('conanfile.py') }}" - - echo "conan-key=conan-cppstd-$hash" | tee -a "$GITHUB_OUTPUT" - - - name: Restore Conan cache - id: cache-conan - uses: actions/cache/restore@v4 - with: - key: ${{ steps.generate-cache-key.outputs.conan-key }} - path: "${{ env.CONAN_HOME }}/p" - lookup-only: true - - - name: Clean Conan cache (pre-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - conan remove --confirm "*" - - - name: Copy Conan settings - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - if [ ! -f "$CONAN_HOME/settings.yml" ]; then - cp "/root/.conan2/settings.yml" "$CONAN_HOME" - fi - - - name: Install build dependencies (c++20) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan install . \ - --build=missing \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ - -s build_type=Debug \ - -s compiler.libcxx=libstdc++11 \ - -s compiler.cppstd=20 - - - name: Install build dependencies (c++23) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan install . \ - --build=missing \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ - -s build_type=Debug \ - -s compiler.libcxx=libstdc++11 \ - -s compiler.cppstd=23 - - - name: Clean Conan cache (post-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source + build-conan-deps-cpp20: + name: Build Conan deps + uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main + with: + os: ubuntu-24.04 + cppstd: 20 - - name: Save Conan cache - uses: actions/cache/save@v4 - if: steps.cache-conan.outputs.cache-hit != 'true' - with: - key: ${{ steps.generate-cache-key.outputs.conan-key }} - path: "${{ env.CONAN_HOME }}/p" + build-conan-deps-cpp23: + name: Build Conan deps + uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main + with: + os: ubuntu-24.04 + cppstd: 23 build-project: runs-on: ubuntu-latest name: Build project - needs: [build-deps] + needs: + - build-conan-deps-cpp20 + - build-conan-deps-cpp23 container: image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-clang-19 options: "--user=root" @@ -167,33 +99,46 @@ jobs: ccache_key_prefix="ccache-cppstd-${{ matrix.cppstd }}-$conanfile_hash" - echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a "$GITHUB_OUTPUT" - echo "ccache-restore-key=$ccache_key_prefix" | tee -a "$GITHUB_OUTPUT" + echo "ccache-key=${ccache_key_prefix}-$GITHUB_REF-${current_date}" | tee -a "$GITHUB_OUTPUT" + echo "ccache-restore-key-1=$ccache_key_prefix-$GITHUB_REF" | tee -a "$GITHUB_OUTPUT" + echo "ccache-restore-key-2=$ccache_key_prefix" | tee -a "$GITHUB_OUTPUT" - - name: Restore Conan cache - id: cache-conan + - name: Restore Conan cache (C++20) + if: matrix.cppstd == '20' uses: actions/cache/restore@v4 with: - key: ${{ needs.build-deps.outputs.cache-key }} - path: "${{ env.CONAN_HOME }}/p" + key: ${{ needs.build-conan-deps-cpp20.outputs.conan-key }} + path: ${{ env.CONAN_HOME }}/p fail-on-cache-miss: true - - name: Copy Conan settings - run: | - if [ ! -f "$CONAN_HOME/settings.yml" ]; then - cp "/root/.conan2/settings.yml" "$CONAN_HOME" - fi + - name: Restore CMake configs (C++20) + if: matrix.cppstd == '20' + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps-cpp20.outputs.cmake-prefix-debug-key }} + path: /tmp/cmake-prefix-dbg.tar + fail-on-cache-miss: true + + - name: Restore Conan cache (C++23) + if: matrix.cppstd == '23' + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps-cpp23.outputs.conan-key }} + path: ${{ env.CONAN_HOME }}/p + fail-on-cache-miss: true + + - name: Restore CMake configs (C++23) + if: matrix.cppstd == '23' + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps-cpp23.outputs.cmake-prefix-debug-key }} + path: /tmp/cmake-prefix-dbg.tar + fail-on-cache-miss: true - - name: Install build dependencies + - name: Extract CMake configs run: | - conan install . \ - --build=never \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ - -s build_type=Debug \ - -s compiler.libcxx=libstdc++11 \ - -s compiler.cppstd=${{ matrix.cppstd }} \ - --output-folder=build + mkdir conan-env + tar -xf /tmp/cmake-prefix-dbg.tar -C conan-env/ --strip-components=1 - name: Install Python headers run: | @@ -208,7 +153,7 @@ jobs: -exec sed -i 's/set(CMAKE_CXX_STANDARD 17)/set(CMAKE_CXX_STANDARD ${{ matrix.cppstd }})/' {} + cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_PREFIX_PATH="$PWD/build" \ + -DCMAKE_PREFIX_PATH="$PWD/conan-env" \ -DHICTK_ENABLE_TESTING=ON \ -DHICTK_ENABLE_FUZZY_TESTING=ON \ -DHICTK_BUILD_TOOLS=OFF \ @@ -222,7 +167,8 @@ jobs: id: cache-ccache uses: actions/cache/restore@v4 with: - key: ${{ steps.generate-cache-key.outputs.ccache-restore-key }} + key: ${{ steps.generate-cache-key.outputs.ccache-restore-key-1 }} + restore-keys: ${{ steps.generate-cache-key.outputs.ccache-restore-key-2 }} path: ${{ env.CCACHE_DIR }} - name: Reset Ccache stats @@ -272,12 +218,9 @@ jobs: if: ${{ always() }} runs-on: ubuntu-latest needs: - - build-deps - build-project steps: - name: Collect job results - if: | - needs.build-deps.result != 'success' || - needs.build-project.result != 'success' + if: needs.build-project.result != 'success' run: exit 1 diff --git a/.github/workflows/fuzzy-testing.yml b/.github/workflows/fuzzy-testing.yml index 1be3331a..488b5180 100644 --- a/.github/workflows/fuzzy-testing.yml +++ b/.github/workflows/fuzzy-testing.yml @@ -7,17 +7,17 @@ on: push: branches: [main] paths: + - ".github/workflows/build-conan-deps.yml" - ".github/workflows/fuzzy-testing.yml" - "cmake/**" - "src/**" - "test/fuzzer/**" - "CMakeLists.txt" - "conanfile.py" - tags: - - "v*.*.*" pull_request: paths: + - ".github/workflows/build-conan-deps.yml" - ".github/workflows/fuzzy-testing.yml" - "cmake/**" - "src/**" @@ -53,7 +53,15 @@ defaults: shell: bash jobs: + build-conan-deps: + name: Build Conan deps + uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main + with: + os: ubuntu-20.04 + build-project: + name: Build project + needs: build-conan-deps runs-on: ubuntu-latest strategy: fail-fast: false @@ -131,72 +139,42 @@ jobs: run: | chown -R $(id -u):$(id -g) $PWD - - name: Generate cache key - id: cache-key - run: | - conanfile_hash="${{ hashFiles('conanfile.py') }}" - - echo "conan-key=fuzzy-testing-$conanfile_hash" | tee -a "$GITHUB_OUTPUT" - - name: Install Python run: | apt-get update apt-get install -y python3.12 python3.12-dev - name: Restore Conan cache - id: cache-conan uses: actions/cache/restore@v4 with: - key: ${{ steps.cache-key.outputs.conan-key }} + key: ${{ needs.build-conan-deps.outputs.conan-key }} path: ${{ env.CONAN_HOME }}/p + fail-on-cache-miss: true - - name: Clean Conan cache (pre-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - conan remove --confirm "*" + - name: Restore CMake configs + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps.outputs.cmake-prefix-release-key }} + path: /tmp/cmake-prefix-rel.tar + fail-on-cache-miss: true - - name: Install build dependencies - run: | - conan install . \ - --build=missing \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ - -s build_type=Release \ - -s compiler.libcxx=libstdc++11 \ - -s compiler.cppstd=17 \ - --output-folder=build - - - name: Clean Conan cache (post-build) - if: steps.cache-conan.outputs.cache-hit != 'true' + - name: Extract CMake configs run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - - - name: Save Conan cache - uses: actions/cache/save@v4 - if: steps.cache-conan.outputs.cache-hit != 'true' - with: - key: ${{ steps.cache-key.outputs.conan-key }} - path: ${{ env.CONAN_HOME }}/p - env: - ZSTD_CLEVEL: 19 + mkdir conan-env + tar -xf /tmp/cmake-prefix-rel.tar -C conan-env/ --strip-components=1 - name: Configure project run: | - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="$PWD/build" \ - -DENABLE_DEVELOPER_MODE=ON \ - -DOPT_ENABLE_CLANG_TIDY=OFF \ - -DOPT_ENABLE_CPPCHECK=OFF \ - -DHICTK_ENABLE_TESTING=ON \ - -DHICTK_ENABLE_FUZZY_TESTING=ON \ - -DHICTK_BUILD_EXAMPLES=OFF \ - -DHICTK_DOWNLOAD_TEST_DATASET=OFF \ - -S . \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="$PWD/conan-env" \ + -DENABLE_DEVELOPER_MODE=ON \ + -DOPT_ENABLE_CLANG_TIDY=OFF \ + -DOPT_ENABLE_CPPCHECK=OFF \ + -DHICTK_ENABLE_TESTING=ON \ + -DHICTK_ENABLE_FUZZY_TESTING=ON \ + -DHICTK_BUILD_EXAMPLES=OFF \ + -DHICTK_DOWNLOAD_TEST_DATASET=OFF \ + -S . \ -B build - name: Build hictk_fuzzer diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml index 9afa89db..ffa004fe 100644 --- a/.github/workflows/macos-ci.yml +++ b/.github/workflows/macos-ci.yml @@ -7,6 +7,9 @@ on: push: branches: [main] paths: + - ".github/workflows/build-conan-deps.yml" + - ".github/workflows/cache-test-dataset.yml" + - ".github/workflows/evict-gha-cache.yml" - ".github/workflows/macos-ci.yml" - "cmake/**" - "examples/**" @@ -15,11 +18,12 @@ on: - "test/units/**" - "CMakeLists.txt" - "conanfile.py" - tags: - - "v*.*.*" pull_request: paths: + - ".github/workflows/build-conan-deps.yml" + - ".github/workflows/cache-test-dataset.yml" + - ".github/workflows/evict-gha-cache.yml" - ".github/workflows/macos-ci.yml" - "cmake/**" - "examples/**" @@ -64,21 +68,36 @@ jobs: var includes = [] - includes.push({ compiler_name: 'apple-clang', compiler_version: '14', os: 'macos-12', conan: '2.4.*', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-13', conan: '2.4.*', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-14', conan: '2.4.*', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-13', conan: '2.4.*', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-14', conan: '2.4.*', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler_name: 'apple-clang', compiler_version: '14', os: 'macos-12', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-13', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-14', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-13', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler_name: 'apple-clang', compiler_version: '15', os: 'macos-14', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) return { include: includes } cache-test-dataset: - uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main name: Cache test dataset + uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main + + build-conan-deps-x86: + name: Build Conan deps + uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main + with: + os: macos-12 + + build-conan-deps-arm64: + name: Build Conan deps + uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main + with: + os: macos-14 build-project: name: Build project - needs: matrix-factory + needs: + - matrix-factory + - build-conan-deps-x86 + - build-conan-deps-arm64 runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -99,9 +118,7 @@ jobs: - uses: actions/checkout@v4 - name: Generate requirements.txt for pip - run: | - echo 'conan==${{ matrix.conan }}' > requirements.txt - echo 'cmake==${{ matrix.cmake }}' >> requirements.txt + run: echo 'cmake==${{ matrix.cmake }}' > requirements.txt - uses: actions/setup-python@v5 with: @@ -136,64 +153,69 @@ jobs: # This can be used by to always update a cache entry (useful e.g. for ccache) current_date="$(date '+%s')" - conan_key_prefix="conan-$os-$compiler-$compiler_version-$conanfile_hash-$build_type" - ccache_key_prefix="ccache-$os-$compiler-$compiler_version-$GITHUB_REF-$conanfile_hash-$build_type" + ccache_key_prefix="ccache-$os-$compiler-$compiler_version-$conanfile_hash-$build_type" - echo "conan-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT + echo "ccache-key=${ccache_key_prefix}-$GITHUB_REF-${current_date}" | tee -a $GITHUB_OUTPUT + echo "ccache-restore-key-1=$ccache_key_prefix-$GITHUB_REF" | tee -a $GITHUB_OUTPUT + echo "ccache-restore-key-2=$ccache_key_prefix" | tee -a $GITHUB_OUTPUT - echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a $GITHUB_OUTPUT - echo "ccache-restore-key=$ccache_key_prefix" | tee -a $GITHUB_OUTPUT + - name: Restore Conan cache (x86) + if: matrix.os == 'macos-12' || matrix.os == 'macos-13' + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps-x86.outputs.conan-key }} + path: ${{ env.CONAN_HOME }}/p + fail-on-cache-miss: true - - name: Restore Conan cache - id: cache-conan + - name: Restore Conan cache (arm64) + if: matrix.os == 'macos-14' uses: actions/cache/restore@v4 with: - key: ${{ steps.cache-key.outputs.conan-key }} + key: ${{ needs.build-conan-deps-arm64.outputs.conan-key }} path: ${{ env.CONAN_HOME }}/p + fail-on-cache-miss: true - - name: Configure Conan - run: conan profile detect --force + - name: Restore CMake configs (x86; Debug) + if: matrix.build_type == 'Debug' && (matrix.os == 'macos-12' || matrix.os == 'macos-13') + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps-x86.outputs.cmake-prefix-debug-key }} + path: /tmp/cmake-prefix-dbg.tar + fail-on-cache-miss: true - - name: Clean Conan cache (pre-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - conan remove --confirm "*" + - name: Restore CMake configs (x86; Release) + if: matrix.build_type == 'Release' && (matrix.os == 'macos-12' || matrix.os == 'macos-13') + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps-x86.outputs.cmake-prefix-release-key }} + path: /tmp/cmake-prefix-rel.tar + fail-on-cache-miss: true - - name: Install build dependencies - run: | - conan install . \ - --build=missing \ - -pr default \ - -s "build_type=${{ matrix.build_type }}" \ - -s "compiler=${{ matrix.compiler_name }}" \ - -s "compiler.version=${{ matrix.compiler_version }}" \ - -s compiler.libcxx=libc++ \ - -s compiler.cppstd=17 \ - --output-folder="${{ github.workspace }}/build" - - - name: Clean Conan cache (post-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source + - name: Restore CMake configs (arm64; Debug) + if: matrix.build_type == 'Debug' && matrix.os == 'macos-14' + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps-arm64.outputs.cmake-prefix-debug-key }} + path: /tmp/cmake-prefix-dbg.tar + fail-on-cache-miss: true - - name: Save Conan cache - uses: actions/cache/save@v4 - if: steps.cache-conan.outputs.cache-hit != 'true' + - name: Restore CMake configs (arm64; Release) + if: matrix.build_type == 'Release' && matrix.os == 'macos-14' + uses: actions/cache/restore@v4 with: - key: ${{ steps.cache-key.outputs.conan-key }} - path: ${{ env.CONAN_HOME }}/p - env: - ZSTD_CLEVEL: 19 + key: ${{ needs.build-conan-deps-arm64.outputs.cmake-prefix-release-key }} + path: /tmp/cmake-prefix-rel.tar + fail-on-cache-miss: true + + - name: Extract CMake configs + run: | + mkdir conan-env + tar -xf /tmp/cmake-prefix-*.tar -C conan-env/ --strip-components=1 - name: Configure project run: | cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DCMAKE_PREFIX_PATH="${{ github.workspace }}/build" \ + -DCMAKE_PREFIX_PATH="$PWD/conan-env/" \ -DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \ -DHICTK_ENABLE_TESTING=ON \ -DHICTK_BUILD_EXAMPLES=ON \ @@ -209,7 +231,8 @@ jobs: id: cache-ccache uses: actions/cache/restore@v4 with: - key: ${{ steps.cache-key.outputs.ccache-restore-key }} + key: ${{ steps.cache-key.outputs.ccache-restore-key-1 }} + restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key-2 }} path: ${{ env.CCACHE_DIR }} - name: Reset Ccache stats @@ -227,7 +250,7 @@ jobs: - name: Package unit tests run: | rm -r build/src - gtar -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst + gtar --exclude='*.o' -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst - name: Upload unit tests uses: actions/upload-artifact@v4 @@ -504,8 +527,6 @@ jobs: if: ${{ always() }} runs-on: ubuntu-latest needs: - - matrix-factory - - cache-test-dataset - build-project - run-unit-tests - run-integration-tests @@ -513,8 +534,6 @@ jobs: steps: - name: Collect job results if: | - needs.matrix-factory.result != 'success' || - needs.cache-test-dataset.result != 'success' || needs.build-project.result != 'success' || needs.run-unit-tests.result != 'success' || needs.run-integration-tests.result != 'success' diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 890da06d..b4f519b2 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -7,17 +7,17 @@ on: push: branches: [main] paths: + - ".github/workflows/build-conan-deps.yml" - ".github/workflows/packaging.yml" - "cmake/**" - "src/**" - "test/packaging/**" - "CMakeLists.txt" - "conanfile.py" - tags: - - "v*.*.*" pull_request: paths: + - ".github/workflows/build-conan-deps.yml" - ".github/workflows/packaging.yml" - "cmake/**" - "src/**" @@ -35,84 +35,16 @@ defaults: shell: bash jobs: - build-deps: - runs-on: ubuntu-latest - name: Build dependencies - container: - image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-clang-18 - options: "--user=root" - - outputs: - cache-key: ${{ steps.generate-cache-key.outputs.conan-key }} - - env: - CONAN_HOME: "/opt/conan/" - - steps: - - uses: actions/checkout@v4 - - - name: Fix permissions - run: | - chown -R $(id -u):$(id -g) $PWD - - - name: Generate cache key - id: generate-cache-key - run: | - conanfile_hash="${{ hashFiles('conanfile.py') }}" - echo "conan-key=packaging-$conanfile_hash" | tee -a "$GITHUB_OUTPUT" - - - name: Restore Conan cache - id: cache-conan - uses: actions/cache/restore@v4 - with: - key: ${{ steps.generate-cache-key.outputs.conan-key }} - path: ${{ env.CONAN_HOME }}/p - - - name: Clean Conan cache (pre-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - conan remove --confirm "*" - - - name: Copy Conan settings - run: | - if [ ! -f "$CONAN_HOME/settings.yml" ]; then - cp "/root/.conan2/settings.yml" "$CONAN_HOME" - fi - - - name: Install build dependencies - run: | - conan install . \ - --build=missing \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ - -s build_type=Release \ - -s compiler.libcxx=libstdc++11 \ - -s compiler.cppstd=17 \ - --output-folder=build - - - name: Clean Conan cache (post-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - - - name: Save Conan cache - uses: actions/cache/save@v4 - if: steps.cache-conan.outputs.cache-hit != 'true' - with: - key: ${{ steps.generate-cache-key.outputs.conan-key }} - path: ${{ env.CONAN_HOME }}/p - env: - ZSTD_CLEVEL: 19 + build-conan-deps: + name: Build Conan deps + uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main + with: + os: ubuntu-20.04 test-find-package: runs-on: ubuntu-latest name: Test find_package() - needs: [build-deps] + needs: build-conan-deps container: image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-clang-18 options: "--user=root" @@ -128,31 +60,32 @@ jobs: chown -R $(id -u):$(id -g) $PWD - name: Restore Conan cache - id: cache-conan uses: actions/cache/restore@v4 with: - key: ${{ needs.build-deps.outputs.cache-key }} + key: ${{ needs.build-conan-deps.outputs.conan-key }} path: ${{ env.CONAN_HOME }}/p fail-on-cache-miss: true - - name: Install build dependencies + - name: Restore CMake configs + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps.outputs.cmake-prefix-release-key }} + path: /tmp/cmake-prefix-rel.tar + fail-on-cache-miss: true + + - name: Extract CMake configs run: | - conan install . \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ - -s build_type=Release \ - -s compiler.libcxx=libstdc++11 \ - -s compiler.cppstd=17 \ - --output-folder=build + mkdir conan-env + tar -xf /tmp/cmake-prefix-rel.tar -C conan-env/ --strip-components=1 - name: Configure project run: | - cmake -DCMAKE_BUILD_TYPE=Release \ - -DHICTK_ENABLE_TESTING=OFF \ - -DHICTK_BUILD_TOOLS=OFF \ - -DCMAKE_PREFIX_PATH="$PWD/build" \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DHICTK_ENABLE_TESTING=OFF \ + -DHICTK_BUILD_TOOLS=OFF \ + -DCMAKE_PREFIX_PATH="$PWD/conan-env" \ -DCMAKE_INSTALL_PREFIX=hictk_install \ - -S . \ + -S . \ -B hictk_build - name: Install hictk @@ -162,24 +95,23 @@ jobs: - name: Configure test project run: | - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="$PWD/build;$PWD/hictk_install/lib/cmake/hictk/" \ - -S test/packaging/test_find_package \ - -B test_find_package_build + CMAKE_PREFIX_PATH="$PWD/conan-env;$PWD/hictk_install/lib/cmake/hictk/" + + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="$CMAKE_PREFIX_PATH" \ + -S test/packaging/test_find_package \ + -B test_find_package_build - name: Build test project - run: | - cmake --build test_find_package_build + run: cmake --build test_find_package_build - name: Test binary - run: | - test_find_package_build/hictk_test_find_package + run: test_find_package_build/hictk_test_find_package test-add-subdirectory: runs-on: ubuntu-latest name: Test add_subdirectory() - needs: [build-deps] + needs: build-conan-deps container: image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-clang-18 options: "--user=root" @@ -195,54 +127,50 @@ jobs: chown -R $(id -u):$(id -g) $PWD - name: Restore Conan cache - id: cache-conan uses: actions/cache/restore@v4 with: - key: ${{ needs.build-deps.outputs.cache-key }} + key: ${{ needs.build-conan-deps.outputs.conan-key }} path: ${{ env.CONAN_HOME }}/p fail-on-cache-miss: true - - name: Install build dependencies + - name: Restore CMake configs + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps.outputs.cmake-prefix-release-key }} + path: /tmp/cmake-prefix-rel.tar + fail-on-cache-miss: true + + - name: Extract CMake configs run: | - conan install . \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ - -s build_type=Release \ - -s compiler.libcxx=libstdc++11 \ - -s compiler.cppstd=17 \ - --output-folder=build + mkdir conan-env + tar -xf /tmp/cmake-prefix-rel.tar -C conan-env/ --strip-components=1 - name: Configure project run: | ln -s "$PWD" test/packaging/test_add_subdirectory/hictk_root - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="$PWD/build" \ - -S test/packaging/test_add_subdirectory \ - -B test_add_subdirectory_build + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="$PWD/conan-env" \ + -S test/packaging/test_add_subdirectory \ + -B test_add_subdirectory_build - name: Build test project - run: | - cmake --build test_add_subdirectory_build + run: cmake --build test_add_subdirectory_build - name: Test binary - run: | - test_add_subdirectory_build/hictk_test_add_subdirectory + run: test_add_subdirectory_build/hictk_test_add_subdirectory packaging-status-check: name: Status Check (packaging) if: ${{ always() }} runs-on: ubuntu-latest needs: - - build-deps - test-find-package - test-add-subdirectory steps: - name: Collect job results if: | - needs.build-deps.result != 'success' || needs.test-find-package.result != 'success' || needs.test-add-subdirectory.result != 'success' run: exit 1 diff --git a/.github/workflows/run-clang-tidy.yml b/.github/workflows/run-clang-tidy.yml index d72bf426..109f3da0 100644 --- a/.github/workflows/run-clang-tidy.yml +++ b/.github/workflows/run-clang-tidy.yml @@ -7,7 +7,9 @@ on: push: branches: [main] paths: - - ".github/workflows/run-clang-tidy.yml" + - ".github/workflows/build-conan-deps.yml" + - ".github/workflows/evict-gha-cache.yml" + - ".github/workflows/build-project.yml" - "cmake/**" - "examples/**" - "src/**" @@ -15,12 +17,12 @@ on: - ".clang-tidy" - "CMakeLists.txt" - "conanfile.py" - tags: - - "v*.*.*" pull_request: paths: - - ".github/workflows/run-clang-tidy.yml" + - ".github/workflows/build-conan-deps.yml" + - ".github/workflows/evict-gha-cache.yml" + - ".github/workflows/build-project.yml" - "cmake/**" - "examples/**" - "src/**" @@ -39,7 +41,15 @@ defaults: shell: bash jobs: - run-clang-tidy: + build-conan-deps: + name: Build Conan deps + uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main + with: + os: ubuntu-20.04 + + build-project: + name: Build project + needs: build-conan-deps runs-on: ubuntu-latest container: image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-clang-18 @@ -72,64 +82,35 @@ jobs: # This can be used by to always update a cache entry (useful e.g. for ccache) current_date="$(date '+%s')" - ccache_key_prefix="ccache-clang-tidy-$GITHUB_REF-$conanfile_hash" + ccache_key_prefix="ccache-clang-tidy-$conanfile_hash" - echo "conan-key=conan-clang-tidy-$conanfile_hash" | tee -a "$GITHUB_OUTPUT" - echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a "$GITHUB_OUTPUT" - echo "ccache-restore-key=$ccache_key_prefix" | tee -a "$GITHUB_OUTPUT" + echo "ccache-key=${ccache_key_prefix}-$GITHUB_REF-${current_date}" | tee -a "$GITHUB_OUTPUT" + echo "ccache-restore-key-1=$ccache_key_prefix-$GITHUB_REF" | tee -a "$GITHUB_OUTPUT" + echo "ccache-restore-key-2=$ccache_key_prefix" | tee -a "$GITHUB_OUTPUT" - name: Restore Conan cache - id: cache-conan uses: actions/cache/restore@v4 with: - key: ${{ steps.cache-key.outputs.conan-key }} + key: ${{ needs.build-conan-deps.outputs.conan-key }} path: ${{ env.CONAN_HOME }}/p + fail-on-cache-miss: true - - name: Clean Conan cache (pre-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - conan remove --confirm "*" - - - name: Copy Conan settings - run: | - if [ ! -f "$CONAN_HOME/settings.yml" ]; then - cp "/root/.conan2/settings.yml" "$CONAN_HOME" - fi + - name: Restore CMake configs + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps.outputs.cmake-prefix-debug-key }} + path: /tmp/cmake-prefix-dbg.tar + fail-on-cache-miss: true - - name: Install build dependencies + - name: Extract CMake configs run: | - conan install . \ - --build=missing \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ - -s build_type=Debug \ - -s compiler.libcxx=libstdc++11 \ - -s compiler.cppstd=17 \ - --output-folder=build - - - name: Clean Conan cache (post-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - - - name: Save Conan cache - uses: actions/cache/save@v4 - if: steps.cache-conan.outputs.cache-hit != 'true' - with: - key: ${{ steps.cache-key.outputs.conan-key }} - path: ${{ env.CONAN_HOME }}/p - env: - ZSTD_CLEVEL: 19 + mkdir conan-env + tar -xf /tmp/cmake-prefix-dbg.tar -C conan-env/ --strip-components=1 - name: Configure project run: | cmake -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_PREFIX_PATH="$PWD/build" \ + -DCMAKE_PREFIX_PATH="$PWD/conan-env" \ -DENABLE_DEVELOPER_MODE=ON \ -DOPT_ENABLE_SANITIZER_ADDRESS=OFF \ -DOPT_ENABLE_SANITIZER_UNDEFINED_BEHAVIOR=OFF \ @@ -145,7 +126,8 @@ jobs: id: cache-ccache uses: actions/cache/restore@v4 with: - key: ${{ steps.cache-key.outputs.ccache-restore-key }} + key: ${{ steps.cache-key.outputs.ccache-restore-key-1 }} + restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key-2 }} path: ${{ env.CCACHE_DIR }} - name: Reset Ccache stats @@ -181,7 +163,7 @@ jobs: ZSTD_CLEVEL: 1 clean-stale-cache: - needs: [run-clang-tidy] + needs: [build-project] uses: paulsengroup/hictk/.github/workflows/evict-gha-cache.yml@main name: Clean stale Ccache cache permissions: @@ -195,9 +177,9 @@ jobs: if: ${{ always() }} runs-on: ubuntu-latest needs: - - run-clang-tidy + - build-project steps: - name: Collect job results - if: needs.run-clang-tidy.result != 'success' + if: needs.build-project.result != 'success' run: exit 1 diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index 5c795eab..163d1043 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -7,6 +7,9 @@ on: push: branches: [main] paths: + - ".github/workflows/build-conan-deps.yml" + - ".github/workflows/cache-test-dataset.yml" + - ".github/workflows/evict-gha-cache.yml" - ".github/workflows/ubuntu-ci.yml" - "cmake/**" - "examples/**" @@ -15,11 +18,12 @@ on: - "test/units/**" - "CMakeLists.txt" - "conanfile.py" - tags: - - "v*.*.*" pull_request: paths: + - ".github/workflows/build-conan-deps.yml" + - ".github/workflows/cache-test-dataset.yml" + - ".github/workflows/evict-gha-cache.yml" - ".github/workflows/ubuntu-ci.yml" - "cmake/**" - "examples/**" @@ -40,9 +44,11 @@ defaults: jobs: matrix-factory: + name: Generate job matrix runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-result.outputs.result }} + ci-type: ${{ steps.ci-type.outputs.type }} steps: - name: Checkout repo uses: actions/checkout@v4 @@ -73,67 +79,75 @@ jobs: var includes = [] // Debug builds (short CI) - includes.push({ compiler: 'gcc-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'gcc-14', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'ON' }) - includes.push({ compiler: 'clang-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-18', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'ON' }) + includes.push({ compiler: 'gcc-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-14', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'ON' }) + includes.push({ compiler: 'clang-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-18', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'ON' }) // Release builds (short CI) - includes.push({ compiler: 'gcc-14', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'ON' }) - includes.push({ compiler: 'clang-18', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'ON' }) + includes.push({ compiler: 'gcc-14', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'ON' }) + includes.push({ compiler: 'clang-18', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'ON' }) if (ci_short) { return { include: includes } } // Debug builds (long CI) - includes.push({ compiler: 'gcc-9', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'gcc-10', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'gcc-11', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'gcc-12', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'gcc-13', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-9', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-10', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-13', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-14', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-15', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-16', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-17', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-9', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-10', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-11', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-12', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-13', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-9', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-10', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-13', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-14', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-15', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-16', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-17', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) // Release builds (long CI) - includes.push({ compiler: 'gcc-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'gcc-9', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'gcc-10', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'gcc-11', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'gcc-12', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'gcc-13', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-9', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-10', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-13', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-14', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-15', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-16', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) - includes.push({ compiler: 'clang-17', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-9', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-10', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-11', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-12', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'gcc-13', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-9', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-10', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-13', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-14', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-15', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-16', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-17', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) // Make sure project builds with CMake 3.25 - includes.push({ compiler: 'clang-18', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.25.2', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-18', os: 'ubuntu-24.04', generator: 'Ninja', cmake: '3.25.2', build_type: 'Release', developer_mode: 'OFF' }) // Make sure project builds with make - includes.push({ compiler: 'clang-18', os: 'ubuntu-24.04', generator: 'Unix Makefiles', cmake: '3.29.*', conan: '2.4.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler: 'clang-18', os: 'ubuntu-24.04', generator: 'Unix Makefiles', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) return { include: includes } cache-test-dataset: - uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main name: Cache test dataset + uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main + + build-conan-deps: + name: Build Conan deps + uses: paulsengroup/hictk/.github/workflows/build-conan-deps.yml@main + with: + os: ubuntu-20.04 build-project: name: Build project - needs: matrix-factory + needs: + - matrix-factory + - build-conan-deps runs-on: ubuntu-latest strategy: fail-fast: false @@ -166,22 +180,19 @@ jobs: pattern='[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$' flag=false - if [[ $(cmake --version | grep -o "$pattern") != ${{ matrix.cmake }} || - $(conan --version | grep -o "$pattern") != ${{ matrix.conan }} ]]; then - flag=true + if [[ $(cmake --version | grep -o "$pattern") != ${{ matrix.cmake }} ]]; then + flag=true fi echo "outdated=$flag" | tee -a "$GITHUB_OUTPUT" - name: Update build deps - if: ${{ steps.buld-deps-outdated.outputs.outdated }} + if: ${{ steps.build-deps-outdated.outputs.outdated }} run: | apt-get update apt-get install -y --no-install-recommends python3-pip - python3 -m pip install \ - "cmake==${{ matrix.cmake }}" \ - "conan==${{ matrix.conan }}" + python3 -m pip install "cmake==${{ matrix.cmake }}" - name: Generate cache keys id: cache-key @@ -199,67 +210,46 @@ jobs: # This can be used by to always update a cache entry (useful e.g. for ccache) current_date="$(date '+%s')" - conan_key_prefix="conan-$os-$compiler-$conanfile_hash-$build_type-$generator-$dev_mode" - ccache_key_prefix="ccache-$os-$compiler-$GITHUB_REF-$conanfile_hash-$build_type-$generator-$dev_mode" + ccache_key_prefix="ccache-$os-$compiler-$conanfile_hash-$build_type-$generator-$dev_mode" - echo "conan-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT - - echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a $GITHUB_OUTPUT - echo "ccache-restore-key=$ccache_key_prefix" | tee -a $GITHUB_OUTPUT + echo "ccache-key=${ccache_key_prefix}-$GITHUB_REF-${current_date}" | tee -a $GITHUB_OUTPUT + echo "ccache-restore-key-1=$ccache_key_prefix-$GITHUB_REF" | tee -a $GITHUB_OUTPUT + echo "ccache-restore-key-2=$ccache_key_prefix" | tee -a $GITHUB_OUTPUT - name: Restore Conan cache - id: cache-conan uses: actions/cache/restore@v4 with: - key: ${{ steps.cache-key.outputs.conan-key }} + key: ${{ needs.build-conan-deps.outputs.conan-key }} path: ${{ env.CONAN_HOME }}/p + fail-on-cache-miss: true - - name: Clean Conan cache (pre-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - conan remove --confirm "*" + - name: Restore CMake configs (Debug) + if: matrix.build_type == 'Debug' + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps.outputs.cmake-prefix-debug-key }} + path: /tmp/cmake-prefix-dbg.tar + fail-on-cache-miss: true - - name: Copy Conan settings - run: | - if [ ! -f "$CONAN_HOME/settings.yml" ]; then - cp "/root/.conan2/settings.yml" "$CONAN_HOME" - fi + - name: Restore CMake configs (Release) + if: matrix.build_type == 'Release' + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps.outputs.cmake-prefix-release-key }} + path: /tmp/cmake-prefix-rel.tar + fail-on-cache-miss: true - - name: Install build dependencies - run: | - conan install . \ - --build=missing \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ - -s build_type="${{ matrix.build_type }}" \ - -s compiler.libcxx=libstdc++11 \ - -s compiler.cppstd=17 \ - --output-folder=build - - - name: Clean Conan cache (post-build) - if: steps.cache-conan.outputs.cache-hit != 'true' + - name: Extract CMake configs run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - - - name: Save Conan cache - uses: actions/cache/save@v4 - if: steps.cache-conan.outputs.cache-hit != 'true' - with: - key: ${{ steps.cache-key.outputs.conan-key }} - path: ${{ env.CONAN_HOME }}/p - env: - ZSTD_CLEVEL: 19 + mkdir conan-env + tar -xf /tmp/cmake-prefix-*.tar -C conan-env/ --strip-components=1 - name: Restore Ccache folder id: cache-ccache uses: actions/cache/restore@v4 with: - key: ${{ steps.cache-key.outputs.ccache-restore-key }} + key: ${{ steps.cache-key.outputs.ccache-restore-key-1 }} + restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key-2 }} path: ${{ env.CCACHE_DIR }} - name: Reset Ccache stats @@ -268,7 +258,7 @@ jobs: - name: Configure project run: | cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DCMAKE_PREFIX_PATH="$PWD/build" \ + -DCMAKE_PREFIX_PATH="$PWD/conan-env" \ -DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \ -DOPT_ENABLE_CLANG_TIDY=OFF \ -DOPT_ENABLE_CPPCHECK=OFF \ @@ -292,7 +282,7 @@ jobs: - name: Package unit tests run: | rm -r build/src - tar -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst + tar --exclude='*.o' -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst - name: Upload unit tests uses: actions/upload-artifact@v4 @@ -325,12 +315,14 @@ jobs: --verbose - name: Cleanup Ccache folder + if: needs.matrix-factory.outputs.ci-type == 'short' run: | ccache --evict-older-than=14400s # 4h ccache --recompress=19 --recompress-threads="$(nproc)" ccache --cleanup - name: Print Ccache statistics (post-cleanup) + if: needs.matrix-factory.outputs.ci-type == 'short' run: | ccache --show-stats \ --show-compression \ @@ -619,8 +611,6 @@ jobs: if: ${{ always() }} runs-on: ubuntu-latest needs: - - matrix-factory - - cache-test-dataset - build-project - run-unit-tests - run-integration-tests @@ -628,8 +618,6 @@ jobs: steps: - name: Collect job results if: | - needs.matrix-factory.result != 'success' || - needs.cache-test-dataset.result != 'success' || needs.build-project.result != 'success' || needs.run-unit-tests.result != 'success' || needs.run-integration-tests.result != 'success' diff --git a/.github/workflows/windows-ci.yml b/.github/workflows/windows-ci.yml index 044f1ab2..a35b066b 100644 --- a/.github/workflows/windows-ci.yml +++ b/.github/workflows/windows-ci.yml @@ -7,6 +7,8 @@ on: push: branches: [main] paths: + - ".github/workflows/build-conan-deps.yml" + - ".github/workflows/cache-test-dataset.yml" - ".github/workflows/windows-ci.yml" - "cmake/**" - "examples/**" @@ -15,11 +17,11 @@ on: - "test/units/**" - "CMakeLists.txt" - "conanfile.py" - tags: - - "v*.*.*" pull_request: paths: + - ".github/workflows/build-conan-deps.yml" + - ".github/workflows/cache-test-dataset.yml" - ".github/workflows/windows-ci.yml" - "cmake/**" - "examples/**" @@ -59,8 +61,8 @@ jobs: var includes = [] - includes.push({ compiler_name: 'msvc', os: 'windows-2022', conan: '2.4.*', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) - includes.push({ compiler_name: 'msvc', os: 'windows-2022', conan: '2.4.*', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) + includes.push({ compiler_name: 'msvc', os: 'windows-2022', cmake: '3.29.*', build_type: 'Debug', developer_mode: 'OFF' }) + includes.push({ compiler_name: 'msvc', os: 'windows-2022', cmake: '3.29.*', build_type: 'Release', developer_mode: 'OFF' }) return { include: includes } @@ -68,9 +70,17 @@ jobs: uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main name: Cache test dataset + build-conan-deps: + name: Build Conan deps + uses: robomics/gha-devel/.github/workflows/build-conan-deps.yml@main + with: + os: windows-2019 + build-project: - needs: matrix-factory - runs-on: windows-latest + needs: + - matrix-factory + - build-conan-deps + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} @@ -79,9 +89,7 @@ jobs: - uses: actions/checkout@v4 - name: Generate requirements.txt for pip - run: | - echo 'conan==${{ matrix.conan }}' > requirements.txt - echo 'cmake==${{ matrix.cmake }}' >> requirements.txt + run: echo 'cmake==${{ matrix.cmake }}' > requirements.txt - uses: actions/setup-python@v5 with: @@ -98,97 +106,39 @@ jobs: - name: Install deps run: pip install -r requirements.txt - - name: Generate cache key - id: cache-key - run: | - set -u - - os="${{ matrix.os }}" - compiler="${{ matrix.compiler_name }}" - build_type="${{ matrix.build_type }}" - - conanfile_hash="${{ hashFiles('conanfile.py') }}" - - conan_key_prefix="conan-$os-$compiler-$conanfile_hash-$build_type" - - echo "conan-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT - - name: Restore Conan cache - id: cache-conan uses: actions/cache/restore@v4 with: - key: ${{ steps.cache-key.outputs.conan-key }} + key: ${{ needs.build-conan-deps.outputs.conan-key }} path: ${{ env.CONAN_HOME }}\p + fail-on-cache-miss: true - - name: Configure Conan - run: | - conan profile detect --force - conan_profile="$(conan profile path default)" - - sed -i 's/build_type=Release/build_type=${{ matrix.build_type }}/' "$conan_profile" - sed -i 's/compiler\.cppstd=.*/compiler.cppstd=17/' "$conan_profile" + - name: Restore CMake configs (Debug) + if: matrix.build_type == 'Debug' + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps.outputs.cmake-prefix-debug-key }} + path: ${{ github.workspace }}\cmake-prefix-dbg.tar + fail-on-cache-miss: true - - name: Clean Conan cache (pre-build) - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - conan remove --confirm "*" + - name: Restore CMake configs (Release) + if: matrix.build_type == 'Release' + uses: actions/cache/restore@v4 + with: + key: ${{ needs.build-conan-deps.outputs.cmake-prefix-release-key }} + path: ${{ github.workspace }}\cmake-prefix-rel.tar + fail-on-cache-miss: true - - name: Compile build dependencies - if: steps.cache-conan.outputs.cache-hit != 'true' - run: | - conan install . \ - --build=missing \ - --build="b2/*" \ - --build="catch2/*" \ - -pr:h default \ - -pr:b default \ - -s:h "compiler=${{ matrix.compiler_name }}" \ - -s:h "build_type=${{ matrix.build_type }}" \ - -s:h "compiler.runtime_type=${{ matrix.build_type }}" \ - -s:h compiler.cppstd=17 \ - -s:b "compiler=${{ matrix.compiler_name }}" \ - -s:b "build_type=${{ matrix.build_type }}" \ - -s:b "compiler.runtime_type=${{ matrix.build_type }}" \ - -s:b compiler.cppstd=17 - - - name: Install build dependencies - run: | - conan install . \ - -pr:h default \ - -pr:b default \ - -s:h "compiler=${{ matrix.compiler_name }}" \ - -s:h "build_type=${{ matrix.build_type }}" \ - -s:h "compiler.runtime_type=${{ matrix.build_type }}" \ - -s:h compiler.cppstd=17 \ - -s:b "compiler=${{ matrix.compiler_name }}" \ - -s:b "build_type=${{ matrix.build_type }}" \ - -s:b "compiler.runtime_type=${{ matrix.build_type }}" \ - -s:b compiler.cppstd=17 \ - --output-folder=build - - - name: Clean Conan cache (post-build) - if: steps.cache-conan.outputs.cache-hit != 'true' + - name: Extract CMake configs run: | - conan cache clean "*" --build - conan cache clean "*" --download - conan cache clean "*" --source - - - name: Save Conan cache - uses: actions/cache/save@v4 - if: steps.cache-conan.outputs.cache-hit != 'true' - with: - key: ${{ steps.cache-key.outputs.conan-key }} - path: ${{ env.CONAN_HOME }}\p - env: - ZSTD_CLEVEL: 19 + mkdir conan-env + srcdir="$(cygpath '${{ github.workspace }}')" + tar -xf "$srcdir/"cmake-prefix-*.tar -C conan-env/ --strip-components=1 - name: Configure project run: | cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ - -DCMAKE_PREFIX_PATH="${{ github.workspace }}/build" \ + -DCMAKE_PREFIX_PATH="$PWD/conan-env" \ -DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \ -DHICTK_ENABLE_TESTING=ON \ -DHICTK_BUILD_EXAMPLES=ON \ @@ -212,7 +162,7 @@ jobs: - name: Package unit tests run: | rm -r build/src - tar -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst + tar --exclude='*.obj' -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst - name: Upload unit tests uses: actions/upload-artifact@v4 @@ -474,8 +424,6 @@ jobs: steps: - name: Collect job results if: | - needs.matrix-factory.result != 'success' || - needs.cache-test-dataset.result != 'success' || needs.build-project.result != 'success' || needs.run-unit-tests.result != 'success' || needs.run-integration-tests.result != 'success' diff --git a/Dockerfile b/Dockerfile index 4eb5906e..b5132296 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,20 +28,11 @@ RUN if [ -z "$C_COMPILER" ]; then echo "Missing C_COMPILER --build-arg" && exit ENV CC="$C_COMPILER" ENV CXX="$CXX_COMPILER" -# Install b2 using Conan -RUN printf '[requires]\nb2/5.2.1\n[options]\nb2*:toolset=%s' \ - "$(basename "$(which "$CC")")" | cut -f 1 -d - > /tmp/conanfile.txt - -RUN conan install /tmp/conanfile.txt \ - --build=missing \ - -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ - -pr:h="$CONAN_DEFAULT_PROFILE_PATH" - # Build hictk deps using Conan RUN mkdir -p "$src_dir" -COPY conanfile.py "$src_dir" -RUN conan install "$src_dir/conanfile.py" \ +COPY conanfile.Dockerfile.py "$src_dir/conanfile.py" +RUN conan install "$src_dir/conanfile.py" \ --build=missing \ -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ @@ -70,11 +61,15 @@ RUN if [ -z "$GIT_HASH" ]; then echo "Missing GIT_HASH --build-arg" && exit 1; f && if [ -z "$GIT_IS_DIRTY" ]; then echo "Missing GIT_IS_DIRTY --build-arg" && exit 1; fi \ && if [ -z "$GIT_TAG" ]; then echo "Missing GIT_TAG --build-arg" && exit 1; fi +ARG CCACHE_DISABLE=1 + # Configure project RUN cmake -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_PREFIX_PATH="$build_dir" \ - -DHICTK_ENABLE_TESTING=OFF \ -DENABLE_DEVELOPER_MODE=OFF \ + -DHICTK_ENABLE_TESTING=OFF \ + -DHICTK_WITH_ARROW=OFF \ + -DHICTK_WITH_EIGEN=OFF \ -DCMAKE_INSTALL_PREFIX="$staging_dir" \ -DGIT_RETRIEVED_STATE=true \ -DGIT_TAG="$GIT_TAG" \ @@ -87,7 +82,8 @@ RUN cmake -DCMAKE_BUILD_TYPE=Release \ # Build and install project RUN cmake --build "$build_dir" -t hictk -j "$(nproc)" \ -&& cmake --install "$build_dir" --component Runtime +&& cmake --install "$build_dir" --component Runtime \ +&& rm -r "$build_dir" ARG FINAL_BASE_IMAGE ARG FINAL_BASE_IMAGE_DIGEST diff --git a/conanfile.Dockerfile.py b/conanfile.Dockerfile.py new file mode 100644 index 00000000..47c053d7 --- /dev/null +++ b/conanfile.Dockerfile.py @@ -0,0 +1,97 @@ +# Copyright (C) 2024 Roberto Rossini +# +# SPDX-License-Identifier: MIT + +from conan import ConanFile +from conan.tools.build import check_min_cppstd + +required_conan_version = ">=1.53.0" + + +class HictkConan(ConanFile): + name = "hictk" + description = "Blazing fast toolkit to work with .hic and .cool files." + license = "MIT" + topics = ("hictk", "bioinformatics") + homepage = "https://github.com/paulsengroup/hictk" + url = "https://github.com/paulsengroup/hictk" + package_type = "header-library" + settings = "os", "arch", "compiler", "build_type" + + options = { + "shared": [True, False], + "fPIC": [True, False], + } + + default_options = { + "shared": False, + "fPIC": True, + } + + generators = "CMakeDeps" + + @property + def _min_cppstd(self): + return 17 + + def requirements(self): + self.requires("bshoshany-thread-pool/4.1.0#be1802a8768416a6c9b1393cf0ce5e9c") + self.requires("bzip2/1.0.8#d00dac990f08d991998d624be81a9526") + self.requires("cli11/2.4.2#1b431bda2fb2cd3efed633899abcd8cc") + self.requires("concurrentqueue/1.0.4#1e48e1c712bcfd892087c9c622a51502") + self.requires("fast_float/6.1.1#e29acaa3d0543dee343abe3f6815346e") + self.requires("fmt/10.2.1#9199a7a0611866dea5c8849a77467b25") + self.requires("hdf5/1.14.3#31ccd8d4de83844f5db48471df1944a1") + self.requires("highfive/2.9.0#c57477beed8b0110fadeb6da8f48bcc5") + self.requires("libarchive/3.7.4#db39e5a5cddfd3a1884daaab4b7942fb") + self.requires("libdeflate/1.20#3bd86e0160becf992346aa68b4938c40") + self.requires("lz4/1.10.0#68a01ece147a441b463d8cefea68d555", force=True) + self.requires("lzo/2.10#5725914235423c771cb1c6b607109b45") + self.requires("nlohmann_json/3.11.3#45828be26eb619a2e04ca517bb7b828d") + self.requires("parallel-hashmap/1.3.12#dc7755096d8a1fac7792fdd85760b6ca") + self.requires("readerwriterqueue/1.0.6#aaa5ff6fac60c2aee591e9e51b063b83") + self.requires("span-lite/0.11.0#519fd49fff711674cfed8cd17d4ed422") + self.requires("spdlog/1.14.1#972bbf70be1da4bc57ea589af0efde03") + self.requires("tomlplusplus/3.4.0#85dbfed71376fb8dc23cdcc0570e4727") + self.requires("xz_utils/5.4.5#b885d1d79c9d30cff3803f7f551dbe66") + self.requires("zstd/1.5.6#afefe79a309bc2a7b9f56c2093504c8b", force=True) + self.requires("zlib/1.3.1#f52e03ae3d251dec704634230cd806a2") + + def validate(self): + if self.settings.get_safe("compiler.cppstd"): + check_min_cppstd(self, self._min_cppstd) + + def configure(self): + if self.settings.compiler in ["clang", "gcc"]: + self.settings.compiler.libcxx = "libstdc++11" + + self.options["fmt"].header_only = True + self.options["hdf5"].enable_cxx = False + self.options["hdf5"].hl = False + self.options["hdf5"].threadsafe = False + self.options["hdf5"].parallel = False + self.options["highfive"].with_boost = False + self.options["highfive"].with_eigen = False + self.options["highfive"].with_opencv = False + self.options["highfive"].with_xtensor = False + self.options["libarchive"].with_acl = False + self.options["libarchive"].with_zlib = True + self.options["libarchive"].with_bzip2 = True + self.options["libarchive"].with_libxml2 = False + self.options["libarchive"].with_expat = False + self.options["libarchive"].with_iconv = False + self.options["libarchive"].with_acl = False + self.options["libarchive"].with_pcreposix = False + self.options["libarchive"].with_cng = False + self.options["libarchive"].with_nettle = False + self.options["libarchive"].with_openssl = False + self.options["libarchive"].with_libb2 = False + self.options["libarchive"].with_lz4 = True + self.options["libarchive"].with_lzo = True + self.options["libarchive"].with_lzma = True + self.options["libarchive"].with_zstd = True + self.options["libarchive"].with_mbedtls = False + self.options["libarchive"].with_xattr = False + self.options["libarchive"].with_pcre2 = False + self.options["spdlog"].header_only = True + self.options["zstd"].build_programs = False diff --git a/utils/devel/build_dockerfile.sh b/utils/devel/build_dockerfile.sh index c70323d4..1e103e69 100755 --- a/utils/devel/build_dockerfile.sh +++ b/utils/devel/build_dockerfile.sh @@ -49,7 +49,7 @@ BUILD_BASE_IMAGE="ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-$C_COMP sudo docker pull "$BUILD_BASE_IMAGE" -sudo docker buildx build --platform linux/amd64 \ +sudo docker buildx build --platform linux/amd64 --load \ --build-arg "BUILD_BASE_IMAGE=$BUILD_BASE_IMAGE" \ --build-arg "FINAL_BASE_IMAGE=docker.io/library/ubuntu" \ --build-arg "FINAL_BASE_IMAGE_TAG=24.04" \