From b4d4a3e6eced7389a36f168cbe5cc4116ff210b0 Mon Sep 17 00:00:00 2001 From: Alec Scott Date: Fri, 19 Jul 2024 18:42:43 -0600 Subject: [PATCH] ci: rewrite to react to modified files (#374) * ci: switch to checking files changed * Add regenerate push to update branches * ci: switch to PR specific way of checkout for forks * Use built container if possible * ci: download from artifact if PR * ci: remove scheduled run and exec on new tags --- .github/workflows/ci.yml | 52 +++++++++++++++ .github/workflows/container.yml | 66 +++++++++++++++++++ .github/workflows/containers.yaml | 87 -------------------------- .github/workflows/outputs.yml | 64 +++++++++++++++++++ .github/workflows/regenerate.yaml | 21 ------- {docker => container}/Dockerfile | 2 +- {docker => container}/concretizer.yaml | 0 {docker => container}/config.yaml | 0 {docker => container}/packages.yaml | 0 {docker => container}/tutorial.pub | 0 10 files changed, 183 insertions(+), 109 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/container.yml delete mode 100644 .github/workflows/containers.yaml create mode 100644 .github/workflows/outputs.yml delete mode 100644 .github/workflows/regenerate.yaml rename {docker => container}/Dockerfile (92%) rename {docker => container}/concretizer.yaml (100%) rename {docker => container}/config.yaml (100%) rename {docker => container}/packages.yaml (100%) rename {docker => container}/tutorial.pub (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..f4199fcfe6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,52 @@ +name: ci +on: + push: + branches: + - main + tags: + - '*' + + pull_request: + branches: + - main + +concurrency: + group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + container: ${{ steps.filter.outputs.container }} + outputs: ${{ steps.filter.outputs.outputs }} + + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # @v2 + if: ${{ github.event_name == 'push' }} + with: + fetch-depth: 0 + + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 + id: filter + with: + filters: | + container: + - '.github/**' + - 'container/**' + outputs: + - '.github/**' + - 'outputs/**.sh' + + container: + if: ${{ needs.changes.outputs.container == 'true' }} + needs: changes + uses: ./.github/workflows/container.yml + + outputs: + if: ${{ needs.changes.outputs.outputs == 'true' }} + needs: [ changes, container ] + uses: ./.github/workflows/outputs.yml diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 0000000000..9300104245 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,66 @@ +name: Build Tutorial Container +on: + workflow_call: + +jobs: + build: + runs-on: ubuntu-latest + + permissions: + packages: write + + steps: + - name: Remove pre-installed apps to make space for build + run: | + sudo apt-get remove -y '^llvm-.*' + sudo apt-get remove -y '^dotnet-.*' + sudo apt-get remove -y 'php.*' + sudo apt-get remove -y '^mongodb-.*' + sudo apt-get remove -y '^mysql-.*' + sudo apt-get autoremove -y + sudo apt-get clean + sudo rm -rf /usr/share/dotnet/ + sudo rm -rf /usr/local/share/powershell + sudo rm -rf /usr/local/share/chromium + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/local/lib/node_modules + + - name: Set up QEMU + uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 + + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate Container Metadata + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 + with: + images: ghcr.io/spack/tutorial + + - name: Build & Push + uses: docker/build-push-action@1ca370b3a9802c92e886402e0dd88098a2533b12 + env: + DOCKER_BUILD_SUMMARY: false + DOCKER_BUILD_RECORD_UPLOAD: false + with: + context: "{{defaultContext}}:container" + platforms: linux/amd64 + cache-from: type=registry,ref=ghcr.io/spack/tutorial:buildcache + cache-to: ${{ github.ref == 'refs/heads/main' && 'type=registry,ref=ghcr.io/spack/tutorial:buildcache,mode=max' || '' }} + outputs: ${{ github.ref != 'refs/heads/main' && 'type=docker,dest=/tmp/tutorial-container.tar' || 'type=registry' }} + + - name: Upload container as artifact for PRs + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b + with: + name: tutorial-container + path: /tmp/tutorial-container.tar + retention-days: 1 + compression-level: 9 diff --git a/.github/workflows/containers.yaml b/.github/workflows/containers.yaml deleted file mode 100644 index e94cfd6bf0..0000000000 --- a/.github/workflows/containers.yaml +++ /dev/null @@ -1,87 +0,0 @@ -name: Containers - -on: - workflow_dispatch: - - push: - branches: - - main - - pull_request: - branches: - - main - - schedule: - - cron: '27 1 * * *' - - release: - types: [published] - -jobs: - tutorial-images: - runs-on: ubuntu-latest - permissions: - packages: write - strategy: - fail-fast: false - matrix: - # 0: image name, - # 1: Dockerfile - dockerfile: [ - [tutorial, docker/Dockerfile] - ] - name: Build ${{ matrix.dockerfile[0] }} - steps: - - name: Make space for build - run: | - sudo apt-get remove -y '^llvm-.*' - sudo apt-get remove -y '^dotnet-.*' - sudo apt-get autoremove -y - rm -rf /usr/share/dotnet/ - - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - - - name: Set Container Tag Normal (Nightly) - run: | - container="${{ matrix.dockerfile[0] }}:latest" - echo "container=${container}" >> $GITHUB_ENV - echo "versioned=${container}" >> $GITHUB_ENV - - - name: Set Container Tag on Release - if: github.event_name == 'release' - run: | - versioned="${{matrix.dockerfile[0]}}:${GITHUB_REF##*/}" - echo "versioned=${versioned}" >> $GITHUB_ENV - - - name: Check ${{ matrix.dockerfile[1] }} Exists - run: | - printf "Preparing to build ${{ env.container }} from ${{ matrix.dockerfile[1] }}" - if [ ! -f "${{ matrix.dockerfile[1]}}" ]; then - printf "Dockerfile ${{ matrix.dockerfile[1]}} does not exist" - exit 1; - fi - - - name: Set up QEMU - uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build & Deploy ${{ matrix.dockerfile[0] }} - uses: docker/build-push-action@1ca370b3a9802c92e886402e0dd88098a2533b12 - with: - file: ${{matrix.dockerfile[1]}} - platforms: linux/amd64 - context: ./docker - push: ${{ github.event_name != 'pull_request' }} - tags: | - ghcr.io/spack/${{ env.container }} - ghcr.io/spack/${{ env.versioned }} diff --git a/.github/workflows/outputs.yml b/.github/workflows/outputs.yml new file mode 100644 index 0000000000..a72560d81f --- /dev/null +++ b/.github/workflows/outputs.yml @@ -0,0 +1,64 @@ +name: Generate outputs +on: + workflow_call: + +jobs: + re-generate: + runs-on: ubuntu-latest + + permissions: + contents: write + + steps: + - name: Remove pre-installed apps to make space for build + run: | + sudo apt-get remove -y '^llvm-.*' + sudo apt-get remove -y '^dotnet-.*' + sudo apt-get autoremove -y + sudo rm -rf /usr/share/dotnet/ + sudo docker system prune -af + + - name: Checkout (Push) + if: github.event_name != 'pull_request' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + with: + ref: ${{ github.head_ref }} + + - name: Checkout (PR) + if: github.event_name == 'pull_request' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + + - name: Download container from artifact if PR + if: github.event_name == 'pull_request' + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 + with: + name: tutorial-container + path: /tmp + + - name: Import container from PR artifact + if: github.event_name == 'pull_request' + run: | + docker load --input /tmp/tutorial-container.tar + docker tag $(docker image ls -q) ghcr.io/spack/tutorial:latest + docker image ls -a + + - name: Re-generate outputs from tutorial scripts + run: | + chmod -R a+w outputs + make -C outputs -j $(nproc) DOCKER_RUN_OPTS="-e COLUMNS=94 -e LINES=24" container=ghcr.io/spack/tutorial + + - name: Show diff of modified files + run: | + git diff HEAD . + + - name: Push modified files to branch + if: ( github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork ) || github.event_name != 'pull_request' + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add . + git commit -m "Update generated outputs on behalf of [@${{ github.actor }}]" + git push diff --git a/.github/workflows/regenerate.yaml b/.github/workflows/regenerate.yaml deleted file mode 100644 index cef201c176..0000000000 --- a/.github/workflows/regenerate.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: Generate outputs - -on: workflow_dispatch - -jobs: - generate-outputs: - runs-on: ubuntu-latest - name: Generate ouputs - steps: - - name: Checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 - - - name: Run - run: | - chmod -R a+w outputs - make -C outputs -j $(nproc) DOCKER_RUN_OPTS="-e COLUMNS=94 -e LINES=24" - - - name: Create pull request - run: gh pr create -B main -H regenerate-outputs --title "Regenerate outputs" --body "Automated update" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/docker/Dockerfile b/container/Dockerfile similarity index 92% rename from docker/Dockerfile rename to container/Dockerfile index e3a1a5654e..6c0ad0294c 100644 --- a/docker/Dockerfile +++ b/container/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/spack/tutorial-ubuntu-22.04:v2023-10-30 +FROM ghcr.io/spack/tutorial-ubuntu-22.04:v2024-05-07 ENV DEBIAN_FRONTEND=noninteractive diff --git a/docker/concretizer.yaml b/container/concretizer.yaml similarity index 100% rename from docker/concretizer.yaml rename to container/concretizer.yaml diff --git a/docker/config.yaml b/container/config.yaml similarity index 100% rename from docker/config.yaml rename to container/config.yaml diff --git a/docker/packages.yaml b/container/packages.yaml similarity index 100% rename from docker/packages.yaml rename to container/packages.yaml diff --git a/docker/tutorial.pub b/container/tutorial.pub similarity index 100% rename from docker/tutorial.pub rename to container/tutorial.pub