diff --git a/.github/workflows/run-tutorial-notebook.yml b/.github/workflows/run-tutorial-notebook.yml new file mode 100644 index 0000000..6f58bf1 --- /dev/null +++ b/.github/workflows/run-tutorial-notebook.yml @@ -0,0 +1,103 @@ +# TODO: Move it to the scripts directory when the following issue is resolved: +# https://github.com/orgs/community/discussions/10773#discussioncomment-2107255 + +# NOTE: this is a reusable workflow intended to be called from the "Test notebooks" workflow. +name: Run tutorial notebook in Docker + +on: + workflow_call: + inputs: + working-directory: + description: Working directory + required: false + default: . + type: string + notebook: + description: Notebook file to be executed + required: true + type: string + outputs: + description: Output files to be uploaded + required: false + default: '' + type: string + timeout: + description: Timeout per cell in seconds + required: false + default: 3600 + type: number + +jobs: + run: + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Download the image + uses: actions/download-artifact@v4 + with: + name: image + path: /tmp + + - name: Load the image + run: | + docker load --input /tmp/image.tar + docker image ls -a madminer-jupyter-env + + - name: Prepare the shared volume + run: | + mkdir madminer_shared + git -C madminer_shared clone --depth=1 https://github.com/madminer-tool/madminer.git + + - name: Download output files uploaded by previous jobs + uses: actions/download-artifact@v4 + with: + pattern: output-* + merge-multiple: true + + - name: Run the notebook in a container + id: run-notebook + continue-on-error: true + run: > + docker run + --rm + -v $(pwd)/madminer_shared:/home/shared + -v $(pwd)/.github/workflows/scripts/run-tutorial-notebook.sh:/tmp/run-tutorial-notebook.sh:ro + madminer-jupyter-env:latest + /tmp/run-tutorial-notebook.sh /home/shared/madminer/${{ inputs.working-directory }} ${{ inputs.notebook }} ${{ inputs.timeout }} + + - name: Upload notebook and logs + uses: actions/upload-artifact@v4 + with: + name: log-${{ inputs.notebook }} + path: | + *madminer_shared/madminer/${{ inputs.working-directory }}/${{ inputs.notebook }} + *madminer_shared/madminer/${{ inputs.working-directory }}/**/*.log + if-no-files-found: ignore + overwrite: true + + # Abort before saving output files, if the notebook execution failed. + - name: Error handling + if: ${{ steps.run-notebook.outcome == 'failure' }} + run: exit 1 + + # Prepend the prefix to each path in the given list. + # The wildcard at the beginning is necessary to preserve the directory structure. + - name: Construct artifact path + id: artifact-path + run: | + eof=EOF$(openssl rand -hex 8) + echo "value<<$eof" >>$GITHUB_OUTPUT + for f in ${{ inputs.outputs }}; do + echo '*madminer_shared/madminer/${{ inputs.working-directory }}'/"$f" >>$GITHUB_OUTPUT + done + echo "$eof" >>$GITHUB_OUTPUT + + - name: Upload output files as artifacts + uses: actions/upload-artifact@v4 + if: inputs.outputs != '' + with: + name: output-${{ inputs.notebook }} + path: ${{ steps.artifact-path.outputs.value }} + if-no-files-found: error diff --git a/.github/workflows/scripts/run-tutorial-notebook.sh b/.github/workflows/scripts/run-tutorial-notebook.sh new file mode 100755 index 0000000..da0edd5 --- /dev/null +++ b/.github/workflows/scripts/run-tutorial-notebook.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# NOTE: this script is intended to be executed within a container. +set -eu + +working_directory=$1 +notebook_file=$2 +timeout=$3 + +if [ ! -f /.dockerenv ]; then + echo 'error: this script must be executed within a container' >&2 + exit 1 +fi + +echo "::group::Install Papermill" +time { + apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + # papermill 2.4.0+ is not compatible with jupyter-client 8.0.0+. + # https://github.com/scikit-hep/pyhf/issues/2104 + python3 -m pip install --no-cache-dir papermill==2.6.0 jupyter-client==7.4.9 +} +echo "::endgroup::" + +cd "$working_directory" + +# Mitigate the risk of the DataLoader freezing during training. +# https://github.com/pytorch/pytorch/issues/15808#issuecomment-1291514752 +export OMP_NUM_THREADS=1 +export MKL_NUM_THREADS=1 + +# The workflow aborts if it hits the total 6-hour limit in GitHub Actions. +# We may set a timeout (for each cell) to avoid the abort and to ensure +# in-progress notebooks and logs are saved. +if [ "$timeout" -ne 0 ]; then + papermill --execution-timeout "$timeout" "$notebook_file" "$notebook_file" +else + papermill "$notebook_file" "$notebook_file" +fi diff --git a/.github/workflows/test-notebooks.yml b/.github/workflows/test-notebooks.yml new file mode 100644 index 0000000..db41c33 --- /dev/null +++ b/.github/workflows/test-notebooks.yml @@ -0,0 +1,191 @@ +name: Test notebooks + +on: + pull_request: + push: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and export + uses: docker/build-push-action@v6 + with: + tags: madminer-jupyter-env:latest + outputs: type=docker,dest=/tmp/image.tar + cache-from: type=gha + cache-to: type=gha + + - name: Upload the image as an artifact + uses: actions/upload-artifact@v4 + with: + name: image + path: /tmp/image.tar + + tutorial_particle_physics_1: + needs: build + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: 1_setup.ipynb + outputs: data/setup.h5 + + tutorial_particle_physics_2a: + needs: tutorial_particle_physics_1 + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: 2a_parton_level_analysis.ipynb + outputs: >- + data/lhe_data_shuffled.h5 + data/lhe_data.h5 + + tutorial_particle_physics_2b: + needs: tutorial_particle_physics_1 + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: 2b_delphes_level_analysis.ipynb + outputs: >- + data/delphes_data_shuffled.h5 + data/delphes_data.h5 + + tutorial_particle_physics_3a: + needs: tutorial_particle_physics_2a + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: 3a_likelihood_ratio.ipynb + outputs: >- + models/alices_pt_settings.json + models/alices_pt_state_dict.pt + models/alices_pt_theta_means.npy + models/alices_pt_theta_stds.npy + models/alices_pt_x_means.npy + models/alices_pt_x_stds.npy + models/alices_settings.json + models/alices_state_dict.pt + models/alices_theta_means.npy + models/alices_theta_stds.npy + models/alices_x_means.npy + models/alices_x_stds.npy + + tutorial_particle_physics_3b: + needs: tutorial_particle_physics_2a + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: 3b_score.ipynb + outputs: >- + models/sally_settings.json + models/sally_state_dict.pt + models/sally_x_means.npy + models/sally_x_stds.npy + + tutorial_particle_physics_3c: + needs: tutorial_particle_physics_2a + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: 3c_likelihood.ipynb + outputs: >- + models/scandal_settings.json + models/scandal_state_dict.pt + models/scandal_theta_means.npy + models/scandal_theta_stds.npy + models/scandal_x_means.npy + models/scandal_x_stds.npy + + tutorial_particle_physics_4a: + # "no SCANDAL" (w/o 3c) as in the output of the current notebook. + needs: [tutorial_particle_physics_3a, tutorial_particle_physics_3b] + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: 4a_limits.ipynb + outputs: limits/limits.npy + + tutorial_particle_physics_4b: + needs: [tutorial_particle_physics_3a, tutorial_particle_physics_3b] + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: 4b_fisher_information.ipynb + + tutorial_particle_physics_4c: + needs: tutorial_particle_physics_4a + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: 4c_information_geometry.ipynb + + tutorial_particle_physics_a1: + needs: tutorial_particle_physics_1 + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: A1_systematic_uncertainties.ipynb + outputs: >- + data/lhe_data_systematics.h5 + data/setup_systematics.h5 + + tutorial_particle_physics_a2: + needs: build + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: A2_ensemble_methods.ipynb + + tutorial_particle_physics_a3: + needs: tutorial_particle_physics_1 + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: A3_reweighting_existing_samples.ipynb + outputs: data/setup_with_extra_benchmark.h5 + + tutorial_particle_physics_a4: + needs: tutorial_particle_physics_3a + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: A4_lh_nosyst.ipynb + + tutorial_particle_physics_a5: + needs: tutorial_particle_physics_a1 + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: A5_test_new_likelihood_module.ipynb + + tutorial_particle_physics_a6: + needs: build + if: ${{ !cancelled() && !failure() }} + uses: ./.github/workflows/run-tutorial-notebook.yml + with: + working-directory: examples/tutorial_particle_physics + notebook: A6_finite_differences.ipynb + outputs: >- + data/lhe_data_fd.h5 + data/setup_fd.h5