From 468cb918971f161fce0f9df54c34770614a2f9ee Mon Sep 17 00:00:00 2001
From: Victor Lin <13424970+victorlin@users.noreply.github.com>
Date: Tue, 28 Feb 2023 14:26:47 -0800
Subject: [PATCH] CI: Validate images across platforms

Start off by checking software in which versions are known to be
non-deterministic.

Co-authored-by: Thomas Sibley <tsibley@fredhutch.org>
---
 .github/workflows/ci.yml |  22 ++++++++-
 devel/validate-platforms | 101 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 121 insertions(+), 2 deletions(-)
 create mode 100755 devel/validate-platforms

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 14180b7c..9206974c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -79,6 +79,24 @@ jobs:
       env:
         DOCKER_DEFAULT_PLATFORM: ${{ matrix.platform }}
 
+  validate-platforms:
+    name: Validate platforms
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: docker/setup-qemu-action@v2
+
+    - uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: nextstrain-bot
+        password: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_MANAGE_PACKAGES }}
+
+    - name: Validate final images
+      run: ./devel/validate-platforms -r ghcr.io -t ${{ needs.build.outputs.tag }}
+
   push-branch:
     if: startsWith(needs.build.outputs.tag, 'branch-') && github.event_name != 'pull_request'
     needs: build
@@ -103,7 +121,7 @@ jobs:
 
   push-build:
     if: startsWith(needs.build.outputs.tag, 'build-')
-    needs: [build, test]
+    needs: [build, test, validate-platforms]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
@@ -126,7 +144,7 @@ jobs:
 
   cleanup-registry:
     if: always()
-    needs: [build, test, push-branch, push-build]
+    needs: [build, test, validate-platforms, push-branch, push-build]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
diff --git a/devel/validate-platforms b/devel/validate-platforms
new file mode 100755
index 00000000..71db2781
--- /dev/null
+++ b/devel/validate-platforms
@@ -0,0 +1,101 @@
+#!/bin/bash
+#
+# Validate different platform builds of the final Nextstrain image.
+#
+set -euo pipefail
+
+# Set default values.
+registry=localhost:5000
+tag=latest
+
+# Read command-line arguments.
+while getopts "r:t:" opt; do
+    case "$opt" in
+        r) registry="$OPTARG";;
+        t) tag="$OPTARG";;
+        *) echo "Usage: $0 [-r <registry>] [-t <tag>]" 1>&2; exit 1;;
+    esac
+done
+
+IMAGE="$registry/nextstrain/base:$tag"
+PLATFORMS=(linux/amd64 linux/arm64)
+
+main() {
+    # Check software where downloaded versions are not deterministic at
+    # different points in time (e.g. if "latest" is downloaded).
+
+    local report_dir
+    report_dir="$(mktemp -dt "$(basename "$0")")"
+
+    for platform in "${PLATFORMS[@]}"; do
+        echo "[$platform] Pulling image..."
+        docker pull -q --platform "$platform" "$IMAGE"
+
+        echo "[$platform] Checking that the platform is expected..."
+        check-platform "$platform"
+
+        # Initialize a directory for the report file, ensuring slashes in the
+        # platform name are subdirs.
+        mkdir -p "$(dirname "$report_dir/$platform")"
+
+        # Create a report file for the platform.
+        echo "[$platform] Determining software versions..."
+        docker-run "$platform" bash -c '
+            PS4="\$ "
+            set -x
+
+            nextclade --version
+            augur --version
+            auspice --version
+            pip show evofr | grep Version
+            datasets --version
+            dataformat version
+        ' >"$report_dir/$platform" 2>&1
+    done
+
+    # Compare contents of the first platform's report file against others.
+    first_report="$report_dir/${PLATFORMS[0]}"
+    echo "The report for ${PLATFORMS[0]} has the following contents:"
+    cat "$first_report"
+
+    echo "Comparing against other platforms..."
+    # NOTE: if running on macOS ≥13, you may need to install GNU diff for the
+    # --from-file option.
+    if cd "$report_dir" && diff --unified=1 --from-file="${PLATFORMS[0]}" "${PLATFORMS[@]:1}"; then
+        echo "Success!  All versions the same." >&2
+    else
+        echo "Failure!" >&2
+        exit 1
+     fi
+}
+
+check-platform() {
+    # Check that the platform is actually what we expect it to be.
+    local platform="$1"
+
+    python_platform_string="$(docker-run "$platform" python -c "import platform; print(platform.platform())")"
+
+    case "$platform" in
+        linux/amd64)
+            if [[ "$python_platform_string" != *"x86_64"* ]]; then
+                echo "Platform $platform not detected." 1>&2; exit 1
+            fi;;
+        linux/arm64)
+            if [[ "$python_platform_string" != *"aarch64"* ]]; then
+                echo "Platform $platform not detected." 1>&2; exit 1
+            fi;;
+        *)
+            echo "Platform $platform not supported." 1>&2; exit 1;;
+    esac
+}
+
+docker-run() {
+    # Run a command under the final Nextstrain image built for a specific
+    # platform.
+    local platform="$1"
+    local command=("${@:2}")
+
+    docker run --platform "$platform" "$IMAGE" "${command[@]}"
+}
+
+main