Cron job to start rebuild everything pipelines

.github/workflows/custom_docker_builds.yml

@@ -45,6 +45,8 @@ jobs:
             image-tags: ghcr.io/spack/django:0.2.8
           - docker-image: ./images/ci-prune-buildcache
             image-tags: ghcr.io/spack/ci-prune-buildcache:0.0.3
+          - docker-image: ./images/git-remote-sync
+            image-tags: ghcr.io/spack/git-remote-sync:0.0.1
     steps:
       - name: Checkout
         uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3

images/git-remote-sync/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.10
+
+WORKDIR /scripts/
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+COPY sync-git-remotes.py ./
+
+ENTRYPOINT [ "python", "./sync-git-remotes.py"]
+

images/git-remote-sync/requirements.txt

@@ -0,0 +1 @@
+sentry-sdk==1.32.0

images/git-remote-sync/sync-git-remotes.py

@@ -0,0 +1,116 @@
+import argparse
+import base64
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from contextlib import contextmanager
+
+try:
+    import sentry_sdk
+    sentry_sdk.init(
+        # This cron job only runs once weekly,
+        # so just record all transactions.
+        traces_sample_rate=1.0,
+    )
+except Exception:
+    print("Could not configure sentry.")
+
+
+# Copied from SpackCIBridge.py
+def setup_ssh(self, ssh_key_base64):
+    """Start the ssh agent."""
+    print("Starting ssh-agent")
+    output = subprocess.run(["ssh-agent", "-s"], check=True, stdout=subprocess.PIPE).stdout
+
+    # Search for PID in output.
+    pid_regexp = re.compile(r"SSH_AGENT_PID=([0-9]+)")
+    match = pid_regexp.search(output.decode("utf-8"))
+    if match is None:
+        print("WARNING: could not detect ssh-agent PID.", file=sys.stderr)
+        print("ssh-agent will not be killed upon program termination", file=sys.stderr)
+    else:
+        pid = match.group(1)
+        os.environ["SSH_AGENT_PID"] = pid
+        self.cleanup_ssh_agent = True
+
+    # Search for socket in output.
+    socket_regexp = re.compile(r"SSH_AUTH_SOCK=([^;]+);")
+    match = socket_regexp.search(output.decode("utf-8"))
+    if match is None:
+        print("WARNING: could not detect ssh-agent socket.", file=sys.stderr)
+        print("Key will be added to caller's ssh-agent (if any)", file=sys.stderr)
+    else:
+        socket = match.group(1)
+        os.environ["SSH_AUTH_SOCK"] = socket
+
+    # Add the key.
+    ssh_key = base64.b64decode(ssh_key_base64)
+    ssh_key = ssh_key.replace(b"\r", b"")
+    with tempfile.NamedTemporaryFile() as fp:
+        fp.write(ssh_key)
+        fp.seek(0)
+        subprocess.run(["ssh-add", fp.name], check=True)
+
+
+def setup_repo(pull_remote, push_remote, branch):
+    subprocess.run(["git", "init"], check=True)
+    subprocess.run(["git", "config", "user.email", "noreply@spack.io"], check=True)
+    subprocess.run(["git", "config", "user.name", "spackbot"], check=True)
+    subprocess.run(["git", "config", "advice.detachedHead", "false"], check=True)
+
+    subprocess.run(["git", "remote", "add", "origin", pull_remote], check=True)
+    subprocess.run(["git", "remote", "set-url", "--push", "origin", push_remote], check=True)
+
+    fetch_args = ["git", "fetch", "-q", "origin", branch]
+    subprocess.run(fetch_args, check=True, stdout=subprocess.PIPE).stdout
+    subprocess.run(["git", "checkout", branch], check=True)
+
+
+@contextmanager
+def chworkingdir(path):
+    save_dir = os.path.realpath(os.curdir)
+    if path:
+        if not os.path.exists(path):
+            os.makedirs(path)
+        os.chdir(path)
+        yield path
+    else:
+        with tempfile.TemporaryDirectory() as tmpdir:
+            os.chdir(tmpdir)
+            yield tmpdir
+
+    os.chdir(save_dir)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Pull and repush a branch from one remote to another")
+    parser.add_argument("--push-remote", help="Host to push to")
+    parser.add_argument("--push-branch", help="Name of pushed branch")
+
+    parser.add_argument("--tag", action="store_true", help="Use a tag instead of a branch alias")
+
+    parser.add_argument("--pull-remote", help="Host to pull to")
+    parser.add_argument("--pull-branch", help="Name of pulled branch")
+
+    parser.add_argument("--working-dir", help="Set the working directory")
+
+    args = parser.parse_args()
+
+    # Configure ssh for pushing to gitlab
+    ssh_key_base64 = os.getenv("GITLAB_SSH_KEY_BASE64")
+    if ssh_key_base64 is None:
+        raise Exception("GITLAB_SSH_KEY_BASE64 environment is not set")
+    setup_ssh(ssh_key_base64)
+
+    with chworkingdir(args.working_dir):
+        setup_repo(args.pull_remote, args.push_remote, args.pull_branch)
+
+        if args.tag:
+            subprocess.run(["git", "tag", args.push_branch], check=True)
+        else:
+            subprocess.run(["git", "checkout", "-b", args.push_branch], check=True)
+
+        # Push the renamed branch to the push remote
+        subprocess.run(["git", "push", "-f", "origin", args.push_branch], check=True)

k8s/production/custom/rebuild-everything/cron-jobs.yaml

@@ -0,0 +1,42 @@
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: rebuild-everything
+  namespace: custom
+spec:
+  schedule: "0 0 */14 * 6"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 1200 # terminate any running job after 20 minutes
+      backoffLimit: 0
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: sync
+            image: ghcr.io/spack/git-remote-sync:0.0.1
+            imagePullPolicy: IfNotPresent
+            resources:
+              requests:
+                cpu: 500m
+                memory: 500M
+            env:
+            # This secret is double base64 encoded
+            - name: GITLAB_SSH_KEY_BASE64
+              valueFrom:
+                secretKeyRef:
+                  name: rebuild-everything
+                  key: gitlab-ssh-key
+            args:
+            - "--push-mirror"
+            - "git@ssh.gitlab.spack.io:spack/spack.git"
+            - "--push-branch"
+            - "develop-re-$(date --iso-8601)"
+            - "--pull-mirror"
+            - "https://github.com/spack/spack.git"
+            - "--pull-branch"
+            - "develop"
+          nodeSelector:
+            spack.io/node-pool: base

k8s/production/custom/rebuild-everything/sealed-secrets.yaml

@@ -0,0 +1,14 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: rebuild-everything
+  namespace: custom
+spec:
+  encryptedData:
+    # This token must have repo:status permissions
+    gitlab-ssh-key: AgBOg3sj4PYdQRvGEISHOUQVRa1FSEX+oiILWWrvGGUZ5jKrcCX+7vNKJZx/9MQIkMhZGFfkfvCyUjcu7ztehdfljnIZMdnkC3JvYp4l8AmGJhC6S3rJwHbQZaAiv+zWTrHHLztGI5rm9CnhzzLM5eAdWrFy8lPYoF89dzQuRrJgNQ/7eDpfEhtfXVQkNO/pzLTBTlQ4vBjj/eC3d34SuYygfmUXmmJvYMrw/3NsRFOBiUQiywU+m13VA2l+xRG9CNSBMxktyLj1oW7MKLFuIRb0Jnv5hOg6rBoXhRATaHeTj2n4VNr4ztbZtSeq2k/ZL1bgk7nppfSIc/K8crpBXApQBHkORjQnhnGGWSZqii5LC7sU0tRQOUu0987xAtO6V/EvpAa+ZsthhGyCv1E7VK/NemFVHipf+2cO39y7AC5Xk6wUn+FpW18yWPpPofylvIdLjmh5qDlqKPx8ZLgcwx+/1IHjmod7iSKHFw8wWQgX8W0NmAjqWmblHaCoXLrqEEOvQIyT0LH7+VjRJtprEnyPUFTAy75Kk6VFa3UkBkzorsdGw7R2vhEVVzV/WnZSY7rAL//CLdiXP/ut/tf8aHj4A1zezSRo3nm7lXl7DfCa9NAUNg+Y20AGpNKKAGmWRwAcxw92jmK09lrWVmFxAxRoxz3HexzpAvJxb2YEU9/C4gkP0+OYZ1pqKQI+UfISjrzNco3eYWXX9rrTEYqrcHu/FCRKTGp2oKndsAH5ehPONo8nV23uatUS7uhBfZCF0MfqZrfLaPhCMxiMlvuVtFc1WmtzJ4NwxoaBXhrtYgwVP+k2nPXEVYvbwNNG7/1UdGdgdJMhJmvkoPgwXbRF9PddI+9tm4jAp6fXj837yAeVA5Je/kNdXhlw4LmADm7J5E1jNIDMJg6MqGAOZ1aNJypb+YVZ/pbULTUnP963jlzzp1FFuHT/CnaB14/MeXWYCBtqUHiQ5pExxSxgaI4Fjm63loQSEaDiO3+PkghsNWkNQ10liFznAGSoABfiIaOcadnBWsI2zJ+bE1ST2U8mVF2ClvzloJ8lk23p3MvstuliEQqRlKLDWPERAEBIljd5GtQBH2LVq7FsKoHWIqLSXjdUxNE13ns+ngDwCIA5snoKyqxec9IJLpBaTMnyDStMdLbxwuE6PJRVYOoktTa+OV1Ja6yBYmyjAQG5hzJMxY1FBQZTSUlB5K029sT5s35o8aRrpiwatcQLKZuKqw6SI0wtSgKIkL5f3BnuwGeUfeUPveW0IAxHH0dUbQEZkNyqjgQbVq6Oui4LGp9ZLgFLpNYmVC0O62y1cOtJmmo5jztPJS+zJ0zJKbgArKUgsZa7+9BVJefTTmY43C1TttFbVQhg5+O0KY9xuoxfxY6y4tWjuTm3fK3l5e/UrA+8SR+PX0CWm2puADR7eNyvWK1YmogNOf//FA==
+  template:
+    metadata:
+      annotations:
+        kustomize.toolkit.fluxcd.io/reconcile: disabled
+        sealedsecrets.bitnami.com/managed: "true"