jobs: add kola-upgrade test

This will test upgrades from older releases to verify old systems
can be updated to the latest versions. For our production FCOS streams
we'll kick off a full matrix to try to test from different points in
the history of the stream. For development/mechanical streams we'll
just kick of a single modulo $date test per build.

config.yaml

@@ -101,3 +101,4 @@ clouds:
 
 misc:
   generate_release_index: true
+  run_extended_upgrade_test_fcos: true

docs/config.yaml

@@ -235,3 +235,5 @@ misc:
   generate_release_index: true
   # OPTIONAL: require kernel + kernel-rt versions to match
   check_kernel_rt_mismatch_rhcos: true
+  # OPTIONAL: whether to run extended upgrade test kola job
+  run_extended_upgrade_test_fcos: true

jobs/build-arch.Jenkinsfile

@@ -379,6 +379,12 @@ lock(resource: "build-${params.STREAM}-${basearch}") {
                 pipeutils.run_cloud_tests(pipecfg, params.STREAM, newBuildID,
                                           s3_stream_dir, basearch, src_config_commit)
             }
+            if (pipecfg.misc?.run_extended_upgrade_test_fcos) {
+                stage('Upgrade Tests') {
+                    pipeutils.run_fcos_upgrade_tests(pipecfg, params.STREAM,
+                                                     newBuildID, basearch, src_config_commit)
+                }
+            }
         }
 
         stage('Destroy Remote') {

jobs/build.Jenkinsfile

@@ -449,6 +449,12 @@ lock(resource: "build-${params.STREAM}") {
                 pipeutils.run_cloud_tests(pipecfg, params.STREAM, newBuildID,
                                           s3_stream_dir, basearch, src_config_commit)
             }
+            if (pipecfg.misc?.run_extended_upgrade_test_fcos) {
+                stage('Upgrade Tests') {
+                    pipeutils.run_fcos_upgrade_tests(pipecfg, params.STREAM,
+                                                     newBuildID, basearch, src_config_commit)
+                }
+            }
         }
 
         // For now, we auto-release all non-production streams builds. That

jobs/kola-upgrade.Jenkinsfile

@@ -0,0 +1,244 @@
+node {
+    checkout scm
+    // these are script global vars
+    pipeutils = load("utils.groovy")
+    pipecfg = pipeutils.load_pipecfg()
+}
+
+properties([
+    pipelineTriggers([]),
+    parameters([
+      choice(name: 'STREAM',
+             choices: pipeutils.get_streams_choices(pipecfg),
+             description: 'CoreOS stream to test'),
+      string(name: 'START_VERSION',
+             description: 'CoreOS Build ID or Fedora major version to start from',
+             trim: true),
+      string(name: 'TARGET_VERSION',
+             description: 'Final CoreOS Build ID that passes test',
+             defaultValue: '',
+             trim: true),
+      string(name: 'ARCH',
+             description: 'Target architecture',
+             defaultValue: 'x86_64',
+             trim: true),
+      string(name: 'COREOS_ASSEMBLER_IMAGE',
+             description: 'Override the coreos-assembler image to use',
+             defaultValue: "quay.io/coreos-assembler/coreos-assembler:main",
+             trim: true),
+      string(name: 'SRC_CONFIG_COMMIT',
+             description: 'The exact config repo git commit to run tests against',
+             defaultValue: '',
+             trim: true),
+    ]),
+    buildDiscarder(logRotator(
+        numToKeepStr: '100',
+        artifactNumToKeepStr: '100'
+    )),
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+])
+
+
+// Map the FCOS stream name to the production stream we should
+// use for the test.
+def start_streams = [
+    'stable':         'stable',
+    'testing':        'testing',
+    'next':           'next',
+    'testing-devel':  'testing',
+    'next-devel':     'next',
+    'branched':       'next',
+    'rawhide':        'next',
+]
+
+// Deadend releases as found in
+// https://github.com/coreos/fedora-coreos-streams/tree/main/updates
+def deadends = ['38.20230310.1.0','30.20190716.1']
+
+// Vars for start/target versions
+def target_version = params.TARGET_VERSION
+def start_version = params.START_VERSION
+def start_stream = start_streams[params.STREAM]
+
+// Because of an x86_64 bootloader issue the oldest that is supported
+// on x86_64 is 32.x: https://github.com/coreos/fedora-coreos-tracker/issues/1448
+if (start_version == '' && params.ARCH == 'x86_64') {
+    start_version = '32'
+}
+
+currentBuild.description = "[${params.STREAM}][${params.ARCH}] - ${start_version}->${target_version}"
+
+// Set the memory request to a reasonable value.
+def cosa_memory_request_mb = 1024
+if (params.ARCH == 'x86_64') {
+    // local (qemu+x86_64) testing will require more memory
+    cosa_memory_request_mb = 3072
+}
+
+lock(resource: "kola-upgrade-${params.ARCH}") {
+    cosaPod(memory: "${cosa_memory_request_mb}Mi",
+            image: params.COREOS_ASSEMBLER_IMAGE,
+            serviceAccount: "jenkins") {
+    timeout(time: 90, unit: 'MINUTES') {
+    try {
+
+        // Determine the start version based on the provided params.START_VERSION
+        // and the releases.json for this stream. The user can provide a full
+        // version, the empty string (implies earliest available), or two digits
+        // (implies earliest available based on the Fedora major).
+        if (start_version.length() > 2) {
+            if (release in deadends) {
+                error("Specified start_version is a deadend release")
+            }
+        } else {
+            shwrap("curl -LO https://builds.coreos.fedoraproject.org/prod/streams/${start_stream}/releases.json")
+            def releases = readJSON file: "releases.json"
+            for (release in releases["releases"]) {
+                def has_arch = release["commits"].find{ commit -> commit["architecture"] == params.ARCH }
+                if (release in deadends || has_arch == null) {
+                    continue // This release has been disqualified
+                }
+                if (start_version.length() == 2) {
+                    if ((release["version"][0..1] as Integer) > (start_version as Integer)) {
+                        echo "There wasn't a release for this architecture for Fedora ${start_version}.. Skipping"
+                        return
+                    } else if (release["version"][0..1] == start_version) {
+                        start_version = release["version"]
+                        break
+                    }
+                } else {
+                    // No restrictions on start_version. Use oldest available
+                    start_version = release["version"]
+                    break
+                }
+            }
+        }
+        echo "Selected ${start_version} as the starting version to test"
+        currentBuild.description = "[${params.STREAM}][${params.ARCH}] - ${start_version}->${target_version}"
+
+        def remoteSession = ""
+        if (params.ARCH != 'x86_64') {
+            // If we're on mArch and using QEMU then initialize the
+            // session on the remote builder
+            stage("Initialize Remote") {
+                pipeutils.withPodmanRemoteArchBuilder(arch: params.ARCH) {
+                    remoteSession = shwrapCapture("""
+                    cosa remote-session create --image ${params.COREOS_ASSEMBLER_IMAGE} --expiration 4h --workdir ${env.WORKSPACE}
+                    """)
+                }
+            }
+        }
+
+        // Run the remaining code in a remote session if we created one.
+        pipeutils.withOptionalExistingCosaRemoteSession(
+                        arch: params.ARCH, session: remoteSession) {
+            stage('BuildFetch') {
+                def commitopt = ''
+                if (params.SRC_CONFIG_COMMIT != '') {
+                    commitopt = "--commit=${params.SRC_CONFIG_COMMIT}"
+                }
+                def ref = pipeutils.get_source_config_ref_for_stream(pipecfg, params.STREAM)
+                pipeutils.shwrapWithAWSBuildUploadCredentials("""
+                cosa init --force --branch ${ref} ${commitopt} ${pipecfg.source_config.url}
+                cosa buildfetch --artifact=qemu --stream=${start_stream} --build=${start_version} --arch=${params.ARCH}
+                cosa decompress --build=${start_version}
+                """)
+
+                // If no target version was specified the target will be the latest build
+                if (target_version == '') {
+                    target_version = shwrapCapture("cosa shell -- jq -r .builds[0].id builds/builds.json")
+                    currentBuild.description = "[${params.STREAM}][${params.ARCH}] - ${start_version}->${target_version}"
+                }
+            }
+
+            // A few independent tasks that can be run in parallel
+            def parallelruns = [:]
+
+            shwrap("""
+            cosa shell -- tee tmp/target_stream.bu <<EOF
+variant: fcos
+version: 1.0.0
+storage:
+  files:
+    - path: /etc/target_stream
+      mode: 0644
+      contents:
+        inline: |
+          ${params.STREAM}
+EOF
+""")
+
+            def kolaparams = [
+                arch: params.ARCH,
+                build: start_version,
+                cosaDir: env.WORKSPACE,
+                extraArgs: "--tag extended-upgrade --allow-rerun-success --append-butane tmp/target_stream.bu",
+                skipBasicScenarios: true,
+                skipUpgrade: true,
+            ]
+            def k1, k2, k3
+
+            switch(params.ARCH) {
+                case 'x86_64':
+                    k1 = kolaparams.clone()
+                    k1.extraArgs += " --qemu-firmware=uefi"
+                    k1.marker = "uefi"
+                    parallelruns['Kola:UEFI'] = { kola(k1) }
+                    if ((start_version[0..1] as Integer) >= 34) {
+                        // SecureBoot doesn't work on < 34 with latest qemu
+                        // https://github.com/coreos/fedora-coreos-tracker/issues/1452
+                        k2 = kolaparams.clone()
+                        k2.extraArgs += " --qemu-firmware=uefi-secure"
+                        k2.marker = "uefi-secure"
+                        parallelruns['Kola:UEFI-SECURE'] = { kola(k2) }
+                    }
+                    k3 = kolaparams.clone()
+                    k3.extraArgs += " --qemu-firmware=bios"
+                    k3.marker = "bios"
+                    parallelruns['Kola:BIOS'] = { kola(k3) }
+                    break;
+                case 'aarch64':
+                    k1 = kolaparams.clone()
+                    k1.extraArgs += " --qemu-firmware=uefi"
+                    k1.marker = "uefi"
+                    parallelruns['Kola:UEFI'] = { kola(k1) }
+                    k2 = kolaparams.clone()
+                    k2.extraArgs += " --qemu-firmware=uefi-secure"
+                    k2.marker = "uefi-secure"
+                    parallelruns['Kola:UEFI-SECURE'] = { kola(k2) }
+                    break;
+                case 's390x':
+                    parallelruns['Kola'] = { kola(kolaparams) }
+                    break;
+                case 'ppc64le':
+                    parallelruns['Kola'] = { kola(kolaparams) }
+                    break;
+                default:
+                    assert false
+                    break;
+            }
+
+            // process this batch
+            parallel parallelruns
+        }
+
+        // Destroy the remote sessions. We don't need them anymore
+        if (remoteSession != "") {
+            stage("Destroy Remote") {
+                pipeutils.withExistingCosaRemoteSession(
+                    arch: params.ARCH, session: remoteSession) {
+                    shwrap("cosa remote-session destroy")
+                }
+            }
+        }
+        currentBuild.result = 'SUCCESS'
+
+    } catch (e) {
+        currentBuild.result = 'FAILURE'
+        throw e
+    } finally {
+        if (currentBuild.result != 'SUCCESS') {
+            pipeutils.trySlackSend(message: "kola-upgrade <${env.BUILD_URL}|#${env.BUILD_NUMBER}> [${params.STREAM}][${params.ARCH}] (${start_version}->${target_version})")
+        }
+    }
+}}} // lock, cosaPod and timeout finish here

utils.groovy

@@ -249,6 +249,15 @@ def withExistingCosaRemoteSession(params = [:], Closure body) {
     }
 }
 
+// Conditionally run a closure in a remote session (iff it exists).
+def withOptionalExistingCosaRemoteSession(params = [:], Closure body) {
+    if (params['session'] == "") {
+        body()
+    } else {
+        withExistingCosaRemoteSession(params, body)
+    }
+}
+
 // Returns true if the build was triggered by a push notification.
 def triggered_by_push() {
     return (currentBuild.getBuildCauses('com.cloudbees.jenkins.GitHubPushCause').size() > 0)
@@ -482,6 +491,65 @@ def run_cloud_tests(pipecfg, stream, version, s3_stream_dir, basearch, commit) {
     parallel testruns
 }
 
+// Runs followup upgrade tests based on conditions
+def run_fcos_upgrade_tests(pipecfg, stream, version, basearch, commit) {
+    def stream_info = pipecfg.streams[stream]
+
+    def min_supported_start_versions = [
+        aarch64: 34,
+        ppc64le: 40, // We haven't released on ppc64le yet
+        s390x:   36,
+        x86_64:  32,
+    ]
+
+    // Because of an x86_64 bootloader issue the oldest that is supported
+    // on x86_64 is 32.x: https://github.com/coreos/fedora-coreos-tracker/issues/1448
+    // All other arches were shipped later than 32 so a min_version of 32 works.
+    def min_version = 32 as Integer
+    // Use a maximum version that matches the current stream Fedora major version
+    def manifest = readYaml(text:shwrapCapture("cosa shell -- cat src/config/manifest.yaml"))
+    def max_version = manifest.releasever as Integer
+
+    // A list of all the start versions we want to run tests for
+    def start_versions = []
+
+    // If this is a production build we'll run a test from the
+    // earliest release of each Fedora major we've supported for this
+    // stream. Iterate over the possibilities and kick off a job for
+    // each.
+    if (stream_info.type == 'production') {
+        for (start_version in min_version..max_version) {
+            start_versions += start_version
+        }
+    }
+
+    // If this is a non-production build we'll run a single test from
+    // a start version determined by a modulo of the number of builds
+    // for this stream. This prevents running the full matrix for each
+    // development build, but gradually covers the full matrix over a
+    // period of time.
+    if (stream_info.type in ['development', 'mechanical']) {
+        def builds = readJSON(text: shwrapCapture("cosa shell -- cat builds/builds.json"))
+        def offset = builds.builds.size() % (max_version - min_version + 1)
+        def start_version = min_version + offset
+        start_versions += start_version
+    }
+
+    for (start_version in start_versions) {
+        // Define a set of parameters that are common to all test.
+        def params = [string(name: 'STREAM', value: stream),
+                      string(name: 'TARGET_VERSION', value: version),
+                      string(name: 'ARCH', value: basearch),
+                      string(name: 'SRC_CONFIG_COMMIT', value: commit)]
+        if (start_version >= min_supported_start_versions[basearch]) {
+            params += string(name: 'START_VERSION', value: start_version as String)
+            build job: 'kola-upgrade', wait: false, parameters: params
+        } else {
+            echo "Skipping ${start_version} upgrade test for ${basearch}"
+        }
+    }
+}
+
 // Run closure, ensuring that any xz process started inside will not go over a
 // certain memory amount, which is especially useful for pods with hard set
 // memory limits.