40ignition-ostree: add coreos-inject-rootmap.service

overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.service

@@ -36,10 +36,6 @@ After=dracut-cmdline.service
 # Any services looking at mounts need to order after this
 # because it causes device re-probing.
 After=coreos-gpt-setup.service
-# Older unit name kept for compatibility. This will be safe to remove once the
-# dracut-ignition changes from [1] are available in the latest ignition RPM.
-# [1] https://github.com/coreos/ignition-dracut/pull/191
-After=coreos-gpt-setup@dev-disk-by\x2dlabel-root.service
 # Since we are mounting /boot/, require the device first
 Requires=dev-disk-by\x2dlabel-boot.device
 After=dev-disk-by\x2dlabel-boot.device

overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.service

@@ -7,7 +7,7 @@ Before=ignition-diskful.target
 Wants=systemd-udevd.service
 After=systemd-udevd.service
 
-# This unit must the first to run when the disk holding the root partition
+# This unit must be the first to run when the disk holding the root partition
 # becomes available. To avoid relying on the name of the root partition which
 # is different between RHCOS LUKS setup and current FCOS setup, we wait for the
 # partition labeled 'boot' to become available. This is reliable as we don't

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-growpart

@@ -8,13 +8,33 @@ set -euo pipefail
 # Growpart /, until we can fix Ignition for separate /var
 # (And eventually we want ignition-disks)
 
+saved_partstate=/run/ignition-ostree-rootfs-partstate.json
+
 path=$1
 shift
 
 # The use of tail is to avoid errors from duplicate mounts;
 # this shouldn't happen for us but we're being conservative.
 src=$(findmnt -nvr -o SOURCE "$path" | tail -n1)
 
+if [ "${path}" == /sysroot ] && [ -f "${saved_partstate}" ]; then
+    # We're still ironing out our rootfs automatic growpart story, see e.g.:
+    # https://github.com/coreos/fedora-coreos-tracker/issues/570
+    # https://github.com/coreos/fedora-coreos-tracker/issues/586
+    #
+    # In the context of rootfs reprovisioning, for now our rule is the
+    # following: if the rootfs partition was moved off of the boot disk or it
+    # was resized, then we don't growpart.
+    #
+    # To detect this, we compare the output of `lsblk -o PATH,SIZE` before and
+    # after `ignition-disks.service`.
+    partstate=$(lsblk "${src}" --nodeps --json -b -o PATH,SIZE | jq -c .)
+    if [ "${partstate}" != "$(cat "${saved_partstate}")" ]; then
+        echo "coreos-growpart: detected rootfs partition changes; not auto-growing"
+        exit 0
+    fi
+fi
+
 # Get the filesystem type before extending the partition.  This matters
 # because the partition, once extended, might include leftover superblocks
 # from the previous contents of the disk (notably ZFS), causing blkid to
@@ -67,4 +87,5 @@ case "${TYPE}" in
     ext4) resize2fs "${src}" ;;
 esac
 
-touch /var/lib/coreos-growpart.stamp
+# this is useful for tests
+touch /run/coreos-growpart.stamp

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-inject-rootmap.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=CoreOS Inject Rootmap
+# If root is specified, assume rootmap is already configured
+ConditionKernelCommandLine=!root
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+
+After=ignition-files.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/rdcore rootmap /sysroot --boot-device /dev/disk/by-label/boot
+RemainAfterExit=yes
+MountFlags=slave

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-rootflags.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+
+rootpath=/dev/disk/by-label/root
+
+# If the rootfs was reprovisioned, then the user is free to define their own
+# rootflags.
+if [ -d /run/ignition-ostree-rootfs ]; then
+    exit 0
+fi
+
+eval $(blkid -o export ${rootpath})
+# this really should always be true, but let's be conservative
+if [ "${TYPE}" == "xfs" ]; then
+    # We use prjquota on XFS by default to aid multi-tenant Kubernetes (and
+    # other container) clusters.  See
+    # https://github.com/coreos/coreos-assembler/pull/303/commits/6103effbd006bb6109467830d6a3e42dd847668d
+    echo "prjquota"
+fi

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-dracut-rootfs.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+set -euo pipefail
+
+# This is implementation details of Ignition; in the future, we should figure
+# out a way to ask Ignition directly whether there's a filesystem with label
+# "root" being set up.
+ignition_cfg=/run/ignition.json
+rootdisk=/dev/disk/by-label/root
+saved_sysroot=/run/ignition-ostree-rootfs
+partstate=/run/ignition-ostree-rootfs-partstate.json
+
+case "${1:-}" in
+    detect)
+        wipes_root=$(jq '.storage?.filesystems? // [] | map(select(.label == "root" and .wipeFilesystem == true)) | length' "${ignition_cfg}")
+        if [ "${wipes_root}" = "0" ]; then
+            exit 0
+        fi
+        echo "Detected rootfs replacement in fetched Ignition config: /run/ignition.json"
+        mkdir "${saved_sysroot}"
+        ;;
+    save)
+        mount "${rootdisk}" /sysroot
+        echo "Moving rootfs to RAM..."
+        cp -aT /sysroot "${saved_sysroot}"
+        # also store the state of the partition
+        lsblk "${rootdisk}" --nodeps --json -b -o PATH,SIZE | jq -c . > "${partstate}"
+        ;;
+    restore)
+        # This one is in a private mount namespace since we're not "offically" mounting
+        mount "${rootdisk}" /sysroot
+        echo "Restoring rootfs from RAM..."
+        cd "${saved_sysroot}"
+        find . -mindepth 1 -maxdepth 1 -exec mv -t /sysroot {} \;
+        chattr +i $(ls -d /sysroot/ostree/deploy/*/deploy/*/)
+        ;;
+    *)
+        echo "Unsupported operation: ${1:-}" 1>&2; exit 1
+        ;;
+esac

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-firstboot-uuid

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -euo pipefail
+# https://github.com/coreos/fedora-coreos-tracker/issues/465
+# coreos-assembler generates disk images which are installed bit-for-bit
+# or booted directly in the cloud.
+# Generate new UUID on firstboot; this is general best practice, but in the future
+# we may use this for mounting by e.g. adding a boot=<uuid> and root=<uuid> kernel args.
+# Note this code should be changed when we land https://github.com/coreos/fedora-coreos-tracker/issues/94
+# so that we only generate a new UUID if we're using the default one.
+
+label=$1
+
+# Keep this in sync with https://github.com/coreos/coreos-assembler/blob/e3905fd2e138de04184c1cd86b99b0fd83cbe5cf/src/create_disk.sh#L17
+bootfs_uuid="96d15588-3596-4b3c-adca-a2ff7279ea63"
+rootfs_uuid="910678ff-f77e-4a7d-8d53-86f2ac47a823"
+
+target=/dev/disk/by-label/${label}
+if ! [ -b "${target}" ]; then
+  echo "$0: Failed to find block device ${target}" 1>&2
+  exit 1
+fi
+
+eval $(blkid -o export ${target})
+case "${label}" in
+  root) orig_uuid="${rootfs_uuid}"; orig_type=xfs ;;
+  boot) orig_uuid="${bootfs_uuid}"; orig_type=ext4 ;;
+  *) echo "unexpected ${label}"; exit 1 ;;
+esac
+
+if [ "${TYPE}" == "${orig_type}" ] && [ "${UUID}" == "${orig_uuid}" ]; then
+  case "${TYPE}" in
+    # For now we need to fsck first, see https://github.com/coreos/coreos-assembler/pull/1452
+    # Basically we're not passing `metadata_csum_seed` as a mkfs.ext4 option
+    # because grub2 barfs on it.
+    ext4) e2fsck -fy "${target}" && tune2fs -U random "${target}" ;;
+    xfs) xfs_admin -U generate "${target}" ;;
+    *) echo "unexpected filesystem type ${TYPE}" 1>&2; exit 1 ;;
+  esac
+  echo "Regenerated UUID for ${target}"
+else
+  echo "No changes required for ${target} TYPE=${TYPE} UUID=${UUID}"
+fi

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-sysroot.sh

@@ -1,20 +1,17 @@
 #!/bin/bash
 set -euo pipefail
 
-# We use prjquota on XFS by default to aid multi-tenant
-# Kubernetes (and other container) clusters.  See
-# https://github.com/coreos/coreos-assembler/pull/303/commits/6103effbd006bb6109467830d6a3e42dd847668d
-# In the future this will be augmented with a check for whether
-# or not we've reprovisioned the rootfs, since we don't want to
-# force on prjquota there.
+# Note that on *new machines* this script is now only ever used on firstboot. On
+# subsequent boots, systemd-fstab-generator mounts /sysroot from the
+# root=UUID=... and rootflags=... kargs.
+
+# We may do a migration window at some point where older machines have these
+# kargs injected so that we can simplify the model further.
+
 rootpath=/dev/disk/by-label/root
 if ! [ -b "${rootpath}" ]; then
   echo "ignition-ostree-mount-sysroot: Failed to find ${rootpath}" 1>&2
   exit 1
 fi
-eval $(blkid -o export ${rootpath})
-mountflags=
-if [ "${TYPE}" == "xfs" ]; then
-  mountflags=prjquota
-fi
-mount -o "${mountflags}" "${rootpath}" /sysroot
+
+mount -o "$(coreos-rootflags)" "${rootpath}" /sysroot

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-detect.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Ignition OSTree: detect rootfs replacement
+DefaultDependencies=false
+After=ignition-fetch.service
+Before=ignition-disks.service
+Before=initrd-root-fs.target
+Before=sysroot.mount
+ConditionKernelCommandLine=ostree
+
+# This stage requires udevd to detect disks
+Requires=systemd-udevd.service
+After=systemd-udevd.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=/run/ignition.env
+ExecStart=/usr/libexec/ignition-ostree-dracut-rootfs detect

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-restore.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Ignition OSTree: restore rootfs
+DefaultDependencies=false
+After=ignition-disks.service
+Before=ignition-ostree-growfs.service
+Before=ignition-ostree-mount-firstboot-sysroot.service
+
+ConditionKernelCommandLine=ostree
+ConditionPathIsDirectory=/run/ignition-ostree-rootfs
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=/run/ignition.env
+# So we can transiently mount sysroot
+MountFlags=slave
+ExecStart=/usr/libexec/ignition-ostree-dracut-rootfs restore

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-save.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Ignition OSTree: save rootfs
+DefaultDependencies=false
+After=ignition-ostree-rootfs-detect.service
+Before=ignition-disks.service
+ConditionKernelCommandLine=ostree
+ConditionPathIsDirectory=/run/ignition-ostree-rootfs
+# Any services looking at mounts need to order after this
+# because it causes device re-probing.
+After=coreos-gpt-setup.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=/run/ignition.env
+# So we can transiently mount sysroot
+MountFlags=slave
+ExecStart=/usr/libexec/ignition-ostree-dracut-rootfs save

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-boot.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=Ignition OSTree: Regenerate filesystem UUID (boot)
+DefaultDependencies=false
+ConditionPathExists=/usr/lib/initrd-release
+ConditionKernelCommandLine=ostree
+ConditionPathExists=!/run/ostree-live
+# We run pretty early
+Before=coreos-copy-firstboot-network.service
+Before=ignition-fetch.service
+Before=ignition-setup-base.service
+Before=ignition-setup-user.service
+# Any services looking at mounts need to order after this
+# because it causes device re-probing.
+After=coreos-gpt-setup.service
+
+Before=systemd-fsck@dev-disk-by\x2dlabel-boot.service
+Requires=dev-disk-by\x2dlabel-boot.device
+After=dev-disk-by\x2dlabel-boot.device
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/sbin/ignition-ostree-firstboot-uuid boot

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-root.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Ignition OSTree: Regenerate filesystem UUID (root)
+# These conditions match mount-firstboot-sysroot.service
+DefaultDependencies=false
+ConditionKernelCommandLine=!root
+ConditionKernelCommandLine=ostree
+ConditionPathExists=!/run/ostree-live
+Before=initrd-root-fs.target
+After=ignition-disks.service
+
+# Avoid racing with fsck
+Before=systemd-fsck@dev-disk-by\x2dlabel-boot.service
+
+# Note we don't have a Requires: /dev/disk/by-label/root here like
+# the -subsequent service does because ignition-disks may have
+# regenerated it.
+Before=ignition-ostree-mount-firstboot-sysroot.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/sbin/ignition-ostree-firstboot-uuid root

overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/module-setup.sh

@@ -3,7 +3,7 @@
 # ex: ts=8 sw=4 sts=4 et filetype=sh
 
 depends() {
-    echo ignition
+    echo ignition rdcore
 }
 
 install_ignition_unit() {
@@ -35,7 +35,9 @@ install() {
         realpath   \
         resize2fs  \
         tail       \
+        tune2fs    \
         touch      \
+        xfs_admin  \
         xfs_growfs \
         wipefs
 
@@ -53,7 +55,8 @@ install() {
         rm        \
         sed       \
         sfdisk    \
-        sgdisk
+        sgdisk    \
+        find
 
     for x in mount populate; do
         install_ignition_unit ignition-ostree-${x}-var.service
@@ -63,17 +66,29 @@ install() {
     inst_simple \
         /usr/lib/udev/rules.d/90-coreos-device-mapper.rules
 
-    inst_simple "$moddir/multipath-generator" \
-        "$systemdutildir/system-generators/multipath-generator"
+    inst_multiple jq chattr
+    inst_script "$moddir/ignition-ostree-dracut-rootfs.sh" "/usr/libexec/ignition-ostree-dracut-rootfs"
+    for x in detect save restore; do
+        install_ignition_unit ignition-ostree-rootfs-${x}.service
+    done
 
     # Disk support
     install_ignition_unit ignition-ostree-mount-firstboot-sysroot.service diskful
+    for p in boot root; do
+        install_ignition_unit ignition-ostree-uuid-${p}.service diskful
+    done
+    inst_script "$moddir/ignition-ostree-firstboot-uuid" \
+        "/usr/sbin/ignition-ostree-firstboot-uuid"
     install_ignition_unit ignition-ostree-mount-subsequent-sysroot.service diskful-subsequent
     inst_script "$moddir/ignition-ostree-mount-sysroot.sh" \
         "/usr/sbin/ignition-ostree-mount-sysroot"
+    inst_script "$moddir/coreos-rootflags.sh" \
+        "/usr/sbin/coreos-rootflags"
 
     install_ignition_unit ignition-ostree-growfs.service
     inst_script "$moddir/coreos-growpart" /usr/libexec/coreos-growpart
 
     inst_script "$moddir/coreos-relabel" /usr/bin/coreos-relabel
+
+    install_ignition_unit coreos-inject-rootmap.service diskful
 }

tests/kola/root-reprovision/filesystem-only/config.ign

@@ -0,0 +1,15 @@
+{
+    "ignition": {
+        "version": "3.0.0"
+    },
+    "storage": {
+        "filesystems": [
+            {
+                "device": "/dev/disk/by-label/root",
+                "wipeFilesystem": true,
+                "format": "ext4",
+                "label": "root"
+            }
+        ]
+    }
+}