diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.service b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.service index 2fd6151b01..fb97ca73ac 100644 --- a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.service +++ b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-network/coreos-copy-firstboot-network.service @@ -36,10 +36,6 @@ After=dracut-cmdline.service # Any services looking at mounts need to order after this # because it causes device re-probing. After=coreos-gpt-setup.service -# Older unit name kept for compatibility. This will be safe to remove once the -# dracut-ignition changes from [1] are available in the latest ignition RPM. -# [1] https://github.com/coreos/ignition-dracut/pull/191 -After=coreos-gpt-setup@dev-disk-by\x2dlabel-root.service # Since we are mounting /boot/, require the device first Requires=dev-disk-by\x2dlabel-boot.device After=dev-disk-by\x2dlabel-boot.device diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.service b/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.service index f5a75b9b8a..6d873665fb 100644 --- a/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.service +++ b/overlay.d/05core/usr/lib/dracut/modules.d/30ignition-coreos/coreos-gpt-setup.service @@ -7,7 +7,7 @@ Before=ignition-diskful.target Wants=systemd-udevd.service After=systemd-udevd.service -# This unit must the first to run when the disk holding the root partition +# This unit must be the first to run when the disk holding the root partition # becomes available. To avoid relying on the name of the root partition which # is different between RHCOS LUKS setup and current FCOS setup, we wait for the # partition labeled 'boot' to become available. This is reliable as we don't diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-growpart b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-growpart index f13fd2d69d..fc16033a7c 100755 --- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-growpart +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-growpart @@ -8,6 +8,8 @@ set -euo pipefail # Growpart /, until we can fix Ignition for separate /var # (And eventually we want ignition-disks) +saved_partstate=/run/ignition-ostree-rootfs-partstate.json + path=$1 shift @@ -15,6 +17,24 @@ shift # this shouldn't happen for us but we're being conservative. src=$(findmnt -nvr -o SOURCE "$path" | tail -n1) +if [ "${path}" == /sysroot ] && [ -f "${saved_partstate}" ]; then + # We're still ironing out our rootfs automatic growpart story, see e.g.: + # https://github.com/coreos/fedora-coreos-tracker/issues/570 + # https://github.com/coreos/fedora-coreos-tracker/issues/586 + # + # In the context of rootfs reprovisioning, for now our rule is the + # following: if the rootfs partition was moved off of the boot disk or it + # was resized, then we don't growpart. + # + # To detect this, we compare the output of `lsblk -o PATH,SIZE` before and + # after `ignition-disks.service`. + partstate=$(lsblk "${src}" --nodeps --json -b -o PATH,SIZE | jq -c .) + if [ "${partstate}" != "$(cat "${saved_partstate}")" ]; then + echo "coreos-growpart: detected rootfs partition changes; not auto-growing" + exit 0 + fi +fi + # Get the filesystem type before extending the partition. This matters # because the partition, once extended, might include leftover superblocks # from the previous contents of the disk (notably ZFS), causing blkid to @@ -67,4 +87,5 @@ case "${TYPE}" in ext4) resize2fs "${src}" ;; esac -touch /var/lib/coreos-growpart.stamp +# this is useful for tests +touch /run/coreos-growpart.stamp diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-inject-rootmap.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-inject-rootmap.service new file mode 100644 index 0000000000..101a3c727c --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-inject-rootmap.service @@ -0,0 +1,14 @@ +[Unit] +Description=CoreOS Inject Rootmap +# If root is specified, assume rootmap is already configured +ConditionKernelCommandLine=!root +OnFailure=emergency.target +OnFailureJobMode=isolate + +After=ignition-files.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/rdcore rootmap /sysroot --boot-device /dev/disk/by-label/boot +RemainAfterExit=yes +MountFlags=slave diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-rootflags.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-rootflags.sh new file mode 100755 index 0000000000..3cfe03576f --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/coreos-rootflags.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail + +rootpath=/dev/disk/by-label/root + +# If the rootfs was reprovisioned, then the user is free to define their own +# rootflags. +if [ -d /run/ignition-ostree-rootfs ]; then + exit 0 +fi + +eval $(blkid -o export ${rootpath}) +# this really should always be true, but let's be conservative +if [ "${TYPE}" == "xfs" ]; then + # We use prjquota on XFS by default to aid multi-tenant Kubernetes (and + # other container) clusters. See + # https://github.com/coreos/coreos-assembler/pull/303/commits/6103effbd006bb6109467830d6a3e42dd847668d + echo "prjquota" +fi diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-dracut-rootfs.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-dracut-rootfs.sh new file mode 100755 index 0000000000..b04640deeb --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-dracut-rootfs.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -euo pipefail + +# This is implementation details of Ignition; in the future, we should figure +# out a way to ask Ignition directly whether there's a filesystem with label +# "root" being set up. +ignition_cfg=/run/ignition.json +rootdisk=/dev/disk/by-label/root +saved_sysroot=/run/ignition-ostree-rootfs +partstate=/run/ignition-ostree-rootfs-partstate.json + +case "${1:-}" in + detect) + wipes_root=$(jq '.storage?.filesystems? // [] | map(select(.label == "root" and .wipeFilesystem == true)) | length' "${ignition_cfg}") + if [ "${wipes_root}" = "0" ]; then + exit 0 + fi + echo "Detected rootfs replacement in fetched Ignition config: /run/ignition.json" + mkdir "${saved_sysroot}" + ;; + save) + mount "${rootdisk}" /sysroot + echo "Moving rootfs to RAM..." + cp -aT /sysroot "${saved_sysroot}" + # also store the state of the partition + lsblk "${rootdisk}" --nodeps --json -b -o PATH,SIZE | jq -c . > "${partstate}" + ;; + restore) + # This one is in a private mount namespace since we're not "offically" mounting + mount "${rootdisk}" /sysroot + echo "Restoring rootfs from RAM..." + cd "${saved_sysroot}" + find . -mindepth 1 -maxdepth 1 -exec mv -t /sysroot {} \; + chattr +i $(ls -d /sysroot/ostree/deploy/*/deploy/*/) + ;; + *) + echo "Unsupported operation: ${1:-}" 1>&2; exit 1 + ;; +esac diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-firstboot-uuid b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-firstboot-uuid new file mode 100755 index 0000000000..7aef29a171 --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-firstboot-uuid @@ -0,0 +1,42 @@ +#!/bin/bash +set -euo pipefail +# https://github.com/coreos/fedora-coreos-tracker/issues/465 +# coreos-assembler generates disk images which are installed bit-for-bit +# or booted directly in the cloud. +# Generate new UUID on firstboot; this is general best practice, but in the future +# we may use this for mounting by e.g. adding a boot= and root= kernel args. +# Note this code should be changed when we land https://github.com/coreos/fedora-coreos-tracker/issues/94 +# so that we only generate a new UUID if we're using the default one. + +label=$1 + +# Keep this in sync with https://github.com/coreos/coreos-assembler/blob/e3905fd2e138de04184c1cd86b99b0fd83cbe5cf/src/create_disk.sh#L17 +bootfs_uuid="96d15588-3596-4b3c-adca-a2ff7279ea63" +rootfs_uuid="910678ff-f77e-4a7d-8d53-86f2ac47a823" + +target=/dev/disk/by-label/${label} +if ! [ -b "${target}" ]; then + echo "$0: Failed to find block device ${target}" 1>&2 + exit 1 +fi + +eval $(blkid -o export ${target}) +case "${label}" in + root) orig_uuid="${rootfs_uuid}"; orig_type=xfs ;; + boot) orig_uuid="${bootfs_uuid}"; orig_type=ext4 ;; + *) echo "unexpected ${label}"; exit 1 ;; +esac + +if [ "${TYPE}" == "${orig_type}" ] && [ "${UUID}" == "${orig_uuid}" ]; then + case "${TYPE}" in + # For now we need to fsck first, see https://github.com/coreos/coreos-assembler/pull/1452 + # Basically we're not passing `metadata_csum_seed` as a mkfs.ext4 option + # because grub2 barfs on it. + ext4) e2fsck -fy "${target}" && tune2fs -U random "${target}" ;; + xfs) xfs_admin -U generate "${target}" ;; + *) echo "unexpected filesystem type ${TYPE}" 1>&2; exit 1 ;; + esac + echo "Regenerated UUID for ${target}" +else + echo "No changes required for ${target} TYPE=${TYPE} UUID=${UUID}" +fi diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-sysroot.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-sysroot.sh index 047ba2da37..c981d0db45 100755 --- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-sysroot.sh +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-mount-sysroot.sh @@ -1,20 +1,17 @@ #!/bin/bash set -euo pipefail -# We use prjquota on XFS by default to aid multi-tenant -# Kubernetes (and other container) clusters. See -# https://github.com/coreos/coreos-assembler/pull/303/commits/6103effbd006bb6109467830d6a3e42dd847668d -# In the future this will be augmented with a check for whether -# or not we've reprovisioned the rootfs, since we don't want to -# force on prjquota there. +# Note that on *new machines* this script is now only ever used on firstboot. On +# subsequent boots, systemd-fstab-generator mounts /sysroot from the +# root=UUID=... and rootflags=... kargs. + +# We may do a migration window at some point where older machines have these +# kargs injected so that we can simplify the model further. + rootpath=/dev/disk/by-label/root if ! [ -b "${rootpath}" ]; then echo "ignition-ostree-mount-sysroot: Failed to find ${rootpath}" 1>&2 exit 1 fi -eval $(blkid -o export ${rootpath}) -mountflags= -if [ "${TYPE}" == "xfs" ]; then - mountflags=prjquota -fi -mount -o "${mountflags}" "${rootpath}" /sysroot + +mount -o "$(coreos-rootflags)" "${rootpath}" /sysroot diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-detect.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-detect.service new file mode 100644 index 0000000000..5a969a7edc --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-detect.service @@ -0,0 +1,18 @@ +[Unit] +Description=Ignition OSTree: detect rootfs replacement +DefaultDependencies=false +After=ignition-fetch.service +Before=ignition-disks.service +Before=initrd-root-fs.target +Before=sysroot.mount +ConditionKernelCommandLine=ostree + +# This stage requires udevd to detect disks +Requires=systemd-udevd.service +After=systemd-udevd.service + +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=/run/ignition.env +ExecStart=/usr/libexec/ignition-ostree-dracut-rootfs detect diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-restore.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-restore.service new file mode 100644 index 0000000000..f02d729325 --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-restore.service @@ -0,0 +1,17 @@ +[Unit] +Description=Ignition OSTree: restore rootfs +DefaultDependencies=false +After=ignition-disks.service +Before=ignition-ostree-growfs.service +Before=ignition-ostree-mount-firstboot-sysroot.service + +ConditionKernelCommandLine=ostree +ConditionPathIsDirectory=/run/ignition-ostree-rootfs + +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=/run/ignition.env +# So we can transiently mount sysroot +MountFlags=slave +ExecStart=/usr/libexec/ignition-ostree-dracut-rootfs restore diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-save.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-save.service new file mode 100644 index 0000000000..69142df5fb --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-rootfs-save.service @@ -0,0 +1,18 @@ +[Unit] +Description=Ignition OSTree: save rootfs +DefaultDependencies=false +After=ignition-ostree-rootfs-detect.service +Before=ignition-disks.service +ConditionKernelCommandLine=ostree +ConditionPathIsDirectory=/run/ignition-ostree-rootfs +# Any services looking at mounts need to order after this +# because it causes device re-probing. +After=coreos-gpt-setup.service + +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=/run/ignition.env +# So we can transiently mount sysroot +MountFlags=slave +ExecStart=/usr/libexec/ignition-ostree-dracut-rootfs save diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-boot.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-boot.service new file mode 100644 index 0000000000..bad9ece0ce --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-boot.service @@ -0,0 +1,23 @@ +[Unit] +Description=Ignition OSTree: Regenerate filesystem UUID (boot) +DefaultDependencies=false +ConditionPathExists=/usr/lib/initrd-release +ConditionKernelCommandLine=ostree +ConditionPathExists=!/run/ostree-live +# We run pretty early +Before=coreos-copy-firstboot-network.service +Before=ignition-fetch.service +Before=ignition-setup-base.service +Before=ignition-setup-user.service +# Any services looking at mounts need to order after this +# because it causes device re-probing. +After=coreos-gpt-setup.service + +Before=systemd-fsck@dev-disk-by\x2dlabel-boot.service +Requires=dev-disk-by\x2dlabel-boot.device +After=dev-disk-by\x2dlabel-boot.device + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/ignition-ostree-firstboot-uuid boot diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-root.service b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-root.service new file mode 100644 index 0000000000..7f005e1f27 --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/ignition-ostree-uuid-root.service @@ -0,0 +1,22 @@ +[Unit] +Description=Ignition OSTree: Regenerate filesystem UUID (root) +# These conditions match mount-firstboot-sysroot.service +DefaultDependencies=false +ConditionKernelCommandLine=!root +ConditionKernelCommandLine=ostree +ConditionPathExists=!/run/ostree-live +Before=initrd-root-fs.target +After=ignition-disks.service + +# Avoid racing with fsck +Before=systemd-fsck@dev-disk-by\x2dlabel-boot.service + +# Note we don't have a Requires: /dev/disk/by-label/root here like +# the -subsequent service does because ignition-disks may have +# regenerated it. +Before=ignition-ostree-mount-firstboot-sysroot.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/sbin/ignition-ostree-firstboot-uuid root diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/module-setup.sh index 9873f4c587..09eb7f2153 100755 --- a/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/module-setup.sh +++ b/overlay.d/05core/usr/lib/dracut/modules.d/40ignition-ostree/module-setup.sh @@ -3,7 +3,7 @@ # ex: ts=8 sw=4 sts=4 et filetype=sh depends() { - echo ignition + echo ignition rdcore } install_ignition_unit() { @@ -35,7 +35,9 @@ install() { realpath \ resize2fs \ tail \ + tune2fs \ touch \ + xfs_admin \ xfs_growfs \ wipefs @@ -53,7 +55,8 @@ install() { rm \ sed \ sfdisk \ - sgdisk + sgdisk \ + find for x in mount populate; do install_ignition_unit ignition-ostree-${x}-var.service @@ -63,17 +66,29 @@ install() { inst_simple \ /usr/lib/udev/rules.d/90-coreos-device-mapper.rules - inst_simple "$moddir/multipath-generator" \ - "$systemdutildir/system-generators/multipath-generator" + inst_multiple jq chattr + inst_script "$moddir/ignition-ostree-dracut-rootfs.sh" "/usr/libexec/ignition-ostree-dracut-rootfs" + for x in detect save restore; do + install_ignition_unit ignition-ostree-rootfs-${x}.service + done # Disk support install_ignition_unit ignition-ostree-mount-firstboot-sysroot.service diskful + for p in boot root; do + install_ignition_unit ignition-ostree-uuid-${p}.service diskful + done + inst_script "$moddir/ignition-ostree-firstboot-uuid" \ + "/usr/sbin/ignition-ostree-firstboot-uuid" install_ignition_unit ignition-ostree-mount-subsequent-sysroot.service diskful-subsequent inst_script "$moddir/ignition-ostree-mount-sysroot.sh" \ "/usr/sbin/ignition-ostree-mount-sysroot" + inst_script "$moddir/coreos-rootflags.sh" \ + "/usr/sbin/coreos-rootflags" install_ignition_unit ignition-ostree-growfs.service inst_script "$moddir/coreos-growpart" /usr/libexec/coreos-growpart inst_script "$moddir/coreos-relabel" /usr/bin/coreos-relabel + + install_ignition_unit coreos-inject-rootmap.service diskful } diff --git a/tests/kola/root-reprovision/filesystem-only/config.ign b/tests/kola/root-reprovision/filesystem-only/config.ign new file mode 100644 index 0000000000..3784c8cbfe --- /dev/null +++ b/tests/kola/root-reprovision/filesystem-only/config.ign @@ -0,0 +1,15 @@ +{ + "ignition": { + "version": "3.0.0" + }, + "storage": { + "filesystems": [ + { + "device": "/dev/disk/by-label/root", + "wipeFilesystem": true, + "format": "ext4", + "label": "root" + } + ] + } +} diff --git a/tests/kola/root-reprovision/filesystem-only/test.sh b/tests/kola/root-reprovision/filesystem-only/test.sh new file mode 100755 index 0000000000..10b2faf8b8 --- /dev/null +++ b/tests/kola/root-reprovision/filesystem-only/test.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# kola: {"platforms": "qemu", "minMemory": 4096} +set -xeuo pipefail + +fstype=$(findmnt -nvr / -o FSTYPE) +[[ $fstype == ext4 ]] + +case "${AUTOPKGTEST_REBOOT_MARK:-}" in + "") + # check that the partition was grown + if [ ! -e /run/coreos-growpart.stamp ]; then + echo "coreos-growpart did not run" + exit 1 + fi + + # reboot once to sanity-check we can find root on second boot + /tmp/autopkgtest-reboot rebooted + ;; + + rebooted) + grep root=UUID= /proc/cmdline + ;; + *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;; +esac diff --git a/tests/kola/root-reprovision/raid1/config.ign b/tests/kola/root-reprovision/raid1/config.ign new file mode 100644 index 0000000000..5eca597dc4 --- /dev/null +++ b/tests/kola/root-reprovision/raid1/config.ign @@ -0,0 +1,25 @@ +{ + "ignition": { + "version": "3.0.0" + }, + "storage": { + "raid": [ + { + "devices": [ + "/dev/disk/by-id/virtio-disk1", + "/dev/disk/by-id/virtio-disk2" + ], + "level": "raid1", + "name": "foobar" + } + ], + "filesystems": [ + { + "device": "/dev/md/foobar", + "format": "xfs", + "wipeFilesystem": true, + "label": "root" + } + ] + } +} diff --git a/tests/kola/root-reprovision/raid1/test.sh b/tests/kola/root-reprovision/raid1/test.sh new file mode 100755 index 0000000000..7ecc340e60 --- /dev/null +++ b/tests/kola/root-reprovision/raid1/test.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# kola: {"platforms": "qemu", "minMemory": 4096, "additionalDisks": ["5G", "5G"]} +set -xeuo pipefail + +srcdev=$(findmnt -nvr / -o SOURCE) +[[ ${srcdev} == $(realpath /dev/md/foobar) ]] + +blktype=$(lsblk -o TYPE "${srcdev}" --noheadings) +[[ ${blktype} == raid1 ]] + +fstype=$(findmnt -nvr / -o FSTYPE) +[[ ${fstype} == xfs ]] + +case "${AUTOPKGTEST_REBOOT_MARK:-}" in + "") + # check that growpart didn't run + if [ -e /run/coreos-growpart.stamp ]; then + echo "coreos-growpart ran" + exit 1 + fi + + # reboot once to sanity-check we can find root on second boot + /tmp/autopkgtest-reboot rebooted + ;; + + rebooted) + grep root=UUID= /proc/cmdline + grep rd.md.uuid= /proc/cmdline + ;; + *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;; +esac