From 1d43a12a3ad35a02ec8fce313d65cc681232d3b0 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Tue, 14 Apr 2020 15:44:24 -0400 Subject: [PATCH 1/3] overlay: add 15coreos-copy-firstboot-network dracut module This dracut module delivers a coreos-copy-firstboot-network systemd service and script that will detect when files have been placed into /boot/ by `coreos-installer install --copy-network` and appropriately copy them in place to be used by the initramfs networking. If files are detected within /boot/coreos-firstboot-network/ then they will be considered to be the only source of networking for that ignition boot (i.e. no networking kargs will be applied). --- .../coreos-copy-firstboot-network.service | 45 +++++++++++++++++++ .../coreos-copy-firstboot-network.sh | 28 ++++++++++++ .../module-setup.sh | 16 +++++++ 3 files changed, 89 insertions(+) create mode 100644 overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.service create mode 100755 overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh create mode 100644 overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/module-setup.sh diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.service b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.service new file mode 100644 index 0000000000..6e2bf3767b --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.service @@ -0,0 +1,45 @@ +# This unit will run early in boot and detect if the user copied +# in firstboot networking config files into the installed system +# (most likely by using `coreos-installer install --copy-network`). +# Since this unit is modifying network configuration there are some +# dependencies that we have: +# +# - Need to look for networking configuration on the /boot partition +# - i.e. after /dev/disk/by-label/boot is available +# - Need to run before networking is brought up. +# - This is done in nm-run.sh [1] that runs as part of dracut-initqueue [2] +# - i.e. Before=dracut-initqueue.service +# - Need to make sure karg networking configuration isn't applied +# - There are two ways to do this. +# - One is to run *before* the nm-config.sh [3] that runs as part of +# dracut-cmdline [4] and `ln -sf /bin/true /usr/libexec/nm-initrd-generator`. +# - i.e. Before=dracut-cmdline.service +# - Another is to run *after* nm-config.sh [3] in dracut-cmdline [4] +# and just delete all the files created by nm-initrd-generator. +# - i.e. After=dracut-cmdline.service, but Before=dracut-initqueue.service +# - We'll go with the second option here because the need for the /boot +# device (mentioned above) means we can't start before dracut-cmdline.service +# +# [1] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/nm-run.sh +# [2] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/module-setup.sh#L37 +# [3] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/nm-config.sh +# [4] https://github.com/dracutdevs/dracut/blob/master/modules.d/35network-manager/module-setup.sh#L36 +# +[Unit] +Description=Copy CoreOS Firstboot Networking Config +ConditionPathExists=/usr/lib/initrd-release +DefaultDependencies=false +Before=ignition-diskful.target +Before=dracut-initqueue.service +After=dracut-cmdline.service +# Since we are mounting /boot/, require the device first +Requires=dev-disk-by\x2dlabel-boot.device +After=dev-disk-by\x2dlabel-boot.device + +[Service] +Type=oneshot +RemainAfterExit=yes +# The MountFlags=slave is so the umount of /boot is guaranteed to happen +# /boot will only be mounted for the lifetime of the unit. +MountFlags=slave +ExecStart=/usr/sbin/coreos-copy-firstboot-network diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh new file mode 100755 index 0000000000..84fec980f9 --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -euo pipefail + +# For a description of how this is used see coreos-copy-firstboot-network.service + +bootmnt=/mnt/boot_partition +firstboot_network_dir="${bootmnt}/coreos-firstboot-network/" +initramfs_network_dir="/run/NetworkManager/system-connections/" + +# Mount /boot. Note that we mount /boot but we don't unmount boot because we +# are run in a systemd unit with MountFlags=slave so it is unmounted for us. +mkdir -p ${bootmnt} +# mount as read-only since we don't strictly need write access and we may be +# running alongside other code that also has it mounted ro +mount -o ro /dev/disk/by-label/boot ${bootmnt} + +if [ -n "$(ls -A ${firstboot_network_dir} 2>/dev/null)" ]; then + # Clear out any files that may have already been generated from + # kargs by nm-initrd-generator + rm -f ${initramfs_network_dir}/* + # Copy files that were placed into boot (most likely by coreos-installer) + # to the appropriate location for NetworkManager to use the configuration. + echo "info: copying files from ${firstboot_network_dir} to ${initramfs_network_dir}" + mkdir -p ${initramfs_network_dir} + cp -v ${firstboot_network_dir}/* ${initramfs_network_dir}/ +else + echo "info: no files to copy from ${firstboot_network_dir}. skipping" +fi diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/module-setup.sh b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/module-setup.sh new file mode 100644 index 0000000000..0933554144 --- /dev/null +++ b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/module-setup.sh @@ -0,0 +1,16 @@ +install_and_enable_unit() { + unit="$1"; shift + target="$1"; shift + inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit" + mkdir -p "$initdir/$systemdsystemunitdir/$target.requires" + ln_r "../$unit" "$systemdsystemunitdir/$target.requires/$unit" +} + +install() { + inst_simple "$moddir/coreos-copy-firstboot-network.sh" \ + "/usr/sbin/coreos-copy-firstboot-network" + # Only run this when ignition runs and only when the system + # has disks. ignition-diskful.target should suffice. + install_and_enable_unit "coreos-copy-firstboot-network.service" \ + "ignition-diskful.target" +} From f9afb84388b39feb7940e915c4379cd293f31c41 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 16 Apr 2020 12:45:46 -0400 Subject: [PATCH 2/3] overlay: adding tmpfiles.d entry for removal of /boot/coreos-firstboot-network This will clean up those files if they existed and were used. --- .../coreos-copy-firstboot-network.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh index 84fec980f9..32af9d7487 100755 --- a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh +++ b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh @@ -4,8 +4,10 @@ set -euo pipefail # For a description of how this is used see coreos-copy-firstboot-network.service bootmnt=/mnt/boot_partition -firstboot_network_dir="${bootmnt}/coreos-firstboot-network/" +firstboot_network_dir_basename="coreos-firstboot-network" +initramfs_firstboot_network_dir="${bootmnt}/${firstboot_network_dir_basename}" initramfs_network_dir="/run/NetworkManager/system-connections/" +realroot_firstboot_network_dir="/boot/${firstboot_network_dir_basename}" # Mount /boot. Note that we mount /boot but we don't unmount boot because we # are run in a systemd unit with MountFlags=slave so it is unmounted for us. @@ -14,15 +16,19 @@ mkdir -p ${bootmnt} # running alongside other code that also has it mounted ro mount -o ro /dev/disk/by-label/boot ${bootmnt} -if [ -n "$(ls -A ${firstboot_network_dir} 2>/dev/null)" ]; then +if [ -n "$(ls -A ${initramfs_firstboot_network_dir} 2>/dev/null)" ]; then # Clear out any files that may have already been generated from # kargs by nm-initrd-generator rm -f ${initramfs_network_dir}/* # Copy files that were placed into boot (most likely by coreos-installer) # to the appropriate location for NetworkManager to use the configuration. - echo "info: copying files from ${firstboot_network_dir} to ${initramfs_network_dir}" + echo "info: copying files from ${initramfs_firstboot_network_dir} to ${initramfs_network_dir}" mkdir -p ${initramfs_network_dir} - cp -v ${firstboot_network_dir}/* ${initramfs_network_dir}/ + cp -v ${initramfs_firstboot_network_dir}/* ${initramfs_network_dir}/ + # If we make it to the realroot (successfully ran ignition) then + # clean up the files in the firstboot network dir + echo "R ${realroot_firstboot_network_dir} - - - - -" > \ + /run/tmpfiles.d/15-coreos-firstboot-network.conf else - echo "info: no files to copy from ${firstboot_network_dir}. skipping" + echo "info: no files to copy from ${initramfs_firstboot_network_dir}. skipping" fi From ba00a21e54db283c9fd745c5c7e3dcb68bb8256f Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Thu, 16 Apr 2020 17:02:55 -0400 Subject: [PATCH 3/3] overlay: 15coreos-copy-firstboot-network: handle race condition Add short while loop to handle a race condition where After=dev-disk-by\x2dlabel-boot.device doesn't seem to be sufficient always. --- .../coreos-copy-firstboot-network.sh | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh index 32af9d7487..38f35454bc 100755 --- a/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh +++ b/overlay.d/05core/usr/lib/dracut/modules.d/15coreos-copy-firstboot-network/coreos-copy-firstboot-network.sh @@ -4,6 +4,8 @@ set -euo pipefail # For a description of how this is used see coreos-copy-firstboot-network.service bootmnt=/mnt/boot_partition +mkdir -p ${bootmnt} +bootdev=/dev/disk/by-label/boot firstboot_network_dir_basename="coreos-firstboot-network" initramfs_firstboot_network_dir="${bootmnt}/${firstboot_network_dir_basename}" initramfs_network_dir="/run/NetworkManager/system-connections/" @@ -11,10 +13,40 @@ realroot_firstboot_network_dir="/boot/${firstboot_network_dir_basename}" # Mount /boot. Note that we mount /boot but we don't unmount boot because we # are run in a systemd unit with MountFlags=slave so it is unmounted for us. -mkdir -p ${bootmnt} -# mount as read-only since we don't strictly need write access and we may be +# Mount as read-only since we don't strictly need write access and we may be # running alongside other code that also has it mounted ro -mount -o ro /dev/disk/by-label/boot ${bootmnt} +mountboot() { + # Wait for up to 5 seconds for the boot device to be available + # The After=...*boot.device in the systemd unit should be enough + # but there appears to be some race in the kernel where the link under + # /dev/disk/by-label exists but mount is not able to use the device yet. + # We saw errors like this in CI: + # + # [ 4.045181] systemd[1]: Found device /dev/disk/by-label/boot. + # [ OK ] Found device /dev/disk/by-label/boot + # [ 4.051500] systemd[1]: Starting Copy CoreOS Firstboot Networking Config... + # Starting Copy CoreOS Firstboot Networking Config + # [ 4.060573] vda: vda1 vda2 vda3 vda4 + # [ 4.063296] coreos-copy-firstboot-network[479]: mount: /mnt/boot_partition: special device /dev/disk/by-label/boot does not exist. + # + mounted=0 + for x in {1..5}; do + if mount -o ro ${bootdev} ${bootmnt}; then + echo "info: ${bootdev} successfully mounted." + mounted=1 + break + else + echo "info: retrying ${bootdev} mount in 1 second..." + sleep 1 + fi + done + if [ "${mounted}" == "0" ]; then + echo "error: ${bootdev} mount did not succeed" 1>&2 + return 1 + fi +} + +mountboot || exit 1 if [ -n "$(ls -A ${initramfs_firstboot_network_dir} 2>/dev/null)" ]; then # Clear out any files that may have already been generated from