Skip to content

Commit

Permalink
feat: split container build into distinct build script stages (#59)
Browse files Browse the repository at this point in the history
  • Loading branch information
joshua-stone committed Mar 7, 2023
1 parent 301b353 commit ca99377
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 86 deletions.
95 changes: 9 additions & 86 deletions Containerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,70 +6,24 @@ FROM ${BASE_IMAGE}:${FEDORA_MAJOR_VERSION} AS builder

ARG NVIDIA_MAJOR_VERSION="${NVIDIA_MAJOR_VERSION:-525}"

RUN sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/fedora-{cisco-openh264,modular,updates-modular}.repo

# nvidia 520.xxx and newer currently don't have a -$VERSIONxx suffix in their
# package names
RUN if [ "${NVIDIA_MAJOR_VERSION}" -ge 520 ]; then echo "nvidia"; else echo "nvidia-${NVIDIA_MAJOR_VERSION}xx"; fi > /tmp/nvidia-package-name.txt

RUN rpm-ostree install \
akmods mock \
xorg-x11-drv-$(cat /tmp/nvidia-package-name.txt)-{,cuda,devel,kmodsrc,power}*:${NVIDIA_MAJOR_VERSION}.*.fc$(rpm -E '%fedora.%_arch')

COPY --from=ghcr.io/ublue-os/config:latest /build /tmp/build
COPY justfile /tmp/build/ublue-os-just/justfile
RUN /tmp/build/ublue-os-just/build.sh


# alternatives cannot create symlinks on its own during a container build
RUN ln -s /usr/bin/ld.bfd /etc/alternatives/ld && ln -s /etc/alternatives/ld /usr/bin/ld
COPY build.sh /tmp/build.sh

ADD certs /tmp/certs

RUN [[ -s "/tmp/certs/private_key.priv" ]] || \
echo "WARNING: Using test signing key. Run './generate-akmods-key' for production builds." && \
cp /tmp/certs/private_key.priv{.test,} && \
cp /tmp/certs/public_key.der{.test,}

RUN install -Dm644 /tmp/certs/public_key.der /etc/pki/akmods/certs/public_key.der
RUN install -Dm644 /tmp/certs/private_key.priv /etc/pki/akmods/private/private_key.priv

# Either successfully build and install the kernel modules, or fail early with debug output
RUN NVIDIA_PACKAGE_NAME="$(cat /tmp/nvidia-package-name.txt)" \
KERNEL_VERSION="$(rpm -q kernel --queryformat '%{VERSION}-%{RELEASE}.%{ARCH}')" \
NVIDIA_VERSION="$(basename "$(rpm -q "xorg-x11-drv-$(cat /tmp/nvidia-package-name.txt)" --queryformat '%{VERSION}-%{RELEASE}')" ".fc$(rpm -E '%fedora')")" \
&& \
akmods --force --kernels "${KERNEL_VERSION}" --kmod "${NVIDIA_PACKAGE_NAME}" \
&& \
modinfo /usr/lib/modules/${KERNEL_VERSION}/extra/${NVIDIA_PACKAGE_NAME}/nvidia{,-drm,-modeset,-peermem,-uvm}.ko.xz > /dev/null \
|| \
(cat /var/cache/akmods/${NVIDIA_PACKAGE_NAME}/${NVIDIA_VERSION}-for-${KERNEL_VERSION}.failed.log && exit 1)

ADD ublue-os-nvidia-addons.spec /tmp/ublue-os-nvidia-addons/ublue-os-nvidia-addons.spec

ADD https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo \
/tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo

RUN sed -i "s@gpgcheck=0@gpgcheck=1@" /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo

ADD files/etc/nvidia-container-runtime/config-rootless.toml \
/tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/config-rootless.toml
ADD https://github.com/raw/NVIDIA/dgx-selinux/master/bin/RHEL9/nvidia-container.pp \
/tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container.pp
ADD files/etc/sway/environment /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/environment

RUN install -D /etc/pki/akmods/certs/public_key.der /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/public_key.der

RUN rpmbuild -ba \
--define '_topdir /tmp/ublue-os-nvidia-addons/rpmbuild' \
--define '%_tmppath %{_topdir}/tmp' \
/tmp/ublue-os-nvidia-addons/ublue-os-nvidia-addons.spec


RUN cp /tmp/nvidia-package-name.txt /var/cache/akmods/nvidia-package-name.txt
RUN echo "${NVIDIA_MAJOR_VERSION}" > /var/cache/akmods/nvidia-major-version.txt
RUN rpm -q "xorg-x11-drv-$(cat /tmp/nvidia-package-name.txt)" \
--queryformat '%{EPOCH}:%{VERSION}-%{RELEASE}.%{ARCH}' > /var/cache/akmods/nvidia-full-version.txt
RUN /tmp/build.sh

FROM ${BASE_IMAGE}:${FEDORA_MAJOR_VERSION}

Expand All @@ -79,41 +33,10 @@ COPY --from=builder /tmp/ublue-os /tmp/ublue-os
COPY --from=builder /var/cache/akmods /tmp/akmods
COPY --from=builder /tmp/ublue-os-nvidia-addons /tmp/ublue-os-nvidia-addons

RUN sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/fedora-{cisco-openh264,modular,updates-modular}.repo

RUN install -D /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo \
/etc/yum.repos.d/nvidia-container-runtime.repo

RUN KERNEL_VERSION="$(rpm -q kernel --queryformat '%{VERSION}-%{RELEASE}.%{ARCH}')" \
NVIDIA_FULL_VERSION="$(cat /tmp/akmods/nvidia-full-version.txt)" \
NVIDIA_PACKAGE_NAME="$(cat /tmp/akmods/nvidia-package-name.txt)" \
&& \
rpm-ostree install \
xorg-x11-drv-${NVIDIA_PACKAGE_NAME}-{,cuda-,devel-,kmodsrc-,power-}${NVIDIA_FULL_VERSION} \
nvidia-container-toolkit nvidia-vaapi-driver \
"/tmp/akmods/${NVIDIA_PACKAGE_NAME}/kmod-${NVIDIA_PACKAGE_NAME}-${KERNEL_VERSION}-${NVIDIA_FULL_VERSION#*:}.rpm" \
/tmp/ublue-os-nvidia-addons/rpmbuild/RPMS/noarch/ublue-os-nvidia-addons-*.rpm \
/tmp/ublue-os/rpmbuild/RPMS/noarch/ublue-os-just-*.noarch.rpm \
&& \
mv /etc/nvidia-container-runtime/config.toml{,.orig} && \
cp /etc/nvidia-container-runtime/config{-rootless,}.toml \
&& \
semodule --verbose --install /usr/share/selinux/packages/nvidia-container.pp \
&& \
sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/rpmfusion-{,non}free{,-updates}.repo \
&& \
ln -s /usr/bin/ld.bfd /etc/alternatives/ld && \
ln -s /etc/alternatives/ld /usr/bin/ld \
&& \
([[ "${IMAGE_NAME}" == "sericea" ]] && \
mv /etc/sway/environment{,.orig} && \
install -Dm644 /usr/share/ublue-os/etc/sway/environment /etc/sway/environment) ||: \
&& \
rm -rf \
/tmp/* \
/var/* \
&& \
ostree container commit \
&& \
mkdir -p /var/tmp && \
chmod -R 1777 /var/tmp
COPY install.sh /tmp/install.sh
COPY post-install.sh /tmp/post-install.sh
RUN /tmp/install.sh
RUN /tmp/post-install.sh
RUN rm -rf /tmp/* /var/*
RUN ostree container commit
RUN mkdir -p /var/tmp && chmod -R 1777 /var/tmp
64 changes: 64 additions & 0 deletions build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#!/bin/sh

set -oeux pipefail

RELEASE="$(rpm -E '%fedora.%_arch')"

sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/fedora-{cisco-openh264,modular,updates-modular}.repo

# nvidia 520.xxx and newer currently don't have a -$VERSIONxx suffix in their
# package names
if [[ "${NVIDIA_MAJOR_VERSION}" -ge 520 ]]; then
NVIDIA_PACKAGE_NAME="nvidia"
else
NVIDIA_PACKAGE_NAME="nvidia-${NVIDIA_MAJOR_VERSION}xx"
fi

rpm-ostree install \
akmod-${NVIDIA_PACKAGE_NAME}*:${NVIDIA_MAJOR_VERSION}.*.fc${RELEASE} \
xorg-x11-drv-${NVIDIA_PACKAGE_NAME}-{,cuda,devel,kmodsrc,power}*:${NVIDIA_MAJOR_VERSION}.*.fc${RELEASE} \
mock

/tmp/build/ublue-os-just/build.sh

# alternatives cannot create symlinks on its own during a container build
ln -s /usr/bin/ld.bfd /etc/alternatives/ld && ln -s /etc/alternatives/ld /usr/bin/ld

if [[ ! -s "/tmp/certs/private_key.priv" ]]; then
echo "WARNING: Using test signing key. Run './generate-akmods-key' for production builds."
cp /tmp/certs/private_key.priv{.test,}
cp /tmp/certs/public_key.der{.test,}
fi

install -Dm644 /tmp/certs/public_key.der /etc/pki/akmods/certs/public_key.der
install -Dm644 /tmp/certs/private_key.priv /etc/pki/akmods/private/private_key.priv

# Either successfully build and install the kernel modules, or fail early with debug output
KERNEL_VERSION="$(rpm -q kernel --queryformat '%{VERSION}-%{RELEASE}.%{ARCH}')"
NVIDIA_AKMOD_VERSION="$(basename "$(rpm -q "akmod-${NVIDIA_PACKAGE_NAME}" --queryformat '%{VERSION}-%{RELEASE}')" ".fc${RELEASE%%.*}")"
NVIDIA_LIB_VERSION="$(basename "$(rpm -q "xorg-x11-drv-${NVIDIA_PACKAGE_NAME}" --queryformat '%{VERSION}-%{RELEASE}')" ".fc${RELEASE%%.*}")"
NVIDIA_FULL_VERSION="$(rpm -q "xorg-x11-drv-${NVIDIA_PACKAGE_NAME}" --queryformat '%{EPOCH}:%{VERSION}-%{RELEASE}.%{ARCH}')"

akmods --force --kernels "${KERNEL_VERSION}" --kmod "${NVIDIA_PACKAGE_NAME}"

modinfo /usr/lib/modules/${KERNEL_VERSION}/extra/${NVIDIA_PACKAGE_NAME}/nvidia{,-drm,-modeset,-peermem,-uvm}.ko.xz > /dev/null || \
(cat /var/cache/akmods/${NVIDIA_PACKAGE_NAME}/${NVIDIA_AKMOD_VERSION}-for-${KERNEL_VERSION}.failed.log && exit 1)

sed -i "s@gpgcheck=0@gpgcheck=1@" /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo

install -D /etc/pki/akmods/certs/public_key.der /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/public_key.der

rpmbuild -ba \
--define '_topdir /tmp/ublue-os-nvidia-addons/rpmbuild' \
--define '%_tmppath %{_topdir}/tmp' \
/tmp/ublue-os-nvidia-addons/ublue-os-nvidia-addons.spec

cat <<EOF > /var/cache/akmods/nvidia-vars
KERNEL_VERSION=${KERNEL_VERSION}
RELEASE=${RELEASE}
NVIDIA_PACKAGE_NAME=${NVIDIA_PACKAGE_NAME}
NVIDIA_MAJOR_VERSION=${NVIDIA_MAJOR_VERSION}
NVIDIA_FULL_VERSION=${NVIDIA_FULL_VERSION}
NVIDIA_AKMOD_VERSION=${NVIDIA_AKMOD_VERSION}
NVIDIA_LIB_VERSION=${NVIDIA_LIB_VERSION}
EOF
17 changes: 17 additions & 0 deletions install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/sh

set -ouex pipefail

sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/fedora-{cisco-openh264,modular,updates-modular}.repo

install -D /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo \
/etc/yum.repos.d/nvidia-container-runtime.repo

source /tmp/akmods/nvidia-vars

rpm-ostree install \
xorg-x11-drv-${NVIDIA_PACKAGE_NAME}-{,cuda-,devel-,kmodsrc-,power-}${NVIDIA_FULL_VERSION} \
nvidia-container-toolkit nvidia-vaapi-driver \
/tmp/akmods/${NVIDIA_PACKAGE_NAME}/kmod-${NVIDIA_PACKAGE_NAME}-${KERNEL_VERSION}-${NVIDIA_AKMOD_VERSION}.fc${RELEASE}.rpm \
/tmp/ublue-os-nvidia-addons/rpmbuild/RPMS/noarch/ublue-os-nvidia-addons-*.rpm \
/tmp/ublue-os/rpmbuild/RPMS/noarch/ublue-os-just-*.noarch.rpm
16 changes: 16 additions & 0 deletions post-install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/bin/sh

set -ouex pipefail

mv /etc/nvidia-container-runtime/config.toml{,.orig}
cp /etc/nvidia-container-runtime/config{-rootless,}.toml

semodule --verbose --install /usr/share/selinux/packages/nvidia-container.pp
sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/rpmfusion-{,non}free{,-updates}.repo
ln -s /usr/bin/ld.bfd /etc/alternatives/ld
ln -s /etc/alternatives/ld /usr/bin/ld

if [[ "${IMAGE_NAME}" == "sericea" ]]; then
mv /etc/sway/environment{,.orig}
install -Dm644 /usr/share/ublue-os/etc/sway/environment /etc/sway/environment
fi

0 comments on commit ca99377

Please sign in to comment.