Skip to content

Commit

Permalink
feat: Add support for using nvidia GPUs inside containers (#43)
Browse files Browse the repository at this point in the history
* feat: Add support for using nvidia GPUs inside containers

* feat: Add container runtime config files into general add-ons package

* feat: Add config.toml for rootless container support

* fix: Keep backup of original nvidia container runtime config.toml
  • Loading branch information
joshua-stone committed Feb 17, 2023
1 parent 4b1c5e6 commit 303bf28
Show file tree
Hide file tree
Showing 5 changed files with 123 additions and 40 deletions.
38 changes: 29 additions & 9 deletions Containerfile
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,24 @@ RUN NVIDIA_PACKAGE_NAME="$(cat /tmp/nvidia-package-name.txt)" \
|| \
(cat /var/cache/akmods/${NVIDIA_PACKAGE_NAME}/${NVIDIA_VERSION}-for-${KERNEL_VERSION}.failed.log && exit 1)

ADD akmods-nvidia-key.spec /tmp/akmods-nvidia-key/akmods-nvidia-key.spec
ADD ublue-os-nvidia-addons.spec /tmp/ublue-os-nvidia-addons/ublue-os-nvidia-addons.spec

RUN install -D /etc/pki/akmods/certs/public_key.der /tmp/akmods-nvidia-key/rpmbuild/SOURCES/public_key.der
ADD https://nvidia.github.io/nvidia-docker/rhel9.0/nvidia-docker.repo \
/tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo

RUN sed -i "s@gpgcheck=0@gpgcheck=1@" /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo

ADD files/etc/nvidia-container-runtime/config-rootless.toml \
/tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/config-rootless.toml
ADD https://github.com/raw/NVIDIA/dgx-selinux/master/bin/RHEL9/nvidia-container.pp \
/tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container.pp

RUN install -D /etc/pki/akmods/certs/public_key.der /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/public_key.der

RUN rpmbuild -ba \
--define '_topdir /tmp/akmods-nvidia-key/rpmbuild' \
--define '_topdir /tmp/ublue-os-nvidia-addons/rpmbuild' \
--define '%_tmppath %{_topdir}/tmp' \
/tmp/akmods-nvidia-key/akmods-nvidia-key.spec
/tmp/ublue-os-nvidia-addons/ublue-os-nvidia-addons.spec


RUN cp /tmp/nvidia-package-name.txt /var/cache/akmods/nvidia-package-name.txt
Expand All @@ -60,8 +70,13 @@ RUN rpm -q "xorg-x11-drv-$(cat /tmp/nvidia-package-name.txt)" \

FROM ${BASE_IMAGE}:${FEDORA_MAJOR_VERSION}

COPY --from=builder /var/cache/akmods /tmp/akmods
COPY --from=builder /tmp/akmods-nvidia-key /tmp/akmods-nvidia-key
COPY --from=builder /var/cache/akmods /tmp/akmods
COPY --from=builder /tmp/ublue-os-nvidia-addons /tmp/ublue-os-nvidia-addons

RUN sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/fedora-{cisco-openh264,modular,updates-modular}.repo

RUN install -D /tmp/ublue-os-nvidia-addons/rpmbuild/SOURCES/nvidia-container-runtime.repo \
/etc/yum.repos.d/nvidia-container-runtime.repo

RUN KERNEL_VERSION="$(rpm -q kernel --queryformat '%{VERSION}-%{RELEASE}.%{ARCH}')" \
NVIDIA_FULL_VERSION="$(cat /tmp/akmods/nvidia-full-version.txt)" \
Expand All @@ -71,13 +86,18 @@ RUN KERNEL_VERSION="$(rpm -q kernel --queryformat '%{VERSION}-%{RELEASE}.%{ARCH}
https://mirrors.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm \
https://mirrors.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm \
&& \
sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/{fedora-{cisco-openh264,modular,updates-modular},rpmfusion-free{,-updates}}.repo \
sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/rpmfusion-free{,-updates}.repo \
&& \
rpm-ostree install \
xorg-x11-drv-${NVIDIA_PACKAGE_NAME}-{,cuda-,devel-,kmodsrc-,power-}${NVIDIA_FULL_VERSION} \
kernel-devel-${KERNEL_VERSION} \
kernel-devel-${KERNEL_VERSION} nvidia-container-toolkit \
"/tmp/akmods/${NVIDIA_PACKAGE_NAME}/kmod-${NVIDIA_PACKAGE_NAME}-${KERNEL_VERSION}-${NVIDIA_FULL_VERSION#*:}.rpm" \
/tmp/akmods-nvidia-key/rpmbuild/RPMS/noarch/akmods-nvidia-key-*.rpm \
/tmp/ublue-os-nvidia-addons/rpmbuild/RPMS/noarch/ublue-os-nvidia-addons-*.rpm \
&& \
mv /etc/nvidia-container-runtime/config.toml{,.orig} && \
cp /etc/nvidia-container-runtime/config{-rootless,}.toml \
&& \
semodule --verbose --install /usr/share/selinux/packages/nvidia-container.pp \
&& \
sed -i 's@enabled=1@enabled=0@g' /etc/yum.repos.d/rpmfusion-nonfree{,-updates}.repo \
&& \
Expand Down
13 changes: 13 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,19 @@ $ podman build \
--tag build-test:latest
```

## Using Nvidia GPUs in containers

[There is support for enabling Nvidia GPUs in containers](https://www.redhat.com/en/blog/how-use-gpus-containers-bare-metal-rhel-8). This can can be verified by running the following:

```
$ podman run \
--user 1000:1000 \
--security-opt=no-new-privileges \
--cap-drop=ALL \
--security-opt label=type:nvidia_container_t \
docker.io/mirrorgooglecontainers/cuda-vector-add:v0.1
```

## Acknowledgements

Thanks to Alex Diaz for advice, and who got this working first, check out this repo:
Expand Down
31 changes: 0 additions & 31 deletions akmods-nvidia-key.spec

This file was deleted.

34 changes: 34 additions & 0 deletions files/etc/nvidia-container-runtime/config-rootless.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
disable-require = false
#swarm-resource = "DOCKER_RESOURCE_GPU"
#accept-nvidia-visible-devices-envvar-when-unprivileged = true
#accept-nvidia-visible-devices-as-volume-mounts = false

[nvidia-container-cli]
#root = "/run/nvidia/driver"
#path = "/usr/bin/nvidia-container-cli"
environment = []
#debug = "/var/log/nvidia-container-toolkit.log"
#ldcache = "/etc/ld.so.cache"
load-kmods = true
#no-cgroups = false
no-cgroups = true
#user = "root:video"
ldconfig = "@/sbin/ldconfig"

[nvidia-container-runtime]
#debug = "/var/log/nvidia-container-runtime.log"
debug = "~/.local/nvidia-container-runtime.log"
log-level = "info"

# Specify the runtimes to consider. This list is processed in order and the PATH
# searched for matching executables unless the entry is an absolute path.
runtimes = [
"docker-runc",
"runc",
]

mode = "auto"

[nvidia-container-runtime.modes.csv]

mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
47 changes: 47 additions & 0 deletions ublue-os-nvidia-addons.spec
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
Name: ublue-os-nvidia-addons
Version: 0.2
Release: 1%{?dist}
Summary: Additional files for nvidia driver support

License: MIT
URL: https://github.com/ublue-os/nvidia

BuildArch: noarch
Supplements: mokutil policycoreutils

Source0: public_key.der
Source1: nvidia-container-runtime.repo
Source2: config-rootless.toml
Source3: nvidia-container.pp

%description
Adds various runtime files for nvidia support. These include a key for importing with mokutil to enable secure boot for nvidia kernel modules

%prep
%setup -q -c -T


%install
# Have different name for *.der in case kmodgenca is needed for creating more keys
install -Dm0644 %{SOURCE0} %{buildroot}%{_sysconfdir}/pki/akmods/certs/akmods-nvidia.der
install -Dm0644 %{SOURCE1} %{buildroot}%{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo
install -Dm0644 %{SOURCE2} %{buildroot}%{_sysconfdir}/nvidia-container-runtime/config-rootless.toml
install -Dm0644 %{SOURCE3} %{buildroot}%{_datadir}/selinux/packages/nvidia-container.pp

sed -i 's@enabled=1@enabled=0@g' %{buildroot}%{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo

%files
%attr(0644,root,root) %{_sysconfdir}/pki/akmods/certs/akmods-nvidia.der
%attr(0644,root,root) %{_sysconfdir}/yum.repos.d/nvidia-container-runtime.repo
%attr(0644,root,root) %{_sysconfdir}/nvidia-container-runtime/config-rootless.toml
%attr(0644,root,root) %{_datadir}/selinux/packages/nvidia-container.pp

%changelog
* Thu Feb 16 2023 Joshua Stone <joshua.gage.stone@gmail.com> - 0.2
- Add nvidia-container-runtime repo
- Add nvidia-container-runtime selinux policy file
- Re-purpose into a general-purpose add-on package
- Update URL to point to ublue-os project

* Fri Feb 03 2023 Joshua Stone <joshua.gage.stone@gmail.com> - 0.1
- Add key for enrolling kernel modules in alpha builds

0 comments on commit 303bf28

Please sign in to comment.