From 55a4ba343f76937cee4ace536ddedecb171800bf Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Fri, 27 Sep 2024 22:09:54 +0300 Subject: [PATCH 1/8] fix(docker): Setup system locales in containers where it actually counts, not before --- Dockerfile | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index f28202a65..23578d077 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,22 +7,11 @@ FROM docker.io/library/archlinux:base-devel$ARCHTAG AS builder ARG RUNTIME_DEPS ARG BUILD_DEPS -# Enable system locales for everything we have localizations for so tools like -# `date` will output matching localized strings. By default Arch Docker images -# have almost all locale data stripped out. This also makes it easier to -# rebuild custom Docker images with extra languages supported. -RUN sed -i -e '/^NoExtract.*locale/d' /etc/pacman.conf - # Freshen all base system packages RUN pacman-key --init RUN pacman --needed --noconfirm -Syq archlinux-keyring RUN pacman --needed --noconfirm -Suq -# Make sure *at least* glibc actually got reinstalled after enabling -# extraaction of locale files even if the version was fresh so we can use the -# locale support out of it later. -RUN pacman --noconfirm -Sq glibc && yes | pacman -Sccq - # Install run-time dependencies RUN pacman --needed --noconfirm -Sq $RUNTIME_DEPS $BUILD_DEPS @@ -32,10 +21,6 @@ ARG REVISION COPY ./ /src WORKDIR /src -# Rebuild locale database after having added our supported locales. -RUN ls i18n/ | sed 's/[.-].*$/_/;s/^/^/' | sort -u | grep -Ef - /usr/share/i18n/SUPPORTED > /etc/locale.gen -RUN locale-gen - # GitHub Actions builder stopped providing git history :( # See feature request at https://github.com/actions/runner/issues/767 RUN build-aux/docker-bootstrap.sh @@ -58,6 +43,12 @@ ARG REVISION # Allow `su` with no root password so non-priv users can install dependencies RUN sed -i -e '/.so$/s/$/ nullok/' /etc/pam.d/su +# Enable system locales for everything we have localizations for so tools like +# `date` will output matching localized strings. By default Arch Docker images +# have almost all locale data stripped out. This also makes it easier to +# rebuild custom Docker images with extra languages supported. +RUN sed -i -e '/^NoExtract.*locale/d' /etc/pacman.conf + # Set system locale to something other than 'C' that resolves to a real language ENV LANG=en_US.UTF-8 @@ -66,6 +57,11 @@ RUN pacman-key --init RUN pacman --needed --noconfirm -Syq archlinux-keyring && yes | pacman -Sccq RUN pacman --needed --noconfirm -Suq && yes | pacman -Sccq +# Make sure *at least* glibc actually got reinstalled after enabling +# extraction of locale files even if the version was fresh so we can use the +# locale support out of it later. +RUN pacman --noconfirm -Sq glibc && yes | pacman -Sccq + # Install run-time dependencies RUN pacman --needed --noconfirm -Sq $RUNTIME_DEPS && yes | pacman -Sccq @@ -82,6 +78,11 @@ COPY build-aux/docker-fontconfig.conf /etc/fonts/conf.d/99-docker.conf COPY --from=builder /pkgdir / COPY --from=builder /src/src/sile-entry.sh /usr/local/bin + +# Rebuild locale database after having added our supported locales. +RUN ls /usr/local/share/sile/i18n/ | sed 's/[.-].*$/_/;s/^/^/' | sort -u | grep -Ef - /usr/share/i18n/SUPPORTED > /etc/locale.gen +RUN locale-gen + RUN sile --version WORKDIR /data From d53acf366c54b5ac59a2dda1c8d883b0d7bfb991 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Fri, 27 Sep 2024 23:03:44 +0300 Subject: [PATCH 2/8] chore(docker): Update Arch system package name for Libertinus Fonts --- hooks/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/build b/hooks/build index f982069db..2a7a82560 100755 --- a/hooks/build +++ b/hooks/build @@ -7,7 +7,7 @@ set -e REVISION=$(git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g') RUNTIME_DEPS='fontconfig freetype2 gentium-plus-font glibc harfbuzz icu libpng luajit zlib'\ -' libertinus-font luarocks noto-fonts-cjk ttf-hack' +' otf-libertinus luarocks noto-fonts-cjk ttf-hack' BUILD_DEPS='cargo git jq lua51 poppler' : "${DOCKER_BUILDKIT:=1}" From 57ab6b758c1e8716e20d694a62194af75d444b90 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Fri, 27 Sep 2024 23:16:17 +0300 Subject: [PATCH 3/8] refactor(docker): Eliminate Lua 5.1 during build, use LuaJIT as default interpreter --- Dockerfile | 6 ++++++ hooks/build | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 23578d077..23445a7ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,12 @@ RUN pacman --needed --noconfirm -Suq # Install run-time dependencies RUN pacman --needed --noconfirm -Sq $RUNTIME_DEPS $BUILD_DEPS +# Setup LuaRocks for use with LuaJIT roughly matching SILE's internal VM +RUN luarocks config lua_version 5.1 && \ + luarocks config lua_interpreter luajit && \ + luarocks config variables.LUA "$(command -v luajit)" && \ + luarocks config variables.LUA_INCDIR /usr/include/luajit-2.1/ + # Set at build time, forces Docker’s layer caching to reset at this point ARG REVISION diff --git a/hooks/build b/hooks/build index 2a7a82560..cd3190e32 100755 --- a/hooks/build +++ b/hooks/build @@ -8,7 +8,7 @@ REVISION=$(git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g') RUNTIME_DEPS='fontconfig freetype2 gentium-plus-font glibc harfbuzz icu libpng luajit zlib'\ ' otf-libertinus luarocks noto-fonts-cjk ttf-hack' -BUILD_DEPS='cargo git jq lua51 poppler' +BUILD_DEPS='cargo git jq poppler' : "${DOCKER_BUILDKIT:=1}" export DOCKER_BUILDKIT From 388b1df7f806d4c2d24da2379d23801a2819c1aa Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Sat, 28 Sep 2024 00:06:20 +0300 Subject: [PATCH 4/8] refactor(docker): Redo container using base/builder/final model For less code repetition and smaller images. --- Dockerfile | 76 +++++++++++++++++++++++++++++------------------------ hooks/build | 4 +-- 2 files changed, 43 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 23445a7ed..372eddc93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,18 +2,36 @@ ARG ARCHTAG -FROM docker.io/library/archlinux:base-devel$ARCHTAG AS builder +FROM docker.io/library/archlinux:$ARCHTAG AS base + +# Initialize keys so we can do package management +RUN pacman-key --init && pacman-key --populate + +# This hack can convince Docker its cache is obsolete; e.g. when the contents +# of downloaded resources have changed since being fetched. It's helpful to have +# this as a separate layer because it saves time for local builds. Incrementing +# this when pushing dependency updates to Caleb's Arch user repository or just +# when the remote Docker Hub builds die should be enough. +ARG DOCKER_HUB_CACHE=1 ARG RUNTIME_DEPS -ARG BUILD_DEPS + +# Enable system locales for everything we have localizations for so tools like +# `date` will output matching localized strings. By default Arch Docker images +# have almost all locale data stripped out. This also makes it easier to +# rebuild custom Docker images with extra languages supported. +RUN sed -i -e '/^NoExtract.*locale/d' /etc/pacman.conf # Freshen all base system packages -RUN pacman-key --init -RUN pacman --needed --noconfirm -Syq archlinux-keyring -RUN pacman --needed --noconfirm -Suq +RUN pacman --needed --noconfirm -Syuq && yes | pacman -Sccq + +# Make sure *at least* glibc actually got reinstalled after enabling +# extraction of locale files even if the version was fresh so we can use the +# locale support out of it later. +RUN pacman --noconfirm -Sq glibc && yes | pacman -Sccq # Install run-time dependencies -RUN pacman --needed --noconfirm -Sq $RUNTIME_DEPS $BUILD_DEPS +RUN pacman --needed --noconfirm -Sq $RUNTIME_DEPS && yes | pacman -Sccq # Setup LuaRocks for use with LuaJIT roughly matching SILE's internal VM RUN luarocks config lua_version 5.1 && \ @@ -21,55 +39,47 @@ RUN luarocks config lua_version 5.1 && \ luarocks config variables.LUA "$(command -v luajit)" && \ luarocks config variables.LUA_INCDIR /usr/include/luajit-2.1/ +# Setup separate image for build so we don’t bloat the final image +FROM base AS builder + +ARG BUILD_DEPS + +# Install build time dependencies +RUN pacman --needed --noconfirm -Sq $BUILD_DEPS && yes | pacman -Sccq + # Set at build time, forces Docker’s layer caching to reset at this point ARG REVISION COPY ./ /src WORKDIR /src +# Take note of SILE's supported locales so the final system can build localized messages +RUN ls i18n/ | sed 's/[.-].*$/_/;s/^/^/' | sort -u | grep -Ef - /usr/share/i18n/SUPPORTED > /etc/locale.gen + # GitHub Actions builder stopped providing git history :( # See feature request at https://github.com/actions/runner/issues/767 RUN build-aux/docker-bootstrap.sh RUN ./bootstrap.sh -RUN ./configure --with-system-lua-sources --without-manual +RUN ./configure --mandir='$prefix}/man' --with-system-lua-sources --without-system-luarocks --without-manual RUN make RUN make install DESTDIR=/pkgdir -# Work around BuiltKit / buildx bug, they can’t copy to symlinks only dirs -RUN mv /pkgdir/usr/local/{share/,}/man - -FROM docker.io/library/archlinux:base$ARCHTAG AS final +FROM base AS final # Same args as above, repeated because they went out of scope with FROM -ARG RUNTIME_DEPS -ARG VERSION ARG REVISION +ARG VERSION # Allow `su` with no root password so non-priv users can install dependencies RUN sed -i -e '/.so$/s/$/ nullok/' /etc/pam.d/su -# Enable system locales for everything we have localizations for so tools like -# `date` will output matching localized strings. By default Arch Docker images -# have almost all locale data stripped out. This also makes it easier to -# rebuild custom Docker images with extra languages supported. -RUN sed -i -e '/^NoExtract.*locale/d' /etc/pacman.conf - # Set system locale to something other than 'C' that resolves to a real language ENV LANG=en_US.UTF-8 -# Freshen all base system packages (and cleanup cache) -RUN pacman-key --init -RUN pacman --needed --noconfirm -Syq archlinux-keyring && yes | pacman -Sccq -RUN pacman --needed --noconfirm -Suq && yes | pacman -Sccq - -# Make sure *at least* glibc actually got reinstalled after enabling -# extraction of locale files even if the version was fresh so we can use the -# locale support out of it later. -RUN pacman --noconfirm -Sq glibc && yes | pacman -Sccq - -# Install run-time dependencies -RUN pacman --needed --noconfirm -Sq $RUNTIME_DEPS && yes | pacman -Sccq +# Rebuild locale database so system apps have localized messages for SILE's supported locales +COPY --from=builder /etc/locale.gen /etc +RUN locale-gen LABEL org.opencontainers.image.title="SILE" LABEL org.opencontainers.image.description="A containerized version of the SILE typesetter" @@ -85,10 +95,6 @@ COPY build-aux/docker-fontconfig.conf /etc/fonts/conf.d/99-docker.conf COPY --from=builder /pkgdir / COPY --from=builder /src/src/sile-entry.sh /usr/local/bin -# Rebuild locale database after having added our supported locales. -RUN ls /usr/local/share/sile/i18n/ | sed 's/[.-].*$/_/;s/^/^/' | sort -u | grep -Ef - /usr/share/i18n/SUPPORTED > /etc/locale.gen -RUN locale-gen - RUN sile --version WORKDIR /data diff --git a/hooks/build b/hooks/build index cd3190e32..70b30b56d 100755 --- a/hooks/build +++ b/hooks/build @@ -1,14 +1,14 @@ #!/usr/bin/env sh set -e -: "${ARCHTAG:=}" +: "${ARCHTAG:=latest}" : "${DOCKERFILE:=Dockerfile}" REVISION=$(git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g') RUNTIME_DEPS='fontconfig freetype2 gentium-plus-font glibc harfbuzz icu libpng luajit zlib'\ ' otf-libertinus luarocks noto-fonts-cjk ttf-hack' -BUILD_DEPS='cargo git jq poppler' +BUILD_DEPS='base-devel cargo git jq poppler' : "${DOCKER_BUILDKIT:=1}" export DOCKER_BUILDKIT From 157bdaacf3c81750098969cf28dcbbcc6ee74be3 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Sat, 28 Sep 2024 18:00:55 +0300 Subject: [PATCH 5/8] chore(docker): Use clang/mold in container bulids for speed --- Dockerfile | 3 +++ hooks/build | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 372eddc93..418fde46b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,6 +60,9 @@ RUN ls i18n/ | sed 's/[.-].*$/_/;s/^/^/' | sort -u | grep -Ef - /usr/share/i18n/ # See feature request at https://github.com/actions/runner/issues/767 RUN build-aux/docker-bootstrap.sh +# Use clang and mold instead of gcc and ld for speed +ENV RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=mold" + RUN ./bootstrap.sh RUN ./configure --mandir='$prefix}/man' --with-system-lua-sources --without-system-luarocks --without-manual RUN make diff --git a/hooks/build b/hooks/build index 70b30b56d..28b002235 100755 --- a/hooks/build +++ b/hooks/build @@ -8,7 +8,7 @@ REVISION=$(git describe --long --tags | sed 's/^v//;s/\([^-]*-g\)/r\1/;s/-/./g') RUNTIME_DEPS='fontconfig freetype2 gentium-plus-font glibc harfbuzz icu libpng luajit zlib'\ ' otf-libertinus luarocks noto-fonts-cjk ttf-hack' -BUILD_DEPS='base-devel cargo git jq poppler' +BUILD_DEPS='base-devel cargo clang git jq mold poppler' : "${DOCKER_BUILDKIT:=1}" export DOCKER_BUILDKIT From 548f42661d777a3c8882a6fd89e79601494fb796 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Sat, 28 Sep 2024 22:47:44 +0300 Subject: [PATCH 6/8] feat(docker): Splice SILE's vendored rocks tree into system LuaRocks --- Dockerfile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Dockerfile b/Dockerfile index 418fde46b..4892e935d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -95,6 +95,16 @@ LABEL org.opencontainers.image.revision="$REVISION" COPY build-aux/docker-fontconfig.conf /etc/fonts/conf.d/99-docker.conf +# Inform the system Lua manifest where SILE's vendored modules are so they are +# available to 3rd party packages even outside of SILE's runtime. Most notably +# useful so that luarocks can find them as existing dependencies when +# installing 3rd party modules. We replace the user tree instead of inserting +# a new one because it doesn't make sense in Docker anyway and the default +# priority works out better having it first. +RUN luarocks config rocks_trees[1].root /usr/local/share/sile/lua_modules && \ + luarocks config rocks_trees[1].name sile && \ + luarocks config deps_mode all + COPY --from=builder /pkgdir / COPY --from=builder /src/src/sile-entry.sh /usr/local/bin From c616286deeeb4359689d70c44d49aa5b3f2a0944 Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Sun, 29 Sep 2024 00:42:56 +0300 Subject: [PATCH 7/8] ci(actions): Fixup test build from packaged sources --- .github/workflows/build.yml | 2 +- tests/Dockerfile-dist | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9aaac0663..1a55ca7f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -124,7 +124,7 @@ jobs: run: | make fake-manual make distdir - - name: Build from package + - name: Build from dist tree run: | make docker-test-dist diff --git a/tests/Dockerfile-dist b/tests/Dockerfile-dist index b8424552b..e8697192b 100644 --- a/tests/Dockerfile-dist +++ b/tests/Dockerfile-dist @@ -1,22 +1,24 @@ #syntax=docker/dockerfile:1.2 -ARG ARCHTAG - -FROM docker.io/library/archlinux:base-devel$ARCHTAG +FROM docker.io/library/archlinux:base ARG RUNTIME_DEPS ARG BUILD_DEPS # Freshen all base system packages -RUN pacman-key --init -RUN pacman --needed --noconfirm -Syq archlinux-keyring -RUN pacman --needed --noconfirm -Suq +RUN pacman --needed --noconfirm -Syuq # Install build and run-time dependencies RUN pacman --needed --noconfirm -Sq $RUNTIME_DEPS $BUILD_DEPS # Remove autotools, building the source distributions should not require them -RUN pacman --noconfirm -Rn autoconf automake libtool base-devel +RUN pacman --noconfirm -Rn autoconf automake libtool base-devel hwloc onetbb clang mold + +# Setup LuaRocks for use with LuaJIT roughly matching SILE's internal VM +RUN luarocks config lua_version 5.1 && \ + luarocks config lua_interpreter luajit && \ + luarocks config variables.LUA "$(command -v luajit)" && \ + luarocks config variables.LUA_INCDIR /usr/include/luajit-2.1/ # Set at build time, forces Docker’s layer caching to reset at this point ARG VERSION @@ -24,7 +26,11 @@ ARG VERSION COPY ./sile-$VERSION /src WORKDIR /src -RUN ./configure --with-system-lua-sources --without-manual +RUN ./configure \ + --disable-embeded-resources \ + --with-system-lua-sources \ + --without-system-luarocks \ + --without-manual RUN make RUN make install From ef0736160d4336355dabbc93f5a0ba2c83820ccb Mon Sep 17 00:00:00 2001 From: Caleb Maclennan Date: Sun, 29 Sep 2024 01:35:38 +0300 Subject: [PATCH 8/8] chore(docker): Disable pointless asset embeding in containers --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4892e935d..e576834ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,7 +64,11 @@ RUN build-aux/docker-bootstrap.sh ENV RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=mold" RUN ./bootstrap.sh -RUN ./configure --mandir='$prefix}/man' --with-system-lua-sources --without-system-luarocks --without-manual +RUN ./configure \ + --disable-embeded-resources \ + --with-system-lua-sources \ + --without-system-luarocks \ + --without-manual RUN make RUN make install DESTDIR=/pkgdir