From 6badd98cae131366bf02f90671906d3328af30ce Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Sat, 30 Dec 2023 23:33:46 +0100 Subject: [PATCH 1/6] update docker stacks parsing --- .build/10activate-conda-env.sh | 8 + .build/Dockerfile | 152 +++++++++++----- .build/docker-stacks | 2 +- .build/docker_healthcheck.py | 9 +- .build/fix-permissions | 14 +- .build/jupyter_server_config.py | 23 ++- .build/run-hooks.sh | 46 +++++ .../activate_notebook_custom_env.py | 24 +++ .../setup-scripts/setup-julia-packages.bash | 30 +++- .build/setup-scripts/setup_julia.py | 85 +++++++++ .build/start-notebook.py | 41 +++++ .build/start-notebook.sh | 23 +-- .build/start-singleuser.py | 23 +++ .build/start-singleuser.sh | 14 +- .build/start.sh | 75 +++----- generate-Dockerfile copy.sh | 169 ++++++++++++++++++ generate-Dockerfile.sh | 71 ++++++-- src/Dockerfile.gpulibs | 6 - src/Dockerfile.header | 2 +- 19 files changed, 642 insertions(+), 175 deletions(-) create mode 100755 .build/10activate-conda-env.sh create mode 100755 .build/run-hooks.sh create mode 100755 .build/setup-scripts/activate_notebook_custom_env.py create mode 100755 .build/setup-scripts/setup_julia.py create mode 100755 .build/start-notebook.py create mode 100755 .build/start-singleuser.py create mode 100755 generate-Dockerfile copy.sh diff --git a/.build/10activate-conda-env.sh b/.build/10activate-conda-env.sh new file mode 100755 index 0000000..ed7347f --- /dev/null +++ b/.build/10activate-conda-env.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# This registers the initialization code for the conda shell code +# It also activates default environment in the end, so we don't need to activate it manually +# Documentation: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html +eval "$(conda shell.bash hook)" diff --git a/.build/Dockerfile b/.build/Dockerfile index 1d45e7b..0fb0e77 100755 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -7,7 +7,7 @@ # Use NVIDIA CUDA as base image and run the same installation as in the other packages. # The version of cuda must match those of the packages installed in src/Dockerfile.gpulibs -FROM nvidia/cuda:12.0.1-cudnn8-runtime-ubuntu22.04 +FROM nvidia/cuda:11.6.2-cudnn8-runtime-ubuntu20.04 LABEL authors="Christoph Schranz , Mathematical Michael " # This is a concatenated Dockerfile, the maintainers of subsequent sections may vary. RUN chmod 1777 /tmp && chmod 1777 /var/tmp @@ -39,12 +39,12 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Install all OS dependencies for notebook server that starts but lacks all -# features (e.g., download as all possible file formats) +# Install all OS dependencies for the Server that starts +# but lacks all features (e.g., download as all possible file formats) ENV DEBIAN_FRONTEND noninteractive RUN apt-get update --yes && \ - # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as - # the ubuntu base image is rebuilt too seldom sometimes (less than once a month) + # - `apt-get upgrade` is run to patch known vulnerabilities in apt-get packages as + # the Ubuntu base image is rebuilt too seldom sometimes (less than once a month) apt-get upgrade --yes && \ apt-get install --yes --no-install-recommends \ # - bzip2 is necessary to extract the micromamba executable. @@ -80,36 +80,37 @@ RUN chmod a+rx /usr/local/bin/fix-permissions # Enable prompt color in the skeleton .bashrc before creating the default NB_USER # hadolint ignore=SC2016 RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ - # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 - echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc + # More information in: https://github.com/jupyter/docker-stacks/pull/2047 + # and docs: https://docs.conda.io/projects/conda/en/latest/dev-guide/deep-dives/activation.html + echo 'eval "$(conda shell.bash hook)"' >> /etc/skel/.bashrc # Create NB_USER with name jovyan user with UID=1000 and in the 'users' group # and make sure these dirs are writable by the `users` group. RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ - useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \ + useradd --no-log-init --create-home --shell /bin/bash --uid "${NB_UID}" --no-user-group "${NB_USER}" && \ mkdir -p "${CONDA_DIR}" && \ chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ chmod g+w /etc/passwd && \ - fix-permissions "${HOME}" && \ - fix-permissions "${CONDA_DIR}" + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" USER ${NB_UID} -# Pin python version here, or set it to "default" +# Pin the Python version here, or set it to "default" ARG PYTHON_VERSION=3.11 # Setup work directory for backward-compatibility RUN mkdir "/home/${NB_USER}/work" && \ fix-permissions "/home/${NB_USER}" -# Download and install Micromamba, and initialize Conda prefix. +# Download and install Micromamba, and initialize the Conda prefix. # # Similar projects using Micromamba: # - Micromamba-Docker: # - repo2docker: -# Install Python, Mamba and jupyter_core +# Install Python, Mamba, and jupyter_core # Cleanup temporary files and remove Micromamba # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the @@ -148,7 +149,15 @@ ENTRYPOINT ["tini", "-g", "--"] CMD ["start.sh"] # Copy local files as late as possible to avoid cache busting -COPY start.sh /usr/local/bin/ +COPY run-hooks.sh start.sh /usr/local/bin/ + +USER root + +# Create dirs for startup hooks +RUN mkdir /usr/local/bin/start-notebook.d && \ + mkdir /usr/local/bin/before-notebook.d + +COPY 10activate-conda-env.sh /usr/local/bin/before-notebook.d/ # Switch back to jovyan to avoid accidental container runs as root USER ${NB_UID} @@ -161,6 +170,7 @@ WORKDIR "${HOME}" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter LABEL maintainer="Jupyter Project " @@ -171,35 +181,35 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Install all OS dependencies for notebook server that starts but lacks all +# Install all OS dependencies for the Server that starts but lacks all # features (e.g., download as all possible file formats) RUN apt-get update --yes && \ apt-get install --yes --no-install-recommends \ fonts-liberation \ # - pandoc is used to convert notebooks to html files - # it's not present in aarch64 ubuntu image, so we install it here + # it's not present in the aarch64 Ubuntu image, so we install it here pandoc \ # - run-one - a wrapper script that runs no more # than one unique instance of some command with a unique set of arguments, - # we use `run-one-constantly` to support `RESTARTABLE` option + # we use `run-one-constantly` to support the `RESTARTABLE` option run-one && \ apt-get clean && rm -rf /var/lib/apt/lists/* USER ${NB_UID} -# Install Jupyter Notebook, Lab, and Hub -# Generate a notebook server config +# Install JupyterLab, Jupyter Notebook, JupyterHub and NBClassic +# Generate a Jupyter Server config # Cleanup temporary files # Correct permissions # Do all this in a single RUN command to avoid duplicating all of the # files across image layers when the permissions change WORKDIR /tmp RUN mamba install --yes \ + 'jupyterlab' \ 'notebook' \ 'jupyterhub' \ - 'jupyterlab' \ 'nbclassic' && \ - jupyter notebook --generate-config && \ + jupyter server --generate-config && \ mamba clean --all -f -y && \ npm cache clean --force && \ jupyter lab clean && \ @@ -211,23 +221,18 @@ ENV JUPYTER_PORT=8888 EXPOSE $JUPYTER_PORT # Configure container startup -CMD ["start-notebook.sh"] +CMD ["start-notebook.py"] # Copy local files as late as possible to avoid cache busting -COPY start-notebook.sh start-singleuser.sh /usr/local/bin/ -# Currently need to have both jupyter_notebook_config and jupyter_server_config to support classic and lab +COPY start-notebook.py start-notebook.sh start-singleuser.py start-singleuser.sh /usr/local/bin/ COPY jupyter_server_config.py docker_healthcheck.py /etc/jupyter/ # Fix permissions on /etc/jupyter as root USER root - -# Legacy for Jupyter Notebook Server, see: [#1205](https://github.com/jupyter/docker-stacks/issues/1205) -RUN sed -re "s/c.ServerApp/c.NotebookApp/g" \ - /etc/jupyter/jupyter_server_config.py > /etc/jupyter/jupyter_notebook_config.py && \ - fix-permissions /etc/jupyter/ +RUN fix-permissions /etc/jupyter/ # HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck -# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server` and `retro` jupyter commands +# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server`, and `retro` jupyter commands # https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799 HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ CMD /etc/jupyter/docker_healthcheck.py || exit 1 @@ -243,6 +248,7 @@ WORKDIR "${HOME}" # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter LABEL maintainer="Jupyter Project " @@ -253,10 +259,11 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] USER root -# Install all OS dependencies for fully functional notebook server +# Install all OS dependencies for a fully functional Server RUN apt-get update --yes && \ apt-get install --yes --no-install-recommends \ # Common useful utilities + curl \ git \ nano-tiny \ tzdata \ @@ -282,7 +289,7 @@ RUN update-alternatives --install /usr/bin/nano nano /bin/nano-tiny 10 # Switch back to jovyan to avoid accidental container runs as root USER ${NB_UID} -# Add R mimetype option to specify how the plot returns from R to the browser +# Add an R mimetype option to specify how the plot returns from R to the browser COPY --chown=${NB_UID}:${NB_GID} Rprofile.site /opt/conda/lib/R/etc/ # Add setup scripts that may be used by downstream images or inherited images @@ -294,6 +301,7 @@ COPY setup-scripts/ /opt/setup-scripts/ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io ARG OWNER=jupyter LABEL maintainer="Jupyter Project " @@ -353,17 +361,15 @@ RUN mamba install --yes \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Install facets which does not have a pip or conda package at the moment +# Install facets package which does not have a `pip` or `conda-forge` package at the moment WORKDIR /tmp -RUN git clone https://github.com/PAIR-code/facets.git && \ +RUN git clone https://github.com/PAIR-code/facets && \ jupyter nbclassic-extension install facets/facets-dist/ --sys-prefix && \ rm -rf /tmp/facets && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Import matplotlib the first time to build the font cache. -ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/" - +# Import matplotlib the first time to build the font cache RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ fix-permissions "/home/${NB_USER}" @@ -371,6 +377,72 @@ USER ${NB_UID} WORKDIR "${HOME}" + ############################################################################ + ################ Dependency: jupyter/datascience-notebook ################## + ############################################################################ + +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +ARG REGISTRY=quay.io +ARG OWNER=jupyter + +LABEL maintainer="Jupyter Project " + +# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 +# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +USER root + +# R pre-requisites +RUN apt-get update --yes && \ + apt-get install --yes --no-install-recommends \ + fonts-dejavu \ + gfortran \ + gcc && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +# Julia dependencies +# install Julia packages in /opt/julia instead of ${HOME} +ENV JULIA_DEPOT_PATH=/opt/julia \ + JULIA_PKGDIR=/opt/julia + +# Setup Julia +RUN /opt/setup-scripts/setup_julia.py + +USER ${NB_UID} + +# Setup IJulia kernel & other packages +RUN /opt/setup-scripts/setup-julia-packages.bash + +# R packages including IRKernel which gets installed globally. +# r-e1071: dependency of the caret R package +RUN mamba install --yes \ + 'r-base' \ + 'r-caret' \ + 'r-crayon' \ + 'r-devtools' \ + 'r-e1071' \ + 'r-forecast' \ + 'r-hexbin' \ + 'r-htmltools' \ + 'r-htmlwidgets' \ + 'r-irkernel' \ + 'r-nycflights13' \ + 'r-randomforest' \ + 'r-rcurl' \ + 'r-rmarkdown' \ + 'r-rodbc' \ + 'r-rsqlite' \ + 'r-shiny' \ + 'r-tidymodels' \ + 'r-tidyverse' \ + 'rpy2' \ + 'unixodbc' && \ + mamba clean --all -f -y && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + ############################################################################ ########################## Dependency: gpulibs ############################# ############################################################################ @@ -381,12 +453,6 @@ LABEL maintainer="Christoph Schranz , Mat # https://www.tensorflow.org/install/source#gpu # installation via conda leads to errors in version 4.8.2 # Install CUDA-specific nvidia libraries and update libcudnn8 before that -USER root -RUN apt-get update && \ - apt-get install -y --no-install-recommends --allow-change-held-packages libcudnn8 && \ - apt-get install -y --no-install-recommends libnvinfer-dev libnvinfer-plugin-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists/* -RUN cd /usr/lib/x86_64-linux-gnu && ln -s libnvinfer_plugin.so.8 libnvinfer_plugin.so.7 && ln -s libnvinfer.so.8 libnvinfer.so.7 USER ${NB_UID} RUN pip install --upgrade pip && \ pip install --no-cache-dir tensorflow==2.15.0 keras==2.15.0 && \ diff --git a/.build/docker-stacks b/.build/docker-stacks index b8d617d..1494233 160000 --- a/.build/docker-stacks +++ b/.build/docker-stacks @@ -1 +1 @@ -Subproject commit b8d617dc0568d60f6583c42f989da51ec80e9af6 +Subproject commit 1494233e27cdc70e3766ea2518e7153ee425fc4f diff --git a/.build/docker_healthcheck.py b/.build/docker_healthcheck.py index 7c35a6b..8f3338e 100755 --- a/.build/docker_healthcheck.py +++ b/.build/docker_healthcheck.py @@ -7,7 +7,7 @@ import requests -# A number of operations below deliberately don't check for possible errors +# Several operations below deliberately don't check for possible errors # As this is a healthcheck, it should succeed or raise an exception on error runtime_dir = Path("/home/") / os.environ["NB_USER"] / ".local/share/jupyter/runtime/" @@ -16,6 +16,11 @@ url = json.loads(json_file.read_bytes())["url"] url = url + "api" -r = requests.get(url, verify=False) # request without SSL verification +proxies = { + "http": "", + "https": "", +} + +r = requests.get(url, proxies=proxies, verify=False) # request without SSL verification r.raise_for_status() print(r.content) diff --git a/.build/fix-permissions b/.build/fix-permissions index d167578..47b6d0e 100755 --- a/.build/fix-permissions +++ b/.build/fix-permissions @@ -1,16 +1,14 @@ #!/bin/bash -# set permissions on a directory -# after any installation, if a directory needs to be (human) user-writable, -# run this script on it. -# It will make everything in the directory owned by the group ${NB_GID} -# and writable by that group. +# Set permissions on a directory +# After any installation, if a directory needs to be (human) user-writable, run this script on it. +# It will make everything in the directory owned by the group ${NB_GID} and writable by that group. # Deployments that want to set a specific user id can preserve permissions # by adding the `--group-add users` line to `docker run`. -# uses find to avoid touching files that already have the right permissions, -# which would cause massive image explosion +# Uses find to avoid touching files that already have the right permissions, +# which would cause a massive image explosion -# right permissions are: +# Right permissions are: # group=${NB_GID} # AND permissions include group rwX (directory-execute) # AND directories have setuid,setgid bits set diff --git a/.build/jupyter_server_config.py b/.build/jupyter_server_config.py index 679f96b..c0cca3a 100755 --- a/.build/jupyter_server_config.py +++ b/.build/jupyter_server_config.py @@ -4,6 +4,7 @@ import os import stat import subprocess +from pathlib import Path from jupyter_core.paths import jupyter_data_dir @@ -24,17 +25,16 @@ [req_distinguished_name] """ if "GEN_CERT" in os.environ: - dir_name = jupyter_data_dir() - pem_file = os.path.join(dir_name, "notebook.pem") - os.makedirs(dir_name, exist_ok=True) + dir_name = Path(jupyter_data_dir()) + dir_name.mkdir(parents=True, exist_ok=True) + pem_file = dir_name / "notebook.pem" # Generate an openssl.cnf file to set the distinguished name - cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf") - if not os.path.isfile(cnf_file): - with open(cnf_file, "w") as fh: - fh.write(OPENSSL_CONFIG) + cnf_file = Path(os.getenv("CONDA_DIR", "/usr/lib")) / "ssl/openssl.cnf" + if not cnf_file.exists(): + cnf_file.write_text(OPENSSL_CONFIG) - # Generate a certificate if one doesn't exist on disk + # Generate a certificate if one doesn't exist on a disk subprocess.check_call( [ "openssl", @@ -50,10 +50,9 @@ ] ) # Restrict access to the file - os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) - c.ServerApp.certfile = pem_file + pem_file.chmod(stat.S_IRUSR | stat.S_IWUSR) + c.ServerApp.certfile = str(pem_file) -# Change default umask for all subprocesses of the notebook server if set in -# the environment +# Change default umask for all subprocesses of the Server if set in the environment if "NB_UMASK" in os.environ: os.umask(int(os.environ["NB_UMASK"], 8)) diff --git a/.build/run-hooks.sh b/.build/run-hooks.sh new file mode 100755 index 0000000..15df23c --- /dev/null +++ b/.build/run-hooks.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# The run-hooks.sh script looks for *.sh scripts to source +# and executable files to run within a passed directory + +if [ "$#" -ne 1 ]; then + echo "Should pass exactly one directory" + return 1 +fi + +if [[ ! -d "${1}" ]]; then + echo "Directory ${1} doesn't exist or is not a directory" + return 1 +fi + +echo "Running hooks in: ${1} as uid: $(id -u) gid: $(id -g)" +for f in "${1}/"*; do + # Handling a case when the directory is empty + [ -e "${f}" ] || continue + case "${f}" in + *.sh) + echo "Sourcing shell script: ${f}" + # shellcheck disable=SC1090 + source "${f}" + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "${f} has failed, continuing execution" + fi + ;; + *) + if [ -x "${f}" ]; then + echo "Running executable: ${f}" + "${f}" + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "${f} has failed, continuing execution" + fi + else + echo "Ignoring non-executable: ${f}" + fi + ;; + esac +done +echo "Done running hooks in: ${1}" diff --git a/.build/setup-scripts/activate_notebook_custom_env.py b/.build/setup-scripts/activate_notebook_custom_env.py new file mode 100755 index 0000000..4d5da9b --- /dev/null +++ b/.build/setup-scripts/activate_notebook_custom_env.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import json +import os +import sys +from pathlib import Path + +env_name = sys.argv[1] +CONDA_DIR = os.environ["CONDA_DIR"] + +file = Path.home() / f".local/share/jupyter/kernels/{env_name}/kernel.json" +content = json.loads(file.read_text()) +content["env"] = { + "XML_CATALOG_FILES": "", + "PATH": f"{CONDA_DIR}/envs/{env_name}/bin:$PATH", + "CONDA_PREFIX": f"{CONDA_DIR}/envs/{env_name}", + "CONDA_PROMPT_MODIFIER": f"({env_name}) ", + "CONDA_SHLVL": "2", + "CONDA_DEFAULT_ENV": env_name, + "CONDA_PREFIX_1": CONDA_DIR, +} + +file.write_text(json.dumps(content, indent=1)) diff --git a/.build/setup-scripts/setup-julia-packages.bash b/.build/setup-scripts/setup-julia-packages.bash index faeee01..fa1421e 100755 --- a/.build/setup-scripts/setup-julia-packages.bash +++ b/.build/setup-scripts/setup-julia-packages.bash @@ -1,10 +1,32 @@ #!/bin/bash set -exuo pipefail # Requirements: -# - Run as non-root user +# - Run as a non-root user # - The JULIA_PKGDIR environment variable is set # - Julia is already set up, with the setup-julia.bash command + +# If we don't specify what CPUs the precompilation should be done for, it's +# *only* done for the target of the host doing the compilation. When the +# container runs on a host that's the same architecture, but a *different* +# generation of CPU than what the build host was, the precompilation is useless +# and Julia takes a long long time to start up. This specific multitarget comes +# from https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L20-L76, +# and may need to be updated as new CPU generations come out. +# If the architecture the container runs on is different, +# precompilation may still have to be re-done on first startup - but this +# *should* catch most of the issues. See +# https://github.com/jupyter/docker-stacks/issues/2015 for more information +if [ "$(uname -m)" == "x86_64" ]; then + # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L24 + # for an explanation of these options + export JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1)" +elif [ "$(uname -m)" == "aarch64" ]; then + # See https://github.com/JuliaCI/julia-buildkite/blob/70bde73f6cb17d4381b62236fc2d96b1c7acbba7/utilities/build_envs.sh#L54 + # for an explanation of these options + export JULIA_CPU_TARGET="generic;cortex-a57;thunderx2t99;carmel" +fi + # Install base Julia packages julia -e ' import Pkg; @@ -17,9 +39,9 @@ Pkg.add([ Pkg.precompile(); ' -# Move the kernelspec out to the system share location. Avoids -# problems with runtime UID change not taking effect properly on the -# .local folder in the jovyan home dir. move kernelspec out of home +# Move the kernelspec out of ${HOME} to the system share location. +# Avoids problems with runtime UID change not taking effect properly +# on the .local folder in the jovyan home dir. mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" chmod -R go+rx "${CONDA_DIR}/share/jupyter" rm -rf "${HOME}/.local" diff --git a/.build/setup-scripts/setup_julia.py b/.build/setup-scripts/setup_julia.py new file mode 100755 index 0000000..0cdbe0c --- /dev/null +++ b/.build/setup-scripts/setup_julia.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. + +# Requirements: +# - Run as the root user +# - The JULIA_PKGDIR environment variable is set + +import os +import platform +import shutil +import subprocess +from pathlib import Path + +import requests + + +def unify_aarch64(platform: str) -> str: + """ + Renames arm64->aarch64 to support local builds on on aarch64 Macs + """ + return { + "aarch64": "aarch64", + "arm64": "aarch64", + "x86_64": "x86_64", + }[platform] + + +def get_latest_julia_url() -> tuple[str, str]: + """ + Get the last stable version of Julia + Based on: https://github.com/JuliaLang/www.julialang.org/issues/878#issuecomment-749234813 + """ + + versions = requests.get( + "https://julialang-s3.julialang.org/bin/versions.json" + ).json() + stable_versions = {k: v for k, v in versions.items() if v["stable"]} + latest_version_files = stable_versions[max(stable_versions)]["files"] + triplet = unify_aarch64(platform.machine()) + "-linux-gnu" + file_info = [vf for vf in latest_version_files if vf["triplet"] == triplet][0] + return file_info["url"], file_info["version"] + + +def download_julia(julia_url: str) -> None: + """ + Downloads and unpacks julia + The resulting julia directory is "/opt/julia-VERSION/" + """ + tmp_file = Path("/tmp/julia.tar.gz") + subprocess.check_call( + ["curl", "--progress-bar", "--location", "--output", tmp_file, julia_url] + ) + shutil.unpack_archive(tmp_file, "/opt/") + tmp_file.unlink() + + +def prepare_julia(julia_version: str) -> None: + """ + Creates /usr/local/bin/julia symlink + Make Julia aware of conda libraries + Creates a directory for Julia user libraries + """ + # Link Julia installed version to /usr/local/bin, so julia launches it + subprocess.check_call( + ["ln", "-fs", f"/opt/julia-{julia_version}/bin/julia", "/usr/local/bin/julia"] + ) + + # Tell Julia where conda libraries are + Path("/etc/julia").mkdir() + Path("/etc/julia/juliarc.jl").write_text( + f'push!(Libdl.DL_LOAD_PATH, "{os.environ["CONDA_DIR"]}/lib")\n' + ) + + # Create JULIA_PKGDIR, where user libraries are installed + JULIA_PKGDIR = Path(os.environ["JULIA_PKGDIR"]) + JULIA_PKGDIR.mkdir() + subprocess.check_call(["chown", os.environ["NB_USER"], JULIA_PKGDIR]) + subprocess.check_call(["fix-permissions", JULIA_PKGDIR]) + + +if __name__ == "__main__": + julia_url, julia_version = get_latest_julia_url() + download_julia(julia_url=julia_url) + prepare_julia(julia_version=julia_version) diff --git a/.build/start-notebook.py b/.build/start-notebook.py new file mode 100755 index 0000000..b99ff31 --- /dev/null +++ b/.build/start-notebook.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +import shlex +import sys + +# If we are in a JupyterHub, we pass on to `start-singleuser.py` instead so it does the right thing +if "JUPYTERHUB_API_TOKEN" in os.environ: + print( + "WARNING: using start-singleuser.py instead of start-notebook.py to start a server associated with JupyterHub." + ) + command = ["/usr/local/bin/start-singleuser.py"] + sys.argv[1:] + os.execvp(command[0], command) + + +# Wrap everything in start.sh, no matter what +command = ["/usr/local/bin/start.sh"] + +# If we want to survive restarts, tell that to start.sh +if os.environ.get("RESTARTABLE") == "yes": + command.append("run-one-constantly") + +# We always launch a jupyter subcommand from this script +command.append("jupyter") + +# Launch the configured subcommand. Note that this should be a single string, so we don't split it +# We default to lab +jupyter_command = os.environ.get("DOCKER_STACKS_JUPYTER_CMD", "lab") +command.append(jupyter_command) + +# Append any optional NOTEBOOK_ARGS we were passed in. This is supposed to be multiple args passed +# on to the notebook command, so we split it correctly with shlex +if "NOTEBOOK_ARGS" in os.environ: + command += shlex.split(os.environ["NOTEBOOK_ARGS"]) + +# Pass through any other args we were passed on the command line +command += sys.argv[1:] + +# Execute the command! +os.execvp(command[0], command) diff --git a/.build/start-notebook.sh b/.build/start-notebook.sh index 4f673d2..c47ebba 100755 --- a/.build/start-notebook.sh +++ b/.build/start-notebook.sh @@ -1,22 +1,5 @@ #!/bin/bash -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. +# Shim to emit warning and call start-notebook.py +echo "WARNING: Use start-notebook.py instead" -set -e - -# The Jupyter command to launch -# JupyterLab by default -DOCKER_STACKS_JUPYTER_CMD="${DOCKER_STACKS_JUPYTER_CMD:=lab}" - -if [[ -n "${JUPYTERHUB_API_TOKEN}" ]]; then - echo "WARNING: using start-singleuser.sh instead of start-notebook.sh to start a server associated with JupyterHub." - exec /usr/local/bin/start-singleuser.sh "$@" -fi - -wrapper="" -if [[ "${RESTARTABLE}" == "yes" ]]; then - wrapper="run-one-constantly" -fi - -# shellcheck disable=SC1091,SC2086 -exec /usr/local/bin/start.sh ${wrapper} jupyter ${DOCKER_STACKS_JUPYTER_CMD} ${NOTEBOOK_ARGS} "$@" +exec /usr/local/bin/start-notebook.py "$@" diff --git a/.build/start-singleuser.py b/.build/start-singleuser.py new file mode 100755 index 0000000..2dcf6c0 --- /dev/null +++ b/.build/start-singleuser.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# Copyright (c) Jupyter Development Team. +# Distributed under the terms of the Modified BSD License. +import os +import shlex +import sys + +command = ["/usr/local/bin/start.sh", "jupyterhub-singleuser"] + +# set default ip to 0.0.0.0 +if "--ip=" not in os.environ.get("NOTEBOOK_ARGS", ""): + command.append("--ip=0.0.0.0") + +# Append any optional NOTEBOOK_ARGS we were passed in. This is supposed to be multiple args passed +# on to the notebook command, so we split it correctly with shlex +if "NOTEBOOK_ARGS" in os.environ: + command += shlex.split(os.environ["NOTEBOOK_ARGS"]) + +# Pass any other args we have been passed through +command += sys.argv[1:] + +# Execute the command! +os.execvp(command[0], command) diff --git a/.build/start-singleuser.sh b/.build/start-singleuser.sh index a2166e2..ecf0e06 100755 --- a/.build/start-singleuser.sh +++ b/.build/start-singleuser.sh @@ -1,13 +1,5 @@ #!/bin/bash -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. +# Shim to emit warning and call start-singleuser.py +echo "WARNING: Use start-singleuser.py instead" -set -e - -# set default ip to 0.0.0.0 -if [[ "${NOTEBOOK_ARGS} $*" != *"--ip="* ]]; then - NOTEBOOK_ARGS="--ip=0.0.0.0 ${NOTEBOOK_ARGS}" -fi - -# shellcheck disable=SC1091,SC2086 -. /usr/local/bin/start.sh jupyterhub-singleuser ${NOTEBOOK_ARGS} "$@" +exec /usr/local/bin/start-singleuser.py "$@" diff --git a/.build/start.sh b/.build/start.sh index 7c5859e..76419b6 100755 --- a/.build/start.sh +++ b/.build/start.sh @@ -4,9 +4,9 @@ set -e -# The _log function is used for everything this script wants to log. It will -# always log errors and warnings, but can be silenced for other messages -# by setting JUPYTER_DOCKER_STACKS_QUIET environment variable. +# The _log function is used for everything this script wants to log. +# It will always log errors and warnings but can be silenced for other messages +# by setting the JUPYTER_DOCKER_STACKS_QUIET environment variable. _log () { if [[ "$*" == "ERROR:"* ]] || [[ "$*" == "WARNING:"* ]] || [[ "${JUPYTER_DOCKER_STACKS_QUIET}" == "" ]]; then echo "$@" @@ -14,39 +14,12 @@ _log () { } _log "Entered start.sh with args:" "$@" -# The run-hooks function looks for .sh scripts to source and executable files to -# run within a passed directory. -run-hooks () { - if [[ ! -d "${1}" ]] ; then - return - fi - _log "${0}: running hooks in ${1} as uid / gid: $(id -u) / $(id -g)" - for f in "${1}/"*; do - case "${f}" in - *.sh) - _log "${0}: running script ${f}" - # shellcheck disable=SC1090 - source "${f}" - ;; - *) - if [[ -x "${f}" ]] ; then - _log "${0}: running executable ${f}" - "${f}" - else - _log "${0}: ignoring non-executable ${f}" - fi - ;; - esac - done - _log "${0}: done running hooks in ${1}" -} - # A helper function to unset env vars listed in the value of the env var # JUPYTER_ENV_VARS_TO_UNSET. unset_explicit_env_vars () { if [ -n "${JUPYTER_ENV_VARS_TO_UNSET}" ]; then for env_var_to_unset in $(echo "${JUPYTER_ENV_VARS_TO_UNSET}" | tr ',' ' '); do - echo "Unset ${env_var_to_unset} due to JUPYTER_ENV_VARS_TO_UNSET" + _log "Unset ${env_var_to_unset} due to JUPYTER_ENV_VARS_TO_UNSET" unset "${env_var_to_unset}" done unset JUPYTER_ENV_VARS_TO_UNSET @@ -62,14 +35,15 @@ else fi # NOTE: This hook will run as the user the container was started with! -run-hooks /usr/local/bin/start-notebook.d +# shellcheck disable=SC1091 +source /usr/local/bin/run-hooks.sh /usr/local/bin/start-notebook.d # If the container started as the root user, then we have permission to refit # the jovyan user, and ensure file permissions, grant sudo rights, and such # things before we run the command passed to start.sh as the desired user # (NB_USER). # -if [ "$(id -u)" == 0 ] ; then +if [ "$(id -u)" == 0 ]; then # Environment variables: # - NB_USER: the desired username and associated home folder # - NB_UID: the desired user id @@ -77,18 +51,18 @@ if [ "$(id -u)" == 0 ] ; then # - NB_GROUP: a group name we want for the group # - GRANT_SUDO: a boolean ("1" or "yes") to grant the user sudo rights # - CHOWN_HOME: a boolean ("1" or "yes") to chown the user's home folder - # - CHOWN_EXTRA: a comma separated list of paths to chown + # - CHOWN_EXTRA: a comma-separated list of paths to chown # - CHOWN_HOME_OPTS / CHOWN_EXTRA_OPTS: arguments to the chown commands - # Refit the jovyan user to the desired the user (NB_USER) - if id jovyan &> /dev/null ; then + # Refit the jovyan user to the desired user (NB_USER) + if id jovyan &> /dev/null; then if ! usermod --home "/home/${NB_USER}" --login "${NB_USER}" jovyan 2>&1 | grep "no changes" > /dev/null; then _log "Updated the jovyan user:" _log "- username: jovyan -> ${NB_USER}" _log "- home dir: /home/jovyan -> /home/${NB_USER}" fi elif ! id -u "${NB_USER}" &> /dev/null; then - _log "ERROR: Neither the jovyan user or '${NB_USER}' exists. This could be the result of stopping and starting, the container with a different NB_USER environment variable." + _log "ERROR: Neither the jovyan user nor '${NB_USER}' exists. This could be the result of stopping and starting, the container with a different NB_USER environment variable." exit 1 fi # Ensure the desired user (NB_USER) gets its desired user id (NB_UID) and is @@ -101,10 +75,10 @@ if [ "$(id -u)" == 0 ] ; then fi # Recreate the desired user as we want it userdel "${NB_USER}" - useradd --home "/home/${NB_USER}" --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 --no-log-init "${NB_USER}" + useradd --no-log-init --home "/home/${NB_USER}" --shell /bin/bash --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 "${NB_USER}" fi - # Move or symlink the jovyan home directory to the desired users home + # Move or symlink the jovyan home directory to the desired user's home # directory if it doesn't already exist, and update the current working # directory to the new location if needed. if [[ "${NB_USER}" != "jovyan" ]]; then @@ -132,7 +106,7 @@ if [ "$(id -u)" == 0 ] ; then fi fi - # Optionally ensure the desired user get filesystem ownership of it's home + # Optionally ensure the desired user gets filesystem ownership of its home # folder and/or additional folders if [[ "${CHOWN_HOME}" == "1" || "${CHOWN_HOME}" == "yes" ]]; then _log "Ensuring /home/${NB_USER} is owned by ${NB_UID}:${NB_GID} ${CHOWN_HOME_OPTS:+(chown options: ${CHOWN_HOME_OPTS})}" @@ -147,9 +121,6 @@ if [ "$(id -u)" == 0 ] ; then done fi - # Update potentially outdated environment variables since image build - export XDG_CACHE_HOME="/home/${NB_USER}/.cache" - # Prepend ${CONDA_DIR}/bin to sudo secure_path sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"${CONDA_DIR}/bin:\1\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path @@ -160,11 +131,13 @@ if [ "$(id -u)" == 0 ] ; then fi # NOTE: This hook is run as the root user! - run-hooks /usr/local/bin/before-notebook.d - + # shellcheck disable=SC1091 + source /usr/local/bin/run-hooks.sh /usr/local/bin/before-notebook.d unset_explicit_env_vars + _log "Running as ${NB_USER}:" "${cmd[@]}" exec sudo --preserve-env --set-home --user "${NB_USER}" \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}" \ PATH="${PATH}" \ PYTHONPATH="${PYTHONPATH:-}" \ "${cmd[@]}" @@ -178,7 +151,7 @@ if [ "$(id -u)" == 0 ] ; then # command. The behavior can be inspected with `sudo -V` run as root. # # ref: `man sudo` https://linux.die.net/man/8/sudo - # ref: `man sudoers` https://www.sudo.ws/man/1.8.15/sudoers.man.html + # ref: `man sudoers` https://www.sudo.ws/docs/man/sudoers.man/ # # - We use the `--preserve-env` flag to pass through most environment # variables, but understand that exceptions are caused by the sudoers @@ -187,10 +160,10 @@ if [ "$(id -u)" == 0 ] ; then # - We use the `--set-home` flag to set the HOME variable appropriately. # # - To reduce the default list of variables deleted by sudo, we could have - # used `env_delete` from /etc/sudoers. It has higher priority than the + # used `env_delete` from /etc/sudoers. It has a higher priority than the # `--preserve-env` flag and the `env_keep` configuration. # - # - We preserve PATH and PYTHONPATH explicitly. Note however that sudo + # - We preserve LD_LIBRARY_PATH, PATH and PYTHONPATH explicitly. Note however that sudo # resolves `${cmd[@]}` using the "secure_path" variable we modified # above in /etc/sudoers.d/path. Thus PATH is irrelevant to how the above # sudo command resolves the path of `${cmd[@]}`. The PATH will be relevant @@ -210,7 +183,7 @@ else # Attempt to ensure the user uid we currently run as has a named entry in # the /etc/passwd file, as it avoids software crashing on hard assumptions # on such entry. Writing to the /etc/passwd was allowed for the root group - # from the Dockerfile during build. + # from the Dockerfile during the build. # # ref: https://github.com/jupyter/docker-stacks/issues/552 if ! whoami &> /dev/null; then @@ -255,8 +228,10 @@ else fi # NOTE: This hook is run as the user we started the container as! - run-hooks /usr/local/bin/before-notebook.d + # shellcheck disable=SC1091 + source /usr/local/bin/run-hooks.sh /usr/local/bin/before-notebook.d unset_explicit_env_vars + _log "Executing the command:" "${cmd[@]}" exec "${cmd[@]}" fi diff --git a/generate-Dockerfile copy.sh b/generate-Dockerfile copy.sh new file mode 100755 index 0000000..1727373 --- /dev/null +++ b/generate-Dockerfile copy.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +cd $(cd -P -- "$(dirname -- "$0")" && pwd -P) + +# Set the path of the generated Dockerfile +export DOCKERFILE=".build/Dockerfile" +export STACKS_DIR=".build/docker-stacks" +# please test the build of the commit in https://github.com/jupyter/docker-stacks/commits/main in advance +export HEAD_COMMIT="b8d617dc0568d60f6583c42f989da51ec80e9af6" + +while [[ "$#" -gt 0 ]]; do case $1 in + -p|--pw|--password) PASSWORD="$2" && USE_PASSWORD=1; shift;; + -c|--commit) HEAD_COMMIT="$2"; shift;; + --no-datascience-notebook) no_datascience_notebook=1;; + --python-only) no_datascience_notebook=1;; + --no-useful-packages) no_useful_packages=1;; + -s|--slim) no_datascience_notebook=1 && no_useful_packages=1;; + -h|--help) HELP=1;; + *) echo "Unknown parameter passed: $1" && HELP=1;; +esac; shift; done + +if [[ "$HELP" == 1 ]]; then + echo "Help for ./generate-Dockerfile.sh:" + echo "Usage: $0 [parameters]" + echo " -h|--help: Show this help." + echo " -p|--pw|--password: Set the password (and update in src/jupyter_notebook_config.json)" + echo " -c|--commit: Set the head commit of the jupyter/docker-stacks submodule (https://github.com/jupyter/docker-stacks/commits/main). default: $HEAD_COMMIT." + echo " --python-only|--no-datascience-notebook: Use not the datascience-notebook from jupyter/docker-stacks, don't install Julia and R." + echo " --no-useful-packages: Don't install the useful packages, specified in src/Dockerfile.usefulpackages" + echo " --slim: no useful packages and no datascience notebook." + exit 21 +fi + +# Clone if docker-stacks doesn't exist, and set to the given commit or the default commit +ls $STACKS_DIR/README.md > /dev/null 2>&1 || (echo "Docker-stacks was not found, cloning repository" \ + && git clone https://github.com/jupyter/docker-stacks.git $STACKS_DIR) +echo "Set docker-stacks to commit '$HEAD_COMMIT'." +if [[ "$HEAD_COMMIT" == "latest" ]]; then + echo "WARNING, the latest commit of docker-stacks is used. This may result in version conflicts" + cd $STACKS_DIR && git pull && cd - +else + export GOT_HEAD="false" + cd $STACKS_DIR && git pull && git reset --hard "$HEAD_COMMIT" > /dev/null 2>&1 && cd - && export GOT_HEAD="true" + echo "$HEAD" + if [[ "$GOT_HEAD" == "false" ]]; then + echo "Error: The provided sha-commit is invalid." + echo "Usage: $0 -c [sha-commit] # set the head commit of the docker-stacks submodule (https://github.com/jupyter/docker-stacks/commits/master)." + echo "Exiting" + exit 2 + else + echo "Set head to given commit." + fi +fi + +# Write the contents into the DOCKERFILE and start with the header +echo "# This Dockerfile is generated by 'generate-Dockerfile.sh' from elements within 'src/' + +# **Please do not change this file directly!** +# To adapt this Dockerfile, adapt 'generate-Dockerfile.sh' or 'src/Dockerfile.usefulpackages'. +# More information can be found in the README under configuration. + +" > $DOCKERFILE +cat src/Dockerfile.header >> $DOCKERFILE + +echo " +############################################################################ +#################### Dependency: jupyter/docker-stacks-foundation ########## +############################################################################ +" >> $DOCKERFILE +cat $STACKS_DIR/docker-stacks-foundation/Dockerfile | grep -v 'BASE_CONTAINER' | grep -v 'FROM $ROOT_CONTAINER' >> $DOCKERFILE + +echo " +############################################################################ +#################### Dependency: jupyter/base-notebook ##################### +############################################################################ +" >> $DOCKERFILE +cat $STACKS_DIR/base-notebook/Dockerfile | grep -v 'BASE_CONTAINER' >> $DOCKERFILE + +# copy files that are used during the build: +cp $STACKS_DIR/docker-stacks-foundation/initial-condarc .build/ +cp $STACKS_DIR/docker-stacks-foundation/fix-permissions .build/ +cp $STACKS_DIR/docker-stacks-foundation/start.sh .build/ +cp $STACKS_DIR/base-notebook/jupyter_server_config.py .build/ +cp $STACKS_DIR/base-notebook/start-notebook.sh .build/ +cp $STACKS_DIR/base-notebook/start-singleuser.sh .build/ +cp $STACKS_DIR/base-notebook/docker_healthcheck.py .build/ +cp -r $STACKS_DIR/minimal-notebook/setup-scripts .build/ +cp $STACKS_DIR/minimal-notebook/Rprofile.site .build/ +chmod 755 .build/* + +echo " +############################################################################ +################# Dependency: jupyter/minimal-notebook ##################### +############################################################################ +" >> $DOCKERFILE +cat $STACKS_DIR/minimal-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + +echo " +############################################################################ +################# Dependency: jupyter/scipy-notebook ####################### +############################################################################ +" >> $DOCKERFILE +cat $STACKS_DIR/scipy-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + +# install Julia and R if not excluded or spare mode is used +if [[ "$no_datascience_notebook" != 1 ]]; then + echo " + ############################################################################ + ################ Dependency: jupyter/datascience-notebook ################## + ############################################################################ + " >> $DOCKERFILE + cat $STACKS_DIR/datascience-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE +else + echo "Set 'no-datascience-notebook' = 'python-only', not installing the datascience-notebook with Julia and R." +fi + +# Note that the following step also installs the cudatoolkit, which is +# essential to access the GPU. +echo " +############################################################################ +########################## Dependency: gpulibs ############################# +############################################################################ +" >> $DOCKERFILE +cat src/Dockerfile.gpulibs >> $DOCKERFILE + +# install useful packages if not excluded or spare mode is used +if [[ "$no_useful_packages" != 1 ]]; then + echo " + ############################################################################ + ############################ Useful packages ############################### + ############################################################################ + " >> $DOCKERFILE + cat src/Dockerfile.usefulpackages >> $DOCKERFILE +else + echo "Set 'no-useful-packages', not installing stuff within src/Dockerfile.usefulpackages." +fi + +# Copy the demo notebooks and change permissions +cp -r extra/Getting_Started data +chmod -R 755 data/ + +# set password +if [[ "$USE_PASSWORD" == 1 ]]; then + echo "Set password to given input" + SALT="3b4b6378355" + HASHED=$(echo -n ${PASSWORD}${SALT} | sha1sum | awk '{print $1}') + unset PASSWORD # delete variable PASSWORD + # build jupyter_notebook_config.json + echo "{ + \"NotebookApp\": { + \"password\": \"sha1:$SALT:$HASHED\" + } +}" > .build/jupyter_notebook_config.json + + # copy the config into .build and append the lines into the Dockerfile + echo >> $DOCKERFILE + echo "# Copy jupyter_notebook_config.json" >> $DOCKERFILE + echo "COPY jupyter_notebook_config.json /etc/jupyter/" >> $DOCKERFILE +fi + +# Set environment variables +export JUPYTER_UID=$(id -u) +export JUPYTER_GID=$(id -g) + +#cp $(find $(dirname $DOCKERFILE) -type f | grep -v $STACKS_DIR | grep -v .gitkeep) . +echo +echo "The GPU Dockerfile was generated successfully in file ${DOCKERFILE}." +echo "To start the GPU-based Juyterlab instance, run:" +echo " docker build -t gpu-jupyter .build/ # will take a while" +echo " docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes -e NB_UID=$(id -u) -e NB_GID=$(id -g) --user root --restart always --name gpu-jupyter_1 gpu-jupyter" diff --git a/generate-Dockerfile.sh b/generate-Dockerfile.sh index 1727373..19b762c 100755 --- a/generate-Dockerfile.sh +++ b/generate-Dockerfile.sh @@ -5,7 +5,7 @@ cd $(cd -P -- "$(dirname -- "$0")" && pwd -P) export DOCKERFILE=".build/Dockerfile" export STACKS_DIR=".build/docker-stacks" # please test the build of the commit in https://github.com/jupyter/docker-stacks/commits/main in advance -export HEAD_COMMIT="b8d617dc0568d60f6583c42f989da51ec80e9af6" +export HEAD_COMMIT="1494233e27cdc70e3766ea2518e7153ee425fc4f" while [[ "$#" -gt 0 ]]; do case $1 in -p|--pw|--password) PASSWORD="$2" && USE_PASSWORD=1; shift;; @@ -66,25 +66,44 @@ echo " #################### Dependency: jupyter/docker-stacks-foundation ########## ############################################################################ " >> $DOCKERFILE -cat $STACKS_DIR/docker-stacks-foundation/Dockerfile | grep -v 'BASE_CONTAINER' | grep -v 'FROM $ROOT_CONTAINER' >> $DOCKERFILE +if [ -f "$STACKS_DIR/images/docker-stacks-foundation/Dockerfile" ]; then + cat $STACKS_DIR/images/docker-stacks-foundation/Dockerfile | grep -v 'BASE_CONTAINER' | grep -v 'FROM $ROOT_CONTAINER' >> $DOCKERFILE + # copy files that are used during the build + cp $STACKS_DIR/images/docker-stacks-foundation/initial-condarc .build/ + cp $STACKS_DIR/images/docker-stacks-foundation/fix-permissions .build/ + cp $STACKS_DIR/images/docker-stacks-foundation/start.sh .build/ + cp $STACKS_DIR/images/docker-stacks-foundation/run-hooks.sh .build/ + cp $STACKS_DIR/images/docker-stacks-foundation/10activate-conda-env.sh .build/ +else + cat $STACKS_DIR/docker-stacks-foundation/Dockerfile | grep -v 'BASE_CONTAINER' | grep -v 'FROM $ROOT_CONTAINER' >> $DOCKERFILE + # copy files that are used during the build + cp $STACKS_DIR/docker-stacks-foundation/initial-condarc .build/ + cp $STACKS_DIR/docker-stacks-foundation/fix-permissions .build/ + cp $STACKS_DIR/docker-stacks-foundation/start.sh .build/ +fi echo " ############################################################################ #################### Dependency: jupyter/base-notebook ##################### ############################################################################ " >> $DOCKERFILE -cat $STACKS_DIR/base-notebook/Dockerfile | grep -v 'BASE_CONTAINER' >> $DOCKERFILE - -# copy files that are used during the build: -cp $STACKS_DIR/docker-stacks-foundation/initial-condarc .build/ -cp $STACKS_DIR/docker-stacks-foundation/fix-permissions .build/ -cp $STACKS_DIR/docker-stacks-foundation/start.sh .build/ -cp $STACKS_DIR/base-notebook/jupyter_server_config.py .build/ -cp $STACKS_DIR/base-notebook/start-notebook.sh .build/ -cp $STACKS_DIR/base-notebook/start-singleuser.sh .build/ -cp $STACKS_DIR/base-notebook/docker_healthcheck.py .build/ -cp -r $STACKS_DIR/minimal-notebook/setup-scripts .build/ -cp $STACKS_DIR/minimal-notebook/Rprofile.site .build/ +if [ -f "$STACKS_DIR/images/base-notebook/Dockerfile" ]; then + cat $STACKS_DIR/images/base-notebook/Dockerfile | grep -v 'BASE_CONTAINER' >> $DOCKERFILE + # copy files that are used during the build + cp $STACKS_DIR/images/base-notebook/jupyter_server_config.py .build/ + cp $STACKS_DIR/images/base-notebook/start-notebook.sh .build/ + cp $STACKS_DIR/images/base-notebook/start-notebook.py .build/ + cp $STACKS_DIR/images/base-notebook/start-singleuser.sh .build/ + cp $STACKS_DIR/images/base-notebook/start-singleuser.py .build/ + cp $STACKS_DIR/images/base-notebook/docker_healthcheck.py .build/ +else + cat $STACKS_DIR/base-notebook/Dockerfile | grep -v 'BASE_CONTAINER' >> $DOCKERFILE + # copy files that are used during the build + cp $STACKS_DIR/base-notebook/jupyter_server_config.py .build/ + cp $STACKS_DIR/base-notebook/start-notebook.sh .build/ + cp $STACKS_DIR/base-notebook/start-singleuser.sh .build/ + cp $STACKS_DIR/base-notebook/docker_healthcheck.py .build/ +fi chmod 755 .build/* echo " @@ -92,14 +111,28 @@ echo " ################# Dependency: jupyter/minimal-notebook ##################### ############################################################################ " >> $DOCKERFILE -cat $STACKS_DIR/minimal-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE +if [ -f "$STACKS_DIR/images/minimal-notebook/Dockerfile" ]; then + cat $STACKS_DIR/images/minimal-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + # copy files that are used during the build + cp -r $STACKS_DIR/images/minimal-notebook/setup-scripts .build/ + cp $STACKS_DIR/images/minimal-notebook/Rprofile.site .build/ +else + cat $STACKS_DIR/minimal-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + # copy files that are used during the build + cp -r $STACKS_DIR/minimal-notebook/setup-scripts .build/ + cp $STACKS_DIR/minimal-notebook/Rprofile.site .build/ +fi echo " ############################################################################ ################# Dependency: jupyter/scipy-notebook ####################### ############################################################################ " >> $DOCKERFILE -cat $STACKS_DIR/scipy-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE +if [ -f "$STACKS_DIR/images/scipy-notebook/Dockerfile" ]; then + cat $STACKS_DIR/images/scipy-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE +else + cat $STACKS_DIR/scipy-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE +fi # install Julia and R if not excluded or spare mode is used if [[ "$no_datascience_notebook" != 1 ]]; then @@ -108,7 +141,11 @@ if [[ "$no_datascience_notebook" != 1 ]]; then ################ Dependency: jupyter/datascience-notebook ################## ############################################################################ " >> $DOCKERFILE - cat $STACKS_DIR/datascience-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + if [ -f "$STACKS_DIR/images/datascience-notebook/Dockerfile" ]; then + cat $STACKS_DIR/images/datascience-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + else + cat $STACKS_DIR/images/datascience-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE + fi else echo "Set 'no-datascience-notebook' = 'python-only', not installing the datascience-notebook with Julia and R." fi diff --git a/src/Dockerfile.gpulibs b/src/Dockerfile.gpulibs index a352c4f..de68319 100644 --- a/src/Dockerfile.gpulibs +++ b/src/Dockerfile.gpulibs @@ -4,12 +4,6 @@ LABEL maintainer="Christoph Schranz , Mat # https://www.tensorflow.org/install/source#gpu # installation via conda leads to errors in version 4.8.2 # Install CUDA-specific nvidia libraries and update libcudnn8 before that -USER root -RUN apt-get update && \ - apt-get install -y --no-install-recommends --allow-change-held-packages libcudnn8 && \ - apt-get install -y --no-install-recommends libnvinfer-dev libnvinfer-plugin-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists/* -RUN cd /usr/lib/x86_64-linux-gnu && ln -s libnvinfer_plugin.so.8 libnvinfer_plugin.so.7 && ln -s libnvinfer.so.8 libnvinfer.so.7 USER ${NB_UID} RUN pip install --upgrade pip && \ pip install --no-cache-dir tensorflow==2.15.0 keras==2.15.0 && \ diff --git a/src/Dockerfile.header b/src/Dockerfile.header index a4d9889..abafde3 100644 --- a/src/Dockerfile.header +++ b/src/Dockerfile.header @@ -1,6 +1,6 @@ # Use NVIDIA CUDA as base image and run the same installation as in the other packages. # The version of cuda must match those of the packages installed in src/Dockerfile.gpulibs -FROM nvidia/cuda:12.0.1-cudnn8-runtime-ubuntu22.04 +FROM nvidia/cuda:11.6.2-cudnn8-runtime-ubuntu20.04 LABEL authors="Christoph Schranz , Mathematical Michael " # This is a concatenated Dockerfile, the maintainers of subsequent sections may vary. RUN chmod 1777 /tmp && chmod 1777 /var/tmp From 61df2836e3840a86216b8e41af3572dde81ca8df Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Sat, 30 Dec 2023 23:35:13 +0100 Subject: [PATCH 2/6] update docker stacks parsing --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 057387c..01e5b2f 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,9 @@ Additionally, data within the host's `data` directory is shared with the contain - `v1.5_cuda-11.8_ubuntu-22.04` (full image) - `v1.5_cuda-11.8_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) - `v1.5_cuda-11.8_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) + - `v1.6_cuda-11.6_ubuntu-20.04` (full image) + - `v1.6_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.6_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) - `v1.5_cuda-11.6_ubuntu-20.04` (full image) - `v1.5_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) - `v1.5_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) From 1620a88ff3fddee04de3bf202aea1bed3d935e7e Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Sun, 31 Dec 2023 09:27:45 +0100 Subject: [PATCH 3/6] removed unused bashscript --- generate-Dockerfile copy.sh | 169 ------------------------------------ 1 file changed, 169 deletions(-) delete mode 100755 generate-Dockerfile copy.sh diff --git a/generate-Dockerfile copy.sh b/generate-Dockerfile copy.sh deleted file mode 100755 index 1727373..0000000 --- a/generate-Dockerfile copy.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env bash -cd $(cd -P -- "$(dirname -- "$0")" && pwd -P) - -# Set the path of the generated Dockerfile -export DOCKERFILE=".build/Dockerfile" -export STACKS_DIR=".build/docker-stacks" -# please test the build of the commit in https://github.com/jupyter/docker-stacks/commits/main in advance -export HEAD_COMMIT="b8d617dc0568d60f6583c42f989da51ec80e9af6" - -while [[ "$#" -gt 0 ]]; do case $1 in - -p|--pw|--password) PASSWORD="$2" && USE_PASSWORD=1; shift;; - -c|--commit) HEAD_COMMIT="$2"; shift;; - --no-datascience-notebook) no_datascience_notebook=1;; - --python-only) no_datascience_notebook=1;; - --no-useful-packages) no_useful_packages=1;; - -s|--slim) no_datascience_notebook=1 && no_useful_packages=1;; - -h|--help) HELP=1;; - *) echo "Unknown parameter passed: $1" && HELP=1;; -esac; shift; done - -if [[ "$HELP" == 1 ]]; then - echo "Help for ./generate-Dockerfile.sh:" - echo "Usage: $0 [parameters]" - echo " -h|--help: Show this help." - echo " -p|--pw|--password: Set the password (and update in src/jupyter_notebook_config.json)" - echo " -c|--commit: Set the head commit of the jupyter/docker-stacks submodule (https://github.com/jupyter/docker-stacks/commits/main). default: $HEAD_COMMIT." - echo " --python-only|--no-datascience-notebook: Use not the datascience-notebook from jupyter/docker-stacks, don't install Julia and R." - echo " --no-useful-packages: Don't install the useful packages, specified in src/Dockerfile.usefulpackages" - echo " --slim: no useful packages and no datascience notebook." - exit 21 -fi - -# Clone if docker-stacks doesn't exist, and set to the given commit or the default commit -ls $STACKS_DIR/README.md > /dev/null 2>&1 || (echo "Docker-stacks was not found, cloning repository" \ - && git clone https://github.com/jupyter/docker-stacks.git $STACKS_DIR) -echo "Set docker-stacks to commit '$HEAD_COMMIT'." -if [[ "$HEAD_COMMIT" == "latest" ]]; then - echo "WARNING, the latest commit of docker-stacks is used. This may result in version conflicts" - cd $STACKS_DIR && git pull && cd - -else - export GOT_HEAD="false" - cd $STACKS_DIR && git pull && git reset --hard "$HEAD_COMMIT" > /dev/null 2>&1 && cd - && export GOT_HEAD="true" - echo "$HEAD" - if [[ "$GOT_HEAD" == "false" ]]; then - echo "Error: The provided sha-commit is invalid." - echo "Usage: $0 -c [sha-commit] # set the head commit of the docker-stacks submodule (https://github.com/jupyter/docker-stacks/commits/master)." - echo "Exiting" - exit 2 - else - echo "Set head to given commit." - fi -fi - -# Write the contents into the DOCKERFILE and start with the header -echo "# This Dockerfile is generated by 'generate-Dockerfile.sh' from elements within 'src/' - -# **Please do not change this file directly!** -# To adapt this Dockerfile, adapt 'generate-Dockerfile.sh' or 'src/Dockerfile.usefulpackages'. -# More information can be found in the README under configuration. - -" > $DOCKERFILE -cat src/Dockerfile.header >> $DOCKERFILE - -echo " -############################################################################ -#################### Dependency: jupyter/docker-stacks-foundation ########## -############################################################################ -" >> $DOCKERFILE -cat $STACKS_DIR/docker-stacks-foundation/Dockerfile | grep -v 'BASE_CONTAINER' | grep -v 'FROM $ROOT_CONTAINER' >> $DOCKERFILE - -echo " -############################################################################ -#################### Dependency: jupyter/base-notebook ##################### -############################################################################ -" >> $DOCKERFILE -cat $STACKS_DIR/base-notebook/Dockerfile | grep -v 'BASE_CONTAINER' >> $DOCKERFILE - -# copy files that are used during the build: -cp $STACKS_DIR/docker-stacks-foundation/initial-condarc .build/ -cp $STACKS_DIR/docker-stacks-foundation/fix-permissions .build/ -cp $STACKS_DIR/docker-stacks-foundation/start.sh .build/ -cp $STACKS_DIR/base-notebook/jupyter_server_config.py .build/ -cp $STACKS_DIR/base-notebook/start-notebook.sh .build/ -cp $STACKS_DIR/base-notebook/start-singleuser.sh .build/ -cp $STACKS_DIR/base-notebook/docker_healthcheck.py .build/ -cp -r $STACKS_DIR/minimal-notebook/setup-scripts .build/ -cp $STACKS_DIR/minimal-notebook/Rprofile.site .build/ -chmod 755 .build/* - -echo " -############################################################################ -################# Dependency: jupyter/minimal-notebook ##################### -############################################################################ -" >> $DOCKERFILE -cat $STACKS_DIR/minimal-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE - -echo " -############################################################################ -################# Dependency: jupyter/scipy-notebook ####################### -############################################################################ -" >> $DOCKERFILE -cat $STACKS_DIR/scipy-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE - -# install Julia and R if not excluded or spare mode is used -if [[ "$no_datascience_notebook" != 1 ]]; then - echo " - ############################################################################ - ################ Dependency: jupyter/datascience-notebook ################## - ############################################################################ - " >> $DOCKERFILE - cat $STACKS_DIR/datascience-notebook/Dockerfile | grep -v BASE_CONTAINER >> $DOCKERFILE -else - echo "Set 'no-datascience-notebook' = 'python-only', not installing the datascience-notebook with Julia and R." -fi - -# Note that the following step also installs the cudatoolkit, which is -# essential to access the GPU. -echo " -############################################################################ -########################## Dependency: gpulibs ############################# -############################################################################ -" >> $DOCKERFILE -cat src/Dockerfile.gpulibs >> $DOCKERFILE - -# install useful packages if not excluded or spare mode is used -if [[ "$no_useful_packages" != 1 ]]; then - echo " - ############################################################################ - ############################ Useful packages ############################### - ############################################################################ - " >> $DOCKERFILE - cat src/Dockerfile.usefulpackages >> $DOCKERFILE -else - echo "Set 'no-useful-packages', not installing stuff within src/Dockerfile.usefulpackages." -fi - -# Copy the demo notebooks and change permissions -cp -r extra/Getting_Started data -chmod -R 755 data/ - -# set password -if [[ "$USE_PASSWORD" == 1 ]]; then - echo "Set password to given input" - SALT="3b4b6378355" - HASHED=$(echo -n ${PASSWORD}${SALT} | sha1sum | awk '{print $1}') - unset PASSWORD # delete variable PASSWORD - # build jupyter_notebook_config.json - echo "{ - \"NotebookApp\": { - \"password\": \"sha1:$SALT:$HASHED\" - } -}" > .build/jupyter_notebook_config.json - - # copy the config into .build and append the lines into the Dockerfile - echo >> $DOCKERFILE - echo "# Copy jupyter_notebook_config.json" >> $DOCKERFILE - echo "COPY jupyter_notebook_config.json /etc/jupyter/" >> $DOCKERFILE -fi - -# Set environment variables -export JUPYTER_UID=$(id -u) -export JUPYTER_GID=$(id -g) - -#cp $(find $(dirname $DOCKERFILE) -type f | grep -v $STACKS_DIR | grep -v .gitkeep) . -echo -echo "The GPU Dockerfile was generated successfully in file ${DOCKERFILE}." -echo "To start the GPU-based Juyterlab instance, run:" -echo " docker build -t gpu-jupyter .build/ # will take a while" -echo " docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes -e NB_UID=$(id -u) -e NB_GID=$(id -g) --user root --restart always --name gpu-jupyter_1 gpu-jupyter" From 13941fe94bc14e7df5fc191e81abe23c0b7c9b8b Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Sun, 31 Dec 2023 10:13:24 +0100 Subject: [PATCH 4/6] updated to cuda 11.8 and restructured customized Dockerfiles --- .build/Dockerfile | 72 +------------------ README.md | 22 +++--- .../gpulibs.Dockerfile | 0 .../header.Dockerfile | 2 +- .../usefulpackages.Dockerfile | 0 generate-Dockerfile.sh | 16 ++--- 6 files changed, 23 insertions(+), 89 deletions(-) rename src/Dockerfile.gpulibs => custom/gpulibs.Dockerfile (100%) rename src/Dockerfile.header => custom/header.Dockerfile (91%) rename src/Dockerfile.usefulpackages => custom/usefulpackages.Dockerfile (100%) diff --git a/.build/Dockerfile b/.build/Dockerfile index 0fb0e77..2629221 100755 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -1,13 +1,13 @@ -# This Dockerfile is generated by 'generate-Dockerfile.sh' from elements within 'src/' +# This Dockerfile is generated by 'generate-Dockerfile.sh' from elements within 'custom/' # **Please do not change this file directly!** -# To adapt this Dockerfile, adapt 'generate-Dockerfile.sh' or 'src/Dockerfile.usefulpackages'. +# To adapt this Dockerfile, adapt 'generate-Dockerfile.sh' or 'custom/usefulpackages.Dockerfile'. # More information can be found in the README under configuration. # Use NVIDIA CUDA as base image and run the same installation as in the other packages. # The version of cuda must match those of the packages installed in src/Dockerfile.gpulibs -FROM nvidia/cuda:11.6.2-cudnn8-runtime-ubuntu20.04 +FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04 LABEL authors="Christoph Schranz , Mathematical Michael " # This is a concatenated Dockerfile, the maintainers of subsequent sections may vary. RUN chmod 1777 /tmp && chmod 1777 /var/tmp @@ -377,72 +377,6 @@ USER ${NB_UID} WORKDIR "${HOME}" - ############################################################################ - ################ Dependency: jupyter/datascience-notebook ################## - ############################################################################ - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -ARG REGISTRY=quay.io -ARG OWNER=jupyter - -LABEL maintainer="Jupyter Project " - -# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 -# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -USER root - -# R pre-requisites -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - fonts-dejavu \ - gfortran \ - gcc && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Julia dependencies -# install Julia packages in /opt/julia instead of ${HOME} -ENV JULIA_DEPOT_PATH=/opt/julia \ - JULIA_PKGDIR=/opt/julia - -# Setup Julia -RUN /opt/setup-scripts/setup_julia.py - -USER ${NB_UID} - -# Setup IJulia kernel & other packages -RUN /opt/setup-scripts/setup-julia-packages.bash - -# R packages including IRKernel which gets installed globally. -# r-e1071: dependency of the caret R package -RUN mamba install --yes \ - 'r-base' \ - 'r-caret' \ - 'r-crayon' \ - 'r-devtools' \ - 'r-e1071' \ - 'r-forecast' \ - 'r-hexbin' \ - 'r-htmltools' \ - 'r-htmlwidgets' \ - 'r-irkernel' \ - 'r-nycflights13' \ - 'r-randomforest' \ - 'r-rcurl' \ - 'r-rmarkdown' \ - 'r-rodbc' \ - 'r-rsqlite' \ - 'r-shiny' \ - 'r-tidymodels' \ - 'r-tidyverse' \ - 'rpy2' \ - 'unixodbc' && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - ############################################################################ ########################## Dependency: gpulibs ############################# ############################################################################ diff --git a/README.md b/README.md index 01e5b2f..f4a98d5 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ we recommend checking out this [tutorial](https://www.youtube.com/watch?v=7wfPqA ## Build Your Image -Building a custom Docker image is the recommended option if you have a different GPU architecture or if you want to customize the pre-installed packages. The Dockerfiles in `src/` can be modified to achieve this. To use a custom base image, modify `src/Dockerfile.header`. To install specific GPU-related libraries, modify `src/Dockerfile.gpulibs`, and to add specific libraries, append them to `src/Dockerfile.usefulpackages`. +Building a custom Docker image is the recommended option if you have a different GPU architecture or if you want to customize the pre-installed packages. The Dockerfiles in `custom/` can be modified to achieve this. To use a custom base image, modify `custom/header.Dockerfile`. To install specific GPU-related libraries, modify `custom/gpulibs.Dockerfile`, and to add specific libraries, append them to `custom/usefulpackages.Dockerfile`. After making the necessary modifications, regenerate the `Dockerfile` in `/.build`. Once you have confirmed that your GPU is accessible within Docker containers by running `docker run --gpus all nvidia/cuda:12.0.1-cudnn8-runtime-ubuntu22.04 nvidia-smi` and seeing the GPU statistics, you can generate, build, and run the Docker image. The following commands will start *GPU-Jupyter* on [localhost:8848](http://localhost:8848) with the default password `gpu-jupyter`. @@ -191,7 +191,7 @@ The script has the following parameters: * `-h|--help`: Show a help message. * `-p|--pw|--password`: Set the password for *GPU-Jupyter* by updating - the salted hashed token in `src/jupyter_notebook_config.json`. + the salted hashed token in `custom/jupyter_notebook_config.json`. * `-c|--commit`: specify a commit or `"latest"` for the `docker-stacks`, the default commit is a working one. @@ -201,22 +201,22 @@ As some installations are not needed by everyone, there is the possibility to sk installations to reduce the size of the image. Here the `docker-stack` `scipy-notebook` is used instead of `datascience-notebook` that comes with Julia and R. -Moreover, none of the packages within `src/Dockerfile.usefulpackages` is installed. +Moreover, none of the packages within `custom/usefulpackages.Dockerfile` is installed. * `--python-only|--no-datascience-notebook`: As the name suggests, the `docker-stack` `datascience-notebook` is not installed -on top of the `scipy-notebook`, but the packages within `src/Dockerfile.usefulpackages` are. +on top of the `scipy-notebook`, but the packages within `custom/usefulpackages.Dockerfile` are. * `--no-useful-packages`: On top of the `docker-stack` `datascience-notebook` (Julia and R), -the essential `gpulibs` are installed, but not the packages within `src/Dockerfile.usefulpackages`. +the essential `gpulibs` are installed, but not the packages within `custom/usefulpackages.Dockerfile`. Note that only one of the parameters `--slim`, `--python-only`, and `--no-useful-packages` can be used at the same time: ### Custom Installations -If you need to install custom packages within the container, you can modify the `src/Dockerfile.usefulpackages` file or do it directly within the container. -**Keep in mind that every time a Dockerfile is generated, the file `.build/Dockerfile` is overwritten, so it's best to append custom installations in `src/Dockerfile.usefulpackages` or `generate-Dockerfile.sh`.** +If you need to install custom packages within the container, you can modify the `custom/usefulpackages.Dockerfile` file or do it directly within the container. +**Keep in mind that every time a Dockerfile is generated, the file `.build/Dockerfile` is overwritten, so it's best to append custom installations in `custom/usefulpackages.Dockerfile` or `generate-Dockerfile.sh`.** Some useful packages are suggested in the [Extension docs](https://jupyterlab.readthedocs.io/en/stable/user/extensions.html) and in this blog article from [neptune.ai](https://neptune.ai/blog/jupyterlab-extensions-for-machine-learning). If you notice that an important package is missing in the default stack, please let us know so we can update it. @@ -233,7 +233,7 @@ There are two ways to set a password for GPU-Jupyter: ```bash bash generate-Dockerfile.sh --password [your_password] ``` - This will update automatically the salted hashed token in the `src/jupyter_notebook_config.json` file. Note that the specified password may be visible in your account's bash history. + This will update automatically the salted hashed token in the `custom/jupyter_notebook_config.json` file. Note that the specified password may be visible in your account's bash history. ### Adaptions for using Tensorboard @@ -260,14 +260,14 @@ If the port is exposed, tensorboard can be accessed in the browser on [localhost #### Update CUDA to another version -The GPU-libraries such as PyTorch and Tensorflow in `src/Docker.gpulibs` must support the CUDA version and NVIDIA drivers on the host machine. Check out the compatibility lists for [PyTorch](https://pytorch.org/get-started/locally/) and [Tensorflow](https://www.tensorflow.org/install/source#gpu) or search online for the explicit versions. In this setup, the NVIDIA Driver has version 530.30.02 and CUDA version 11.6.2 is used, which is compatible with Tensorflow 2.10 and PyTorch 1.12. +The GPU-libraries such as PyTorch and Tensorflow in `custom/Docker.gpulibs` must support the CUDA version and NVIDIA drivers on the host machine. Check out the compatibility lists for [PyTorch](https://pytorch.org/get-started/locally/) and [Tensorflow](https://www.tensorflow.org/install/source#gpu) or search online for the explicit versions. In this setup, the NVIDIA Driver has version 530.30.02 and CUDA version 11.6.2 is used, which is compatible with Tensorflow 2.10 and PyTorch 1.12. -The host's CUDA version must be equal to or higher than that used by the container (set within `Dockerfile.header`). +The host's CUDA version must be equal to or higher than that used by the container (set within `custom/header.Dockerfile`). Check the host's version with `nvcc --version` and the version compatibilities for CUDA-dependent packages as [Pytorch](https://pytorch.org/get-started/locally/) respectively [Tensorflow](https://www.tensorflow.org/install/gpu) previously. Then modify, if supported, the CUDA-version (find all tags [here](https://hub.docker.com/r/nvidia/cuda/tags)) -in `src/Dockerfile.header` to, e.g.: +in `custom/header.Dockerfile` to, e.g.: the line: FROM nvidia/cuda:X.Y-base-ubuntu20.04 diff --git a/src/Dockerfile.gpulibs b/custom/gpulibs.Dockerfile similarity index 100% rename from src/Dockerfile.gpulibs rename to custom/gpulibs.Dockerfile diff --git a/src/Dockerfile.header b/custom/header.Dockerfile similarity index 91% rename from src/Dockerfile.header rename to custom/header.Dockerfile index abafde3..5d59d2b 100644 --- a/src/Dockerfile.header +++ b/custom/header.Dockerfile @@ -1,6 +1,6 @@ # Use NVIDIA CUDA as base image and run the same installation as in the other packages. # The version of cuda must match those of the packages installed in src/Dockerfile.gpulibs -FROM nvidia/cuda:11.6.2-cudnn8-runtime-ubuntu20.04 +FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04 LABEL authors="Christoph Schranz , Mathematical Michael " # This is a concatenated Dockerfile, the maintainers of subsequent sections may vary. RUN chmod 1777 /tmp && chmod 1777 /var/tmp diff --git a/src/Dockerfile.usefulpackages b/custom/usefulpackages.Dockerfile similarity index 100% rename from src/Dockerfile.usefulpackages rename to custom/usefulpackages.Dockerfile diff --git a/generate-Dockerfile.sh b/generate-Dockerfile.sh index 19b762c..dbcea37 100755 --- a/generate-Dockerfile.sh +++ b/generate-Dockerfile.sh @@ -22,10 +22,10 @@ if [[ "$HELP" == 1 ]]; then echo "Help for ./generate-Dockerfile.sh:" echo "Usage: $0 [parameters]" echo " -h|--help: Show this help." - echo " -p|--pw|--password: Set the password (and update in src/jupyter_notebook_config.json)" + echo " -p|--pw|--password: Set the password (and update in custom/jupyter_notebook_config.json)" echo " -c|--commit: Set the head commit of the jupyter/docker-stacks submodule (https://github.com/jupyter/docker-stacks/commits/main). default: $HEAD_COMMIT." echo " --python-only|--no-datascience-notebook: Use not the datascience-notebook from jupyter/docker-stacks, don't install Julia and R." - echo " --no-useful-packages: Don't install the useful packages, specified in src/Dockerfile.usefulpackages" + echo " --no-useful-packages: Don't install the useful packages, specified in custom/usefulpackages.Dockerfile" echo " --slim: no useful packages and no datascience notebook." exit 21 fi @@ -52,14 +52,14 @@ else fi # Write the contents into the DOCKERFILE and start with the header -echo "# This Dockerfile is generated by 'generate-Dockerfile.sh' from elements within 'src/' +echo "# This Dockerfile is generated by 'generate-Dockerfile.sh' from elements within 'custom/' # **Please do not change this file directly!** -# To adapt this Dockerfile, adapt 'generate-Dockerfile.sh' or 'src/Dockerfile.usefulpackages'. +# To adapt this Dockerfile, adapt 'generate-Dockerfile.sh' or 'custom/usefulpackages.Dockerfile'. # More information can be found in the README under configuration. " > $DOCKERFILE -cat src/Dockerfile.header >> $DOCKERFILE +cat custom/header.Dockerfile >> $DOCKERFILE echo " ############################################################################ @@ -157,7 +157,7 @@ echo " ########################## Dependency: gpulibs ############################# ############################################################################ " >> $DOCKERFILE -cat src/Dockerfile.gpulibs >> $DOCKERFILE +cat custom/gpulibs.Dockerfile >> $DOCKERFILE # install useful packages if not excluded or spare mode is used if [[ "$no_useful_packages" != 1 ]]; then @@ -166,9 +166,9 @@ if [[ "$no_useful_packages" != 1 ]]; then ############################ Useful packages ############################### ############################################################################ " >> $DOCKERFILE - cat src/Dockerfile.usefulpackages >> $DOCKERFILE + cat custom/usefulpackages.Dockerfile >> $DOCKERFILE else - echo "Set 'no-useful-packages', not installing stuff within src/Dockerfile.usefulpackages." + echo "Set 'no-useful-packages', not installing stuff within custom/usefulpackages.Dockerfile." fi # Copy the demo notebooks and change permissions From 91ba354a601440447c4e344fc92176b45b3abf1d Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Sun, 31 Dec 2023 12:48:29 +0100 Subject: [PATCH 5/6] adding version 1.6 --- README.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f4a98d5..e4b5ae8 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,9 @@ for creating and maintaining a robust Python, R, and Julia toolstack for Data Sc ```bash cd your-working-directory ll data # this path will be mounted by default - docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.5_cuda-12.0_ubuntu-22.04_python-only + docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.6_cuda-11.8_ubuntu-22.04_python-only ``` - This starts an instance of *GPU-Jupyter* with the tag `v1.5_cuda-12.0_ubuntu-22.04_python-only` at [http://localhost:8848](http://localhost:8848) (port `8848`). + This starts an instance of *GPU-Jupyter* with the tag `v1.6_cuda-11.8_ubuntu-22.04_python-only` at [http://localhost:8848](http://localhost:8848) (port `8848`). To log into Jupyterlab, you have to specify a token that you get from: ```bash docker exec -it [container-ID/name] jupyter server list @@ -82,15 +82,18 @@ for creating and maintaining a robust Python, R, and Julia toolstack for Data Sc Additionally, data within the host's `data` directory is shared with the container. Note that the following images of GPU-Jupyter are available on [Dockerhub](https://hub.docker.com/r/cschranz/gpu-jupyter): + - `v1.6_cuda-11.8_ubuntu-22.04` (full image) + - `v1.6_cuda-11.8_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.6_cuda-11.8_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) + - `v1.6_cuda-11.6_ubuntu-20.04` (full image) + - `v1.6_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.6_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) - `v1.5_cuda-12.0_ubuntu-22.04` (full image) - `v1.5_cuda-12.0_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) - `v1.5_cuda-12.0_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) - `v1.5_cuda-11.8_ubuntu-22.04` (full image) - `v1.5_cuda-11.8_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) - `v1.5_cuda-11.8_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) - - `v1.6_cuda-11.6_ubuntu-20.04` (full image) - - `v1.6_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) - - `v1.6_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) - `v1.5_cuda-11.6_ubuntu-20.04` (full image) - `v1.5_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) - `v1.5_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) @@ -110,7 +113,7 @@ Additionally, data within the host's `data` directory is shared with the contain - `v1.4_cuda-10.1_ubuntu-18.04_python-only` (only with a python interpreter and without Julia and R) - `v1.4_cuda-10.1_ubuntu-18.04_slim` (only with a python interpreter and without additional packages) - The version, e.g. `v1.5`, declares the version of the generator setup. + The version, e.g. `v1.6`, declares the version of the generator setup. The Cuda version, e.g. `cuda-12.0`, must match the CUDA driver version and be supported by the GPU libraries. These and older versions of GPU-Jupyter are listed on [Dockerhub](https://hub.docker.com/r/cschranz/gpu-jupyter/tags?page=1&ordering=last_updated). In case you are using another version or the GPU libraries don't work on your hardware, please try to build the image on your own as described below. @@ -134,7 +137,7 @@ The following commands will start *GPU-Jupyter* on [localhost:8848](http://local git clone https://github.com/iot-salzburg/gpu-jupyter.git cd gpu-jupyter git branch # Check for extisting branches -git checkout v1.5_cuda-12.0_ubuntu-22.04 # select or create a new version +git checkout v1.6_cuda-11.8_ubuntu-22.04 # select or create a new version # generate the Dockerfile with python and without Julia and R (see options: --help) ./generate-Dockerfile.sh --python-only docker build -t gpu-jupyter .build/ # will take a while From fca9567b51997b29434a90a73c9510fd39fb8918 Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Sun, 31 Dec 2023 13:12:56 +0100 Subject: [PATCH 6/6] cuda 12.0 and updated commit hash --- .build/Dockerfile | 2 +- README.md | 9 ++++++--- custom/header.Dockerfile | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.build/Dockerfile b/.build/Dockerfile index 2629221..d135732 100755 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -7,7 +7,7 @@ # Use NVIDIA CUDA as base image and run the same installation as in the other packages. # The version of cuda must match those of the packages installed in src/Dockerfile.gpulibs -FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04 +FROM nvidia/cuda:12.0.1-cudnn8-runtime-ubuntu22.04 LABEL authors="Christoph Schranz , Mathematical Michael " # This is a concatenated Dockerfile, the maintainers of subsequent sections may vary. RUN chmod 1777 /tmp && chmod 1777 /var/tmp diff --git a/README.md b/README.md index e4b5ae8..7a92da5 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,9 @@ for creating and maintaining a robust Python, R, and Julia toolstack for Data Sc ```bash cd your-working-directory ll data # this path will be mounted by default - docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.6_cuda-11.8_ubuntu-22.04_python-only + docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.6_cuda-12.0_ubuntu-22.04 ``` - This starts an instance of *GPU-Jupyter* with the tag `v1.6_cuda-11.8_ubuntu-22.04_python-only` at [http://localhost:8848](http://localhost:8848) (port `8848`). + This starts an instance of *GPU-Jupyter* with the tag `v1.6_cuda-12.0_ubuntu-22.04` at [http://localhost:8848](http://localhost:8848) (port `8848`). To log into Jupyterlab, you have to specify a token that you get from: ```bash docker exec -it [container-ID/name] jupyter server list @@ -82,6 +82,9 @@ for creating and maintaining a robust Python, R, and Julia toolstack for Data Sc Additionally, data within the host's `data` directory is shared with the container. Note that the following images of GPU-Jupyter are available on [Dockerhub](https://hub.docker.com/r/cschranz/gpu-jupyter): + - `v1.6_cuda-12.0_ubuntu-22.04` (full image) + - `v1.6_cuda-12.0_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.6_cuda-12.0_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) - `v1.6_cuda-11.8_ubuntu-22.04` (full image) - `v1.6_cuda-11.8_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) - `v1.6_cuda-11.8_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) @@ -137,7 +140,7 @@ The following commands will start *GPU-Jupyter* on [localhost:8848](http://local git clone https://github.com/iot-salzburg/gpu-jupyter.git cd gpu-jupyter git branch # Check for extisting branches -git checkout v1.6_cuda-11.8_ubuntu-22.04 # select or create a new version +git checkout v1.6_cuda-12.0_ubuntu-22.04 # select or create a new version # generate the Dockerfile with python and without Julia and R (see options: --help) ./generate-Dockerfile.sh --python-only docker build -t gpu-jupyter .build/ # will take a while diff --git a/custom/header.Dockerfile b/custom/header.Dockerfile index 5d59d2b..a4d9889 100644 --- a/custom/header.Dockerfile +++ b/custom/header.Dockerfile @@ -1,6 +1,6 @@ # Use NVIDIA CUDA as base image and run the same installation as in the other packages. # The version of cuda must match those of the packages installed in src/Dockerfile.gpulibs -FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04 +FROM nvidia/cuda:12.0.1-cudnn8-runtime-ubuntu22.04 LABEL authors="Christoph Schranz , Mathematical Michael " # This is a concatenated Dockerfile, the maintainers of subsequent sections may vary. RUN chmod 1777 /tmp && chmod 1777 /var/tmp