From 09a6399f816e1cc87bb57149e64f180821ebdfd1 Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Tue, 16 Jan 2024 17:11:16 +0100 Subject: [PATCH 1/5] pinned cuda-nvcc version --- .build/Dockerfile | 9 ++++++++- custom/gpulibs.Dockerfile | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.build/Dockerfile b/.build/Dockerfile index 2629221..439d634 100755 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -425,7 +425,14 @@ RUN apt-get update && \ # reinstall nvcc with cuda-nvcc to install ptax USER $NB_UID -RUN mamba install -c nvidia cuda-nvcc -y && \ +# These need to be two separate pip install commands, otherwise it will throw an error +# attempting to resolve the nvidia-cuda-nvcc package at the same time as nvidia-pyindex +RUN pip install --no-cache-dir nvidia-pyindex && \ + pip install --no-cache-dir nvidia-cuda-nvcc && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +RUN mamba install -c nvidia cuda-nvcc=11.8.89 -y && \ mamba clean --all -f -y && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/custom/gpulibs.Dockerfile b/custom/gpulibs.Dockerfile index de68319..5ed84fa 100644 --- a/custom/gpulibs.Dockerfile +++ b/custom/gpulibs.Dockerfile @@ -42,7 +42,14 @@ RUN apt-get update && \ # reinstall nvcc with cuda-nvcc to install ptax USER $NB_UID -RUN mamba install -c nvidia cuda-nvcc -y && \ +# These need to be two separate pip install commands, otherwise it will throw an error +# attempting to resolve the nvidia-cuda-nvcc package at the same time as nvidia-pyindex +RUN pip install --no-cache-dir nvidia-pyindex && \ + pip install --no-cache-dir nvidia-cuda-nvcc && \ + fix-permissions "${CONDA_DIR}" && \ + fix-permissions "/home/${NB_USER}" + +RUN mamba install -c nvidia cuda-nvcc=11.8.89 -y && \ mamba clean --all -f -y && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER From 5d6b5f3c9fbc678af5eb3416fd3cc2cd1197d1e4 Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Wed, 24 Jan 2024 13:11:53 +0100 Subject: [PATCH 2/5] static tokden description --- README.md | 64 ++++++++++++++++++++++++------------------ generate-Dockerfile.sh | 2 +- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index a79b48e..8f3c9e0 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ for creating and maintaining a robust Python, R, and Julia toolstack for Data Sc You can optionally set a password in [http://localhost:8848/login](http://localhost:8848/login) or as described [below](#change-the-password) (former default password `gpu-jupyter`). Additionally, data within the host's `data` directory is shared with the container. - Note that the following images of GPU-Jupyter are available on [Dockerhub](https://hub.docker.com/r/cschranz/gpu-jupyter): + The following images of GPU-Jupyter are available on [Dockerhub](https://hub.docker.com/r/cschranz/gpu-jupyter): - `v1.6_cuda-12.0_ubuntu-22.04` (full image) - `v1.6_cuda-12.0_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) - `v1.6_cuda-12.0_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) @@ -95,30 +95,35 @@ Additionally, data within the host's `data` directory is shared with the contain - `v1.6_cuda-11.6_ubuntu-20.04` (full image) - `v1.6_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) - `v1.6_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) - - `v1.5_cuda-12.0_ubuntu-22.04` (full image) - - `v1.5_cuda-12.0_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) - - `v1.5_cuda-12.0_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) - - `v1.5_cuda-11.8_ubuntu-22.04` (full image) - - `v1.5_cuda-11.8_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) - - `v1.5_cuda-11.8_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) - - `v1.5_cuda-11.6_ubuntu-20.04` (full image) - - `v1.5_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) - - `v1.5_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) - - `v1.4_cuda-11.6_ubuntu-20.04` (full image) - - `v1.4_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) - - `v1.4_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) - - `v1.4_cuda-11.2_ubuntu-20.04` (full image) - - `v1.4_cuda-11.2_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) - - `v1.4_cuda-11.2_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) - - `v1.4_cuda-11.0_ubuntu-20.04` (full image) - - `v1.4_cuda-11.0_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) - - `v1.4_cuda-11.0_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) - - `v1.4_cuda-11.0_ubuntu-18.04` (full image) - - `v1.4_cuda-11.0_ubuntu-18.04_python-only` (only with a python interpreter and without Julia and R) - - `v1.4_cuda-11.0_ubuntu-18.04_slim` (only with a python interpreter and without additional packages) - - `v1.4_cuda-10.1_ubuntu-18.04` (full image) - - `v1.4_cuda-10.1_ubuntu-18.04_python-only` (only with a python interpreter and without Julia and R) - - `v1.4_cuda-10.1_ubuntu-18.04_slim` (only with a python interpreter and without additional packages) + +
+ Older images + + - `v1.5_cuda-12.0_ubuntu-22.04` (full image) + - `v1.5_cuda-12.0_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.5_cuda-12.0_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) + - `v1.5_cuda-11.8_ubuntu-22.04` (full image) + - `v1.5_cuda-11.8_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.5_cuda-11.8_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) + - `v1.5_cuda-11.6_ubuntu-20.04` (full image) + - `v1.5_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.5_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) + - `v1.4_cuda-11.6_ubuntu-20.04` (full image) + - `v1.4_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.4_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) + - `v1.4_cuda-11.2_ubuntu-20.04` (full image) + - `v1.4_cuda-11.2_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.4_cuda-11.2_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) + - `v1.4_cuda-11.0_ubuntu-20.04` (full image) + - `v1.4_cuda-11.0_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.4_cuda-11.0_ubuntu-20.04_slim` (only with a python interpreter and without additional packages) + - `v1.4_cuda-11.0_ubuntu-18.04` (full image) + - `v1.4_cuda-11.0_ubuntu-18.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.4_cuda-11.0_ubuntu-18.04_slim` (only with a python interpreter and without additional packages) + - `v1.4_cuda-10.1_ubuntu-18.04` (full image) + - `v1.4_cuda-10.1_ubuntu-18.04_python-only` (only with a python interpreter and without Julia and R) + - `v1.4_cuda-10.1_ubuntu-18.04_slim` (only with a python interpreter and without additional packages) +
The version, e.g. `v1.6`, declares the version of the generator setup. The Cuda version, e.g. `cuda-12.0`, must match the CUDA driver version and be supported by the GPU libraries. @@ -232,9 +237,9 @@ Some useful packages are suggested in the [Extension docs](https://jupyterlab.re If you notice that an important package is missing in the default stack, please let us know so we can update it. -### Authentification +### Authorization -### Set a Static Token +#### Set a Static Token Jupyter by default regenerates a new token on each new start. GPU-Jupyter provides the environment variable `JUPYTER_TOKEN` to set a customized static token. @@ -243,6 +248,7 @@ It is suggested to use a long token such as a UUID: ```bash export JUPYTER_TOKEN=$(uuidgen) +echo $JUPYTER_TOKEN ``` - For Docker add the parameter `-e JUPYTER_TOKEN=${JUPYTER_TOKEN}`, e.g.: @@ -254,7 +260,9 @@ export JUPYTER_TOKEN=$(uuidgen) - In `docker-compose.yml`, the environment variable can be set under `ENVIRONMENT`: - `NB_GID: ${JUPYTER_GID:-1000}` + `JUPYTER_TOKEN: ${JUPYTER_TOKEN}` + + Please note that environment variables might not be accessable for all docker-compose versions. Consider to use a `env_file` in docker-compose or a hard-coded token. The static token can be requested using `docker exec`: ```bash diff --git a/generate-Dockerfile.sh b/generate-Dockerfile.sh index 1a9d893..d91de70 100755 --- a/generate-Dockerfile.sh +++ b/generate-Dockerfile.sh @@ -175,7 +175,7 @@ fi cp -r extra/Getting_Started data chmod -R 755 data/ -# set static token +# set static token (optional if set) # copy jupyter server config token addendum to .build cp custom/jupyter_server_config_token_addendum.py .build/ # append token config into the jupyter server config From 72684156b8ac06db86b06b8496a64e89fb79d2c4 Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Wed, 24 Jan 2024 14:45:17 +0100 Subject: [PATCH 3/5] more explicit static token description --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8f3c9e0..a3c6728 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,9 @@ for creating and maintaining a robust Python, R, and Julia toolstack for Data Sc ```bash cd your-working-directory ll data # this path will be mounted by default - docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.6_cuda-12.0_ubuntu-22.04 + docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.6_cuda-11.8_ubuntu-22.04_python-only ``` - This starts an instance of *GPU-Jupyter* with the tag `v1.6_cuda-12.0_ubuntu-22.04` at [http://localhost:8848](http://localhost:8848) (port `8848`). + This starts an instance of *GPU-Jupyter* with the tag `v1.6_cuda-11.8_ubuntu-22.04_python-only` at [http://localhost:8848](http://localhost:8848) (port `8848`). To log into Jupyterlab, you have to specify a token that you get from: ```bash docker exec -it [container-ID/name] jupyter server list @@ -149,7 +149,7 @@ The following commands will start *GPU-Jupyter* on [localhost:8848](http://local git clone https://github.com/iot-salzburg/gpu-jupyter.git cd gpu-jupyter git branch # Check for extisting branches -git checkout v1.6_cuda-12.0_ubuntu-22.04 # select or create a new version +git checkout v1.6_cuda-11.8_ubuntu-22.04 # select or create a new version # generate the Dockerfile with python and without Julia and R (see options: --help) ./generate-Dockerfile.sh --python-only docker build -t gpu-jupyter .build/ # will take a while @@ -262,7 +262,7 @@ echo $JUPYTER_TOKEN `JUPYTER_TOKEN: ${JUPYTER_TOKEN}` - Please note that environment variables might not be accessable for all docker-compose versions. Consider to use a `env_file` in docker-compose or a hard-coded token. + Please note that environment variables might not be accessable for all docker-compose versions. Consider to setting `JUPYTER_TOKEN` in a separate `.env`-file and using `env_file` in docker-compose or a hard-coded token. The static token can be requested using `docker exec`: ```bash From 3a32e4157dd36f04fba6924bd687a431188dcb2f Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Wed, 24 Jan 2024 14:49:59 +0100 Subject: [PATCH 4/5] update descr to cuda 12.0 --- README.md | 7 ++----- custom/gpulibs.Dockerfile | 2 +- custom/header.Dockerfile | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a3c6728..803e329 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,9 @@ for creating and maintaining a robust Python, R, and Julia toolstack for Data Sc ```bash cd your-working-directory ll data # this path will be mounted by default - docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.6_cuda-11.8_ubuntu-22.04_python-only + docker run --gpus all -d -it -p 8848:8888 -v $(pwd)/data:/home/jovyan/work -e GRANT_SUDO=yes -e JUPYTER_ENABLE_LAB=yes --user root cschranz/gpu-jupyter:v1.6_cuda-12.0_ubuntu-22.04 ``` - This starts an instance of *GPU-Jupyter* with the tag `v1.6_cuda-11.8_ubuntu-22.04_python-only` at [http://localhost:8848](http://localhost:8848) (port `8848`). + This starts an instance of *GPU-Jupyter* with the tag `v1.6_cuda-12.0_ubuntu-22.04` at [http://localhost:8848](http://localhost:8848) (port `8848`). To log into Jupyterlab, you have to specify a token that you get from: ```bash docker exec -it [container-ID/name] jupyter server list @@ -92,9 +92,6 @@ Additionally, data within the host's `data` directory is shared with the contain - `v1.6_cuda-11.8_ubuntu-22.04` (full image) - `v1.6_cuda-11.8_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R) - `v1.6_cuda-11.8_ubuntu-22.04_slim` (only with a python interpreter and without additional packages) - - `v1.6_cuda-11.6_ubuntu-20.04` (full image) - - `v1.6_cuda-11.6_ubuntu-20.04_python-only` (only with a python interpreter and without Julia and R) - - `v1.6_cuda-11.6_ubuntu-20.04_slim` (only with a python interpreter and without additional packages)
Older images diff --git a/custom/gpulibs.Dockerfile b/custom/gpulibs.Dockerfile index 1edb69f..4481775 100644 --- a/custom/gpulibs.Dockerfile +++ b/custom/gpulibs.Dockerfile @@ -58,7 +58,7 @@ RUN pip install --no-cache-dir nvidia-pyindex && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -RUN mamba install -c nvidia cuda-nvcc=11.8.89 -y && \ +RUN mamba install -c nvidia cuda-nvcc=12.0.140 -y && \ mamba clean --all -f -y && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/custom/header.Dockerfile b/custom/header.Dockerfile index 5d59d2b..a4d9889 100644 --- a/custom/header.Dockerfile +++ b/custom/header.Dockerfile @@ -1,6 +1,6 @@ # Use NVIDIA CUDA as base image and run the same installation as in the other packages. # The version of cuda must match those of the packages installed in src/Dockerfile.gpulibs -FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04 +FROM nvidia/cuda:12.0.1-cudnn8-runtime-ubuntu22.04 LABEL authors="Christoph Schranz , Mathematical Michael " # This is a concatenated Dockerfile, the maintainers of subsequent sections may vary. RUN chmod 1777 /tmp && chmod 1777 /var/tmp From b4e021af0ef0c59ce8a410df6e948b8887b8630f Mon Sep 17 00:00:00 2001 From: Christoph Schranz Date: Wed, 24 Jan 2024 14:58:16 +0100 Subject: [PATCH 5/5] set cuda 12.0 --- .build/Dockerfile | 70 ++--------------------------------------------- README.md | 2 +- 2 files changed, 3 insertions(+), 69 deletions(-) diff --git a/.build/Dockerfile b/.build/Dockerfile index 76357fa..6f137e7 100755 --- a/.build/Dockerfile +++ b/.build/Dockerfile @@ -7,7 +7,7 @@ # Use NVIDIA CUDA as base image and run the same installation as in the other packages. # The version of cuda must match those of the packages installed in src/Dockerfile.gpulibs -FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04 +FROM nvidia/cuda:12.0.1-cudnn8-runtime-ubuntu22.04 LABEL authors="Christoph Schranz , Mathematical Michael " # This is a concatenated Dockerfile, the maintainers of subsequent sections may vary. RUN chmod 1777 /tmp && chmod 1777 /var/tmp @@ -377,72 +377,6 @@ USER ${NB_UID} WORKDIR "${HOME}" - ############################################################################ - ################ Dependency: jupyter/datascience-notebook ################## - ############################################################################ - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -ARG REGISTRY=quay.io -ARG OWNER=jupyter - -LABEL maintainer="Jupyter Project " - -# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 -# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -USER root - -# R pre-requisites -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - fonts-dejavu \ - gfortran \ - gcc && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Julia dependencies -# install Julia packages in /opt/julia instead of ${HOME} -ENV JULIA_DEPOT_PATH=/opt/julia \ - JULIA_PKGDIR=/opt/julia - -# Setup Julia -RUN /opt/setup-scripts/setup_julia.py - -USER ${NB_UID} - -# Setup IJulia kernel & other packages -RUN /opt/setup-scripts/setup-julia-packages.bash - -# R packages including IRKernel which gets installed globally. -# r-e1071: dependency of the caret R package -RUN mamba install --yes \ - 'r-base' \ - 'r-caret' \ - 'r-crayon' \ - 'r-devtools' \ - 'r-e1071' \ - 'r-forecast' \ - 'r-hexbin' \ - 'r-htmltools' \ - 'r-htmlwidgets' \ - 'r-irkernel' \ - 'r-nycflights13' \ - 'r-randomforest' \ - 'r-rcurl' \ - 'r-rmarkdown' \ - 'r-rodbc' \ - 'r-rsqlite' \ - 'r-shiny' \ - 'r-tidymodels' \ - 'r-tidyverse' \ - 'rpy2' \ - 'unixodbc' && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - ############################################################################ ########################## Dependency: gpulibs ############################# ############################################################################ @@ -507,7 +441,7 @@ RUN pip install --no-cache-dir nvidia-pyindex && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -RUN mamba install -c nvidia cuda-nvcc=11.8.89 -y && \ +RUN mamba install -c nvidia cuda-nvcc=12.0.140 -y && \ mamba clean --all -f -y && \ fix-permissions $CONDA_DIR && \ fix-permissions /home/$NB_USER diff --git a/README.md b/README.md index 803e329..752fc9e 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Additionally, data within the host's `data` directory is shared with the contain - `v1.6_cuda-11.8_ubuntu-22.04_slim` (only with a python interpreter and without additional packages)
- Older images + Older images - `v1.5_cuda-12.0_ubuntu-22.04` (full image) - `v1.5_cuda-12.0_ubuntu-22.04_python-only` (only with a python interpreter and without Julia and R)