Merge remote-tracking branch 'origin/dev'

README.md

@@ -10,7 +10,7 @@ Modular container build system that provides the latest [**AI/ML packages**](htt
 |---|---|
 | **ML** | [`pytorch`](packages/pytorch) [`tensorflow`](packages/tensorflow) [`jax`](packages/ml/jax) [`onnxruntime`](packages/ml/onnxruntime) [`deepstream`](packages/multimedia/deepstream) [`holoscan`](packages/holoscan) [`CTranslate2`](packages/ml/ctranslate2) [`JupyterLab`](packages/jupyterlab) |
 | **LLM** | [`NanoLLM`](packages/llm/nano_llm) [`transformers`](packages/llm/transformers) [`text-generation-webui`](packages/llm/text-generation-webui) [`ollama`](packages/llm/ollama) [`llama.cpp`](packages/llm/llama_cpp) [`llama-factory`](packages/llm/llama-factory) [`exllama`](packages/llm/exllama) [`MLC`](packages/llm/mlc) [`AWQ`](packages/llm/awq) [`AutoGPTQ`](packages/llm/auto_gptq) [`FlashAttention`](packages/llm/flash-attention) [`optimum`](packages/llm/optimum) [`nemo`](packages/nemo) [`bitsandbytes`](packages/llm/bitsandbytes) [`xformers`](packages/llm/xformers) |
-| **VLM** | [`llava`](packages/vlm/llava) [`llama-vision`](packages/vlm/llama-vision) [`VILA`](packages/vlm/vila) [`MiniGPT4`](packages/vlm/minigpt4) [`NanoLLM`](packages/llm/nano_llm) |
+| **VLM** | [`llava`](packages/vlm/llava) [`llama-vision`](packages/vlm/llama-vision) [`VILA`](packages/vlm/vila) [`NanoLLM`](packages/llm/nano_llm) [`MiniGPT4`](packages/vlm/minigpt4) [`xtuner`](packages/vlm/xtuner) |
 | **VIT** | [`NanoOWL`](packages/vit/nanoowl) [`NanoSAM`](packages/vit/nanosam) [`Segment Anything (SAM)`](packages/vit/sam) [`Track Anything (TAM)`](packages/vit/tam) [`clip_trt`](packages/vit/clip_trt) |
 | **RAG** | [`llama-index`](packages/rag/llama-index) [`langchain`](packages/rag/langchain) [`jetson-copilot`](packages/rag/jetson-copilot) [`NanoDB`](packages/vectordb/nanodb) [`FAISS`](packages/vectordb/faiss) [`RAFT`](packages/ml/rapids/raft) |
 | **L4T** | [`l4t-pytorch`](packages/l4t/l4t-pytorch) [`l4t-tensorflow`](packages/l4t/l4t-tensorflow) [`l4t-ml`](packages/l4t/l4t-ml) [`l4t-diffusion`](packages/l4t/l4t-diffusion) [`l4t-text-generation`](packages/l4t/l4t-text-generation) |

jetson_containers/l4t_version.py

@@ -84,6 +84,7 @@ def get_jetpack_version(l4t_version=get_l4t_version(), default='5.1'):
 
     NVIDIA_JETPACK = {
         # -------- JP6 --------
+        "36.4.0": "6.1 GA",
         "36.3.0": "6.0 GA",
         "36.2.0": "6.0 DP",
         "36.0.0": "6.0 EA",

packages/cuda/cuda/config.py

@@ -116,7 +116,7 @@ def cuda_builtin(version, requires=None) -> list:
     return passthrough, cuda_pip
 
 
-def cuda_samples(version, requires) -> list:
+def cuda_samples(version, requires, branch=None) -> list:
     """
     Generates container that installs/builds the CUDA samples
     """
@@ -131,7 +131,10 @@ def cuda_samples(version, requires) -> list:
     samples['test'] = 'test_samples.sh'
     samples['depends'] = [f'cuda:{version}', 'cmake']
 
-    samples['build_args'] = {'CUDA_BRANCH': 'v' + version}
+    if not branch:
+        branch = version
+
+    samples['build_args'] = {'CUDA_BRANCH': 'v' + branch}
 
     if Version(version) == CUDA_VERSION:
         samples['alias'] = 'cuda:samples'
@@ -150,7 +153,7 @@ def cuda_samples(version, requires) -> list:
     cuda_package('12.6', 'https://developer.download.nvidia.com/compute/cuda/12.6.1/local_installers/cuda-tegra-repo-ubuntu2204-12-6-local_12.6.1-1_arm64.deb', 'cuda-tegra-repo-ubuntu2204-12-6-local', requires='==36.*'), 
     cuda_samples('12.2', requires='==36.*'),
     cuda_samples('12.4', requires='==36.*'),
-    cuda_samples('12.6', requires='==36.*'),
+    cuda_samples('12.6', branch='12.5', requires='==36.*'),
 
     # JetPack 5
     cuda_package('12.2', 'https://developer.download.nvidia.com/compute/cuda/12.2.2/local_installers/cuda-tegra-repo-ubuntu2004-12-2-local_12.2.2-1_arm64.deb', 'cuda-tegra-repo-ubuntu2004-12-2-local', requires='==35.*'),

packages/llm/bitsandbytes/test.py

@@ -11,7 +11,9 @@
 # HF   TinyLlama/TinyLlama-1.1B-Chat-v1.0
 # GPTQ TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ
 # AWQ  TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ
-model_name='microsoft/Phi-3-mini-128k-instruct'
+#
+# microsoft/Phi-3-mini-128k-instruct
+model_name='TinyLlama/TinyLlama-1.1B-Chat-v1.0'
 print(f'loading {model_name} with bitsandbytes (8-bit)')
 
 model = AutoModelForCausalLM.from_pretrained(model_name, device_map='cuda', load_in_8bit=True, trust_remote_code=True)

packages/llm/llama_cpp/config.py

@@ -7,8 +7,6 @@ def llama_cpp(version, branch=None, test=None, default=False, flags=DEFAULT_FLAG
 
     pkg['name'] = f'llama_cpp:{version}'
 
-
-
     if not test:
         test = "test_model.py --model $(huggingface-downloader TheBloke/Llama-2-7B-GGUF/llama-2-7b.Q4_K_S.gguf)"
 
@@ -37,5 +35,6 @@ def llama_cpp(version, branch=None, test=None, default=False, flags=DEFAULT_FLAG
     llama_cpp('0.2.57', flags=LEGACY_FLAGS),
     llama_cpp('0.2.70', flags=LEGACY_FLAGS),
     llama_cpp('0.2.83'),
-    llama_cpp('0.2.90', default=True),
+    llama_cpp('0.2.90'),
+    llama_cpp('0.3.1', default=True),
 ]

packages/llm/ollama/Dockerfile

@@ -32,9 +32,10 @@ RUN CMAKE_VERSION=${CMAKE_VERSION} GOLANG_VERSION=${GOLANG_VERSION} sh /opt/olla
 ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/nvidia/compat:${LD_LIBRARY_PATH} \
     CMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}
 
-# generate llama.cpp backend to bundle with Ollama
+# https://github.com/dusty-nv/jetson-containers/issues/592#issuecomment-2323177164
+# export VERSION=$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g") && \
 RUN cd /opt/ollama && \
-    export VERSION=$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g") && \
+    export VERSION="0.0.0" && \
     export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'" && \
     export GOARCH=arm64 && \
     echo "OLLAMA_VERSION=${VERSION}  GOFLAGS=${GOFLAGS}" && \

packages/llm/tensorrt_llm/config.py

@@ -45,7 +45,7 @@ def tensorrt_llm(version, branch=None, patch=None, src=None, depends=None, requi
     return trt_llm, builder
 
 package = [
-    tensorrt_llm('0.12.0', src='sources/tensorrt_llm-0.12.0.tar.gz', requires=['==r36.*', '>=cu124'], default=True),
+    tensorrt_llm('0.13.0', src='sources/tensorrt_llm-0.13.0.tar.gz', requires=['==r36.*', '>=cu124'], default=True),
     tensorrt_llm('0.10.dev0', src='sources/tensorrt_llm-0.10.0.dev0.tar.gz', requires=['==r36.*', '>=cu124'], default=False),
     #tensorrt_llm('0.9.dev', '118b3d7', patch='patches/118b3d7.diff', requires=['==r36.*', '>=cu124'], default=False),
     tensorrt_llm('0.5', patch='patches/0.5.diff', requires=['==r36.*', '==cu122'], default=True),

packages/mamba/causalconv1d/config.py

@@ -13,16 +13,16 @@ def causalconv1d(version, requires=None, default=False):
         'CASUALCONV1D_VERSION': version,
     }
 
-    #builder = pkg.copy()
+    builder = pkg.copy()
 
-    #builder['name'] = f'causalconv1d:{version}-builder'
-    #builder['build_args'] = {**pkg['build_args'], **{'FORCE_BUILD': 'on'}}
+    builder['name'] = f'causalconv1d:{version}-builder'
+    builder['build_args'] = {**pkg['build_args'], **{'FORCE_BUILD': 'on'}}
 
     if default:
         pkg['alias'] = 'causalconv1d'
-        #builder['alias'] = 'causalconv1d:builder'
+        builder['alias'] = 'causalconv1d:builder'
 
-    return pkg #, builder
+    return pkg, builder
 
 package = [
     causalconv1d('1.4.0', default=True)

packages/mamba/causalconv1d/test.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python3
 import torch
 from causal_conv1d import causal_conv1d_fn
+import causal_conv1d_cuda
 
 batch, dim, seq, width = 10, 5, 17, 4
 x = torch.zeros((batch, dim, seq)).to('cuda')
 weight = torch.zeros((dim, width)).to('cuda')
 bias = torch.zeros((dim, )).to('cuda')
 
 causal_conv1d_fn(x, weight, bias, None)
+conv1d_out = causal_conv1d_cuda.causal_conv1d_fwd(x, weight, bias, None, None, None, True)
 
 print('causal_conv1d OK\n')

packages/mamba/mamba/config.py

@@ -13,16 +13,16 @@ def mamba(version, requires=None, default=False):
         'MAMBA_VERSION': version,
     }
 
-    #builder = pkg.copy()
+    builder = pkg.copy()
 
-    #builder['name'] = f'mamba:{version}-builder'
-    #builder['build_args'] = {**pkg['build_args'], **{'FORCE_BUILD': 'on'}}
+    builder['name'] = f'mamba:{version}-builder'
+    builder['build_args'] = {**pkg['build_args'], **{'FORCE_BUILD': 'on'}}
 
     if default:
         pkg['alias'] = 'mamba'
-        #builder['alias'] = 'mamba:builder'
+        builder['alias'] = 'mamba:builder'
 
-    return pkg #, builder
+    return pkg, builder
 
 package = [
     mamba('2.2.2', default=True)

packages/mamba/zigma/Dockerfile

@@ -0,0 +1,20 @@
+#---
+# name: zigma
+# group: mamba
+# config: config.py
+# depends: [pytorch, torchvision, opencv:4.10.0, transformers, xformers, openai-triton, causalconv1d, mamba]
+# requires: '>=34.1.0'
+# test: test.py
+# notes: https://github.com/CompVis/zigma
+#---
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ARG ZIGMA_VERSION \
+    CUDAARCHS
+
+COPY build.sh /tmp/ZIGMA/
+
+RUN /tmp/ZIGMA/build.sh
+
+CMD ["/bin/bash"]

packages/mamba/zigma/build.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -ex
+
+# Clone the repository if it doesn't exist
+git clone --branch=${ZIGMA_VERSION} --depth=1 --recursive https://github.com/CompVis/zigma opt/zigma || \
+git clone --depth=1 --recursive https://github.com/CompVis/zigma /opt/zigma
+
+# Navigate to the directory containing mamba's setup.py
+cd /opt/zigma 
+pip3 install -U torchdiffeq matplotlib h5py timm diffusers accelerate loguru blobfile ml_collections wandb
+pip3 install -U hydra-core torch-fidelity webdataset einops pytorch_lightning
+pip3 install -U torchmetrics scikit-learn moviepy imageio numpy-hilbert-curve av
+
+cd /opt/zigma
+pip3 install 'numpy<2'

packages/mamba/zigma/config.py

@@ -0,0 +1,29 @@
+from jetson_containers import CUDA_ARCHITECTURES
+
+def zigma(version, requires=None, default=False):
+    pkg = package.copy()
+
+    if requires:
+        pkg['requires'] = requires   
+
+    pkg['name'] = f'zigma:{version}'
+
+    pkg['build_args'] = {
+        'CUDAARCHS': ';'.join([str(x) for x in CUDA_ARCHITECTURES]),
+        'ZIGMA_VERSION': version,
+    }
+
+    builder = pkg.copy()
+
+    builder['name'] = f'zigma:{version}-builder'
+    builder['build_args'] = {**pkg['build_args'], **{'FORCE_BUILD': 'on'}}
+
+    if default:
+        pkg['alias'] = 'zigma'
+        builder['alias'] = 'zigma:builder'
+
+    return pkg, builder
+
+package = [
+    zigma('1.0', default=True)
+]

packages/mamba/zigma/test.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+from huggingface_hub import hf_hub_download
+from model_zigma import ZigMa
+import torch
+hf_hub_download(
+        repo_id="taohu/zigma",
+        filename="faceshq1024_0090000.pt",
+        local_dir="./checkpoints",
+    )
+
+
+img_dim = 32
+in_channels = 3
+
+model = ZigMa(
+in_channels=in_channels,
+embed_dim=640,
+depth=18,
+img_dim=img_dim,
+patch_size=1,
+has_text=True,
+d_context=768,
+n_context_token=77,
+device="cuda",
+scan_type="zigzagN8",
+use_pe=2,
+)
+
+x = torch.rand(10, in_channels, img_dim, img_dim).to("cuda")
+t = torch.rand(10).to("cuda")
+_context = torch.rand(10, 77, 768).to("cuda")
+o = model(x, t, y=_context)
+print(o.shape)
+
+print('zigma OK\n')

packages/opencv/config.py

@@ -1,5 +1,6 @@
 
-from jetson_containers import CUDA_ARCHITECTURES
+from jetson_containers import CUDA_VERSION, CUDA_ARCHITECTURES
+from packaging.version import Version
 
 def opencv(version, requires=None, default=False, url=None):
     cv = package.copy()
@@ -42,8 +43,8 @@ def opencv(version, requires=None, default=False, url=None):
 package = [
     # JetPack 5/6
     opencv('4.5.0', '==35.*', default=False),
-    opencv('4.8.1', '>=35', default=True),
-    opencv('4.10.0', '>=35', default=False), 
+    opencv('4.8.1', '>=35', default=(CUDA_VERSION <= Version('12.2'))),
+    opencv('4.10.0', '>=35', default=(CUDA_VERSION > Version('12.2'))), 
 
     # JetPack 4
     opencv('4.5.0', '==32.*', default=True, url='https://nvidia.box.com/shared/static/5v89u6g5rb62fpz4lh0rz531ajo2t5ef.gz'),

packages/pytorch/torchaudio/build.sh

@@ -16,6 +16,9 @@ git clone --branch v${TORCHAUDIO_VERSION} --recursive --depth=1 https://github.c
 cd /opt/torchaudio
 git checkout v${TORCHAUDIO_VERSION}
 
+# https://github.com/pytorch/audio/pull/3811
+sed -i '1i#include <float.h>' src/libtorchaudio/cuctc/src/ctc_prefix_decoder_kernel_v2.cu || echo "warning:  failed to patch ctc_prefix_decoder_kernel_v2.cu"
+
 BUILD_SOX=1 python3 setup.py bdist_wheel --verbose --dist-dir /opt
 
 cd ../

packages/pytorch/torchaudio/install.sh

@@ -10,5 +10,5 @@ if [ "$FORCE_BUILD" == "on" ]; then
 	exit 1
 fi
 
-pip3 install --no-cache-dir torchaudio==${TORCHAUDIO_VERSION}
+pip3 install --no-cache-dir torchaudio~=${TORCHAUDIO_VERSION}
 

packages/pytorch/torchvision/install.sh

@@ -6,7 +6,7 @@ if [ "$FORCE_BUILD" == "on" ]; then
 	exit 1
 fi
 
-pip3 install --no-cache-dir torchvision==${TORCHVISION_VERSION}
+pip3 install --no-cache-dir torchvision~=${TORCHVISION_VERSION}
 
 if [ $(lsb_release --codename --short) = "focal" ]; then
     # https://github.com/conda/conda/issues/13619

packages/tensorrt/config.py

@@ -103,6 +103,7 @@ def tensorrt_builtin(version=None, requires=None, default=False):
     #tensorrt_tar('9.3', 'https://nvidia.box.com/shared/static/fp3o14iq7qbm67qjuqivdrdch7009axu.gz', cudnn='8.9', requires=['==r36.*', '==cu122']), 
     tensorrt_tar('10.0', 'https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.0.1/tars/TensorRT-10.0.1.6.l4t.aarch64-gnu.cuda-12.4.tar.gz', cudnn='9.0', requires=['==r36.*', '==cu124']), 
     tensorrt_tar('10.4', 'https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.4.0/tars/TensorRT-10.4.0.26.l4t.aarch64-gnu.cuda-12.6.tar.gz', cudnn='9.4', requires=['==r36.*', '==cu126']), 
+    tensorrt_tar('10.5', 'https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.5.0/tars/TensorRT-10.5.0.18.l4t.aarch64-gnu.cuda-12.6.tar.gz', cudnn='9.4', requires=['==r36.*', '==cu126']), 
     # JetPack 4-5 (TensorRT installed in base container)
     tensorrt_builtin(requires='<36', default=True),
 ]

packages/vlm/shape-llm/Dockerfile

@@ -0,0 +1,25 @@
+#---
+# name: shape-llm
+# alias: shape_llm
+# group: vlm
+# depends: [transformers, bitsandbytes, flash-attention, opencv:4.10.0]
+# requires: '>=35'
+# test: [test.sh, test.py]
+#---
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ADD https://github.com/gitapi/repos/qizekun/ShapeLLM/git/refs/heads/main /tmp/shape_llm_version.json
+
+RUN git clone --branch=main --depth=1 --recursive https://github.com/qizekun/ShapeLLM /opt/ShapeLLM && \
+    cd /opt/ShapeLLM && \
+    sed -i 's|torch==.*"|torch"|' pyproject.toml && \
+    sed -i 's|torchvision==.*"|torchvision"|' pyproject.toml && \
+    sed -i 's|bitsandbytes==.*"|bitsandbytes"|' pyproject.toml && \
+    sed -i 's|opencv-python==.*"|opencv-python"|' pyproject.toml && \
+    pip3 install --no-cache-dir --verbose -e '.[all]'
+
+#RUN pip3 install --no-cache-dir --verbose "git+https://github.com/erikwijmans/Pointnet2_PyTorch.git#egg=pointnet2_ops&subdirectory=pointnet2_ops_lib"
+RUN git clone https://github.com/erikwijmans/Pointnet2_PyTorch /opt/pointnet2 && \
+    cd /opt/pointnet2 && \
+    pip3 install --verbose -e pointnet2_ops_lib

packages/vlm/shape-llm/test.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python3
+from llava import LlavaLlamaForCausalLM
+

packages/vlm/shape-llm/test.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -ex
+
+python3 -m llava.serve.cli --help

packages/vlm/xtuner/Dockerfile

@@ -0,0 +1,21 @@
+#---
+# name: xtuner
+# group: vlm
+# depends: [transformers, bitsandbytes, flash-attention]
+# requires: '>=35'
+# test: [test.sh, test.py]
+#---
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ADD https://github.com/gitapi/repos/InternLM/xtuner/git/refs/heads/main /tmp/xtuner_version.json
+
+RUN git clone --branch=main --depth=1 --recursive https://github.com/InternLM/xtuner /opt/xtuner && \
+    cd /opt/xtuner && \
+    sed -i 's|bitsandbytes.*|bitsandbytes|' requirements/runtime.txt && \
+    sed -i 's|mpi4py-mpich.*||' requirements/deepspeed.txt && \
+    pip3 install --no-cache-dir --verbose -e '.[all]'
+
+# mmengine - WARNING - WARNING: command error: '[Errno 2] No such file or directory: 'python''
+RUN ln -s /usr/bin/python3 /usr/bin/python && \
+    ln -s /usr/local/bin/python3 /usr/local/bin/python

packages/vlm/xtuner/test.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python3
+import xtuner
+print('xtuner version:', xtuner.__version__)
+

packages/vlm/xtuner/test.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -ex
+xtuner --help
+xtuner list-cfg