From f37b4f8effcf72002f62654cbee30be8ccee8de1 Mon Sep 17 00:00:00 2001
From: Cappy Ishihara <cappy@cappuchino.xyz>
Date: Sat, 2 Sep 2023 23:35:01 +0700
Subject: [PATCH] Update with ROCM

---
 .github/workflows/docker.yml | 10 ++++++--
 Dockerfile                   | 25 +++++++++++++++----
 docker-compose.yml           |  2 +-
 koboldcpp                    |  2 +-
 nocuda.dockerfile            | 48 ++++++++++++++++++++++++++----------
 requirements.txt             |  2 --
 6 files changed, 65 insertions(+), 24 deletions(-)
 delete mode 100644 requirements.txt

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index bf6e0a9..4e45e8f 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -26,7 +26,10 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           push: true
-          tags: korewachino/koboldcpp:latest,korewachino/koboldcpp:1.41,korewachino/koboldcpp:cuda
+          tags: |
+           korewachino/koboldcpp:latest
+           korewachino/koboldcpp:1.42.1
+           korewachino/koboldcpp:cuda
   docker-nocuda:
     runs-on: ubuntu-latest
     steps:
@@ -48,4 +51,7 @@ jobs:
         with:
           file: nocuda.dockerfile
           push: true
-          tags: korewachino/koboldcpp:nocuda,korewachino/koboldcpp:1.41-nocuda
\ No newline at end of file
+          tags: |
+           korewachino/koboldcpp:nocuda
+           korewachino/koboldcpp:1.42.1-rocm
+           korewachino/koboldcpp:1.42.1-nocuda
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 9ab0b51..5b4392f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,10 @@
-FROM nvidia/cuda:12.2.0-devel-ubuntu20.04
+ARG UBUNTU_VERSION=22.04
+ARG CUDA_VERSION=12.2.0
+
+FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}
+
+# Unless otherwise specified, we make a fat build.
+ARG CUDA_DOCKER_ARCH=all
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl gnupg software-properties-common 
 RUN add-apt-repository -y ppa:cnugteren/clblast && apt-get update
@@ -27,16 +33,25 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
   apt-get clean && \
   rm -rf /var/lib/apt/lists/*
 
-COPY requirements.txt .
-RUN pip install -r requirements.txt
+
 RUN pip install psutil
+
+
 RUN mkdir ./home/koboldcpp
 COPY ./koboldcpp ./home/koboldcpp
 
 WORKDIR /home/koboldcpp
+
+RUN pip install --upgrade pip setuptools wheel \
+    && pip install -r requirements.txt
+
 # You need this environment variable to make sure that the CUDA architecture works for all GPUs
-ENV CUDA_DOCKER_ARCH=all
-RUN make LLAMA_OPENBLAS=1 LLAMA_CUBLAS=1 LLAMA_CLBLAST=1 -j$(nproc)
+ENV CUDA_DOCKER_ARCH=${CUDA_DOCKER_ARCH}
+ENV LLAMA_CUBLAS=1
+ENV LLAMA_OPENBLAS=1
+ENV LLAMA_CUBLAS=1
+ENV LLAMA_CLBLAST=1
+RUN make -j$(nproc)
 
 WORKDIR /
 COPY start_program.sh /home/koboldcpp
diff --git a/docker-compose.yml b/docker-compose.yml
index b257553..d1c9711 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -21,7 +21,7 @@ services:
     working_dir: /home/koboldcpp
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: nocuda.dockerfile
       # dockerfile: nocuda.dockerfile
     ports:
       - "5001:5001"
diff --git a/koboldcpp b/koboldcpp
index 9d5b423..b6914eb 160000
--- a/koboldcpp
+++ b/koboldcpp
@@ -1 +1 @@
-Subproject commit 9d5b4238e831e64c3f76cd12e2173a47a9e30616
+Subproject commit b6914ebd0468b36bbbfb99b1dad71d81e54363c9
diff --git a/nocuda.dockerfile b/nocuda.dockerfile
index 1351620..f40a70e 100644
--- a/nocuda.dockerfile
+++ b/nocuda.dockerfile
@@ -1,16 +1,32 @@
-FROM ubuntu:20.04
+ARG UBUNTU_VERSION=22.04
 
-# Initialize the image
-# Modify to pre-install dev tools and ROCm packages
-ARG ROCM_VERSION=5.3
-ARG AMDGPU_VERSION=5.3
+# This needs to generally match the container host's environment.
+ARG ROCM_VERSION=5.6
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl gnupg software-properties-common && \
-  curl -sL http://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - && \
-  sh -c 'echo deb [arch=amd64] http://repo.radeon.com/rocm/apt/$ROCM_VERSION/ focal main > /etc/apt/sources.list.d/rocm.list' && \
-  sh -c 'echo deb [arch=amd64] https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/ubuntu focal main > /etc/apt/sources.list.d/amdgpu.list'
+# Target the CUDA build image
+ARG BASE_ROCM_DEV_CONTAINER=rocm/dev-ubuntu-${UBUNTU_VERSION}:${ROCM_VERSION}-complete
+
+# Unless otherwise specified, we make a fat build.
+# List from https://github.com/ggerganov/llama.cpp/pull/1087#issuecomment-1682807878
+# This is mostly tied to rocBLAS supported archs.
+ARG ROCM_DOCKER_ARCH=\
+  gfx803 \
+  gfx900 \
+  gfx906 \
+  gfx908 \
+  gfx90a \
+  gfx1010 \
+  gfx1030 \
+  gfx1100 \
+  gfx1101 \
+  gfx1102
+
+
+FROM ${BASE_ROCM_DEV_CONTAINER}
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl gnupg software-properties-common
 RUN add-apt-repository -y ppa:cnugteren/clblast -y && apt-get update
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+RUN apt-get install -y --no-install-recommends \
   sudo \
   libelf1 \
   libnuma-dev \
@@ -36,14 +52,20 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
   apt-get clean && \
   rm -rf /var/lib/apt/lists/*
 
-COPY requirements.txt .
-RUN pip install -r requirements.txt
 RUN pip install psutil
 RUN mkdir ./home/koboldcpp
 COPY ./koboldcpp ./home/koboldcpp
 
 WORKDIR /home/koboldcpp
-RUN make LLAMA_OPENBLAS=1 LLAMA_CLBLAST=1 -j$(nproc)
+RUN pip install -r requirements.txt
+
+ENV LLAMA_OPENBLAS=1
+ENV LLAMA_CLBLAST=1
+ENV LLAMA_HIPBLAS=1
+ENV CC=/opt/rocm/llvm/bin/clang
+ENV CXX=/opt/rocm/llvm/bin/clang++
+
+RUN make -j$(nproc)
 
 WORKDIR /
 COPY start_program.sh /home/koboldcpp
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 6c32cbd..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-numpy==1.24
-sentencepiece==0.1.98