diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 1dbff114e..6b90d1b50 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -30,6 +30,11 @@ jobs:
             python_version: "3.10"
             pytorch: 2.1.1
             torch_cuda_arch_list: "7.0 7.5 8.0 8.6 9.0+PTX"
+          - cuda: "121"
+            cuda_version: 12.1.0
+            python_version: "3.10"
+            pytorch: 2.1.1
+            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 9.0+PTX"
     steps:
       - name: Checkout
         uses: actions/checkout@v3
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 3eb97e55d..2f0b07450 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -29,6 +29,11 @@ jobs:
             python_version: "3.10"
             pytorch: 2.1.1
             axolotl_extras:
+          - cuda: 121
+            cuda_version: 12.1.0
+            python_version: "3.10"
+            pytorch: 2.1.1
+            axolotl_extras:
     runs-on: [self-hosted, gpu, docker]
     steps:
       - name: Checkout
@@ -95,6 +100,11 @@ jobs:
             python_version: "3.10"
             pytorch: 2.1.1
             axolotl_extras:
+          - cuda: 121
+            cuda_version: 12.1.0
+            python_version: "3.10"
+            pytorch: 2.1.1
+            axolotl_extras:
     runs-on: [self-hosted, gpu, docker]
     steps:
       - name: Checkout