Merge pull request #2189 from CoderTH/support-mig-gpu-operator

update mig config & upgrade gpu-operator version
DaoCloud · Jun 20, 2024 · bddc1d7 · bddc1d7
2 parents 05ae57a + 0809d38
commit bddc1d7
Show file tree

Hide file tree

Showing 11 changed files with 164 additions and 64 deletions.
diff --git a/charts/gpu-operator/config b/charts/gpu-operator/config
@@ -17,4 +17,7 @@ export DAOCLOUD_REPO_PROJECT=community
 export CUSTOM_SHELL=custom.sh
 export APPEND_VALUES_FILE=appendValues.yaml
 export NO_IMAGE=true
-export NO_TRIVY=true
+export NO_TRIVY=true
+
+# gpu-operator version
+export CUSTOM_VERSION=${VERSION}+1
diff --git a/charts/gpu-operator/custom.sh b/charts/gpu-operator/custom.sh
diff --git a/charts/gpu-operator/gpu-operator/Chart.yaml b/charts/gpu-operator/gpu-operator/Chart.yaml
@@ -15,7 +15,7 @@ kubeVersion: '>= 1.16.0-0'
 name: gpu-operator
 sources:
   - https://github.com/NVIDIA/gpu-operator
-version: v23.9.0
+version: v23.9.0+1
 dependencies:
   - name: gpu-operator
     version: "v23.9.0"

diff --git a/charts/gpu-operator/gpu-operator/charts/gpu-operator/templates/migstrategy-config.yaml b/charts/gpu-operator/gpu-operator/charts/gpu-operator/templates/migstrategy-config.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: device-plugin-config
+  namespace: {{ .Release.Namespace }}
+data:
+  mixed: |-
+    version: v1
+    flags:
+      migStrategy: mixed
+      failOnInitError: true
+      nvidiaDriverRoot: "/run/nvidia/driver"
+      gdsEnabled: false
+      mofedEnabled: false
+      plugin:
+        passDeviceSpecs: true
+        deviceListStrategy:
+        - envvar
+        deviceIDStrategy: uuid
+        cdiAnnotationPrefix: cdi.k8s.io/
+        nvidiaCTKPath: "/usr/bin/nvidia-ctk"
+        containerDriverRoot: "/run/nvidia/driver"
+    resources:
+      gpus:
+      - pattern: "*"
+        name: nvidia.com/gpu
+  single: |-
+    version: v1
+    flags:
+      migStrategy: single
+      failOnInitError: true
+      nvidiaDriverRoot: "/run/nvidia/driver"
+      gdsEnabled: false
+      mofedEnabled: false
+      plugin:
+        passDeviceSpecs: true
+        deviceListStrategy:
+        - envvar
+        deviceIDStrategy: uuid
+        cdiAnnotationPrefix: cdi.k8s.io/
+        nvidiaCTKPath: "/usr/bin/nvidia-ctk"
+        containerDriverRoot: "/run/nvidia/driver"
+    resources:
+      gpus:
+      - pattern: "*"
+        name: nvidia.com/gpu
+  none: |-
+    version: v1
+    flags:
+      migStrategy: none
+      failOnInitError: true
+      nvidiaDriverRoot: "/run/nvidia/driver"
+      gdsEnabled: false
+      mofedEnabled: false
+      plugin:
+        passDeviceSpecs: true
+        deviceListStrategy:
+        - envvar
+        deviceIDStrategy: uuid
+        cdiAnnotationPrefix: cdi.k8s.io/
+        nvidiaCTKPath: "/usr/bin/nvidia-ctk"
+        containerDriverRoot: "/run/nvidia/driver"
+    resources:
+      gpus:
+      - pattern: "*"
+        name: nvidia.com/gpu
diff --git a/charts/gpu-operator/gpu-operator/charts/gpu-operator/values.yaml b/charts/gpu-operator/gpu-operator/charts/gpu-operator/values.yaml
@@ -110,8 +110,7 @@ operator:
     requests:
       cpu: 200m
       memory: 100Mi
-mig:
-  strategy: single
+mig: {}
 driver:
   enabled: true
   nvidiaDriverCRD:
@@ -261,9 +260,9 @@ devicePlugin:
     # Create a ConfigMap (default: false)
     create: false
     # ConfigMap name (either exiting or to create a new one with create=true above)
-    name: ""
+    name: "device-plugin-config"
     # Default config name within the ConfigMap
-    default: ""
+    default: "none"
     # Data section for the ConfigMap to create (i.e only applies when create=true)
     data: {}
 # standalone dcgm hostengine
@@ -323,7 +322,7 @@ gfd:
       value: "true"
   resources: {}
 migManager:
-  enabled: false
+  enabled: true
   repository: nvcr.m.daocloud.io
   image: nvidia/cloud-native/k8s-mig-manager
   version: v0.5.5-ubuntu20.04

diff --git a/charts/gpu-operator/gpu-operator/values-schema.yaml b/charts/gpu-operator/gpu-operator/values-schema.yaml
@@ -18,12 +18,9 @@ gpu-operator:
     image: nvidia/k8s/container-toolkit
     version: v1.14.3-centos7
   migManager:
-    enabled: false
     config:
       name: "default-mig-parted-config"
       default: "all-disabled"
-  mig:
-    strategy: single
 
   node-feature-discovery:
     enableNodeFeatureApi: true
diff --git a/charts/gpu-operator/gpu-operator/values.schema.json b/charts/gpu-operator/gpu-operator/values.schema.json
@@ -69,9 +69,6 @@
                 "migManager": {
                     "type": "object",
                     "properties": {
-                        "enabled": {
-                            "type": "boolean"
-                        },
                         "config": {
                             "type": "object",
                             "properties": {
@@ -81,26 +78,6 @@
                             }
                         }
                     }
-                },
-                "mig": {
-                    "type": "object",
-                    "properties": {
-                        "strategy": {
-                            "type": "string",
-                            "enum": [
-                                "single",
-                                "mixed"
-                            ]
-                        }
-                    }
-                },
-                "node-feature-discovery": {
-                    "type": "object",
-                    "properties": {
-                        "enableNodeFeatureApi": {
-                            "type": "boolean"
-                        }
-                    }
                 }
             }
         }

diff --git a/charts/gpu-operator/gpu-operator/values.yaml b/charts/gpu-operator/gpu-operator/values.yaml
@@ -111,8 +111,7 @@ gpu-operator:
       requests:
         cpu: 200m
         memory: 100Mi
-  mig:
-    strategy: single
+  mig: {}
   driver:
     enabled: true
     nvidiaDriverCRD:
@@ -262,9 +261,9 @@ gpu-operator:
       # Create a ConfigMap (default: false)
       create: false
       # ConfigMap name (either exiting or to create a new one with create=true above)
-      name: ""
+      name: "device-plugin-config"
       # Default config name within the ConfigMap
-      default: ""
+      default: "none"
       # Data section for the ConfigMap to create (i.e only applies when create=true)
       data: {}
   # standalone dcgm hostengine
@@ -324,7 +323,7 @@ gpu-operator:
         value: "true"
     resources: {}
   migManager:
-    enabled: false
+    enabled: true
     repository: nvcr.m.daocloud.io
     image: nvidia/cloud-native/k8s-mig-manager
     version: v0.5.5-ubuntu20.04

diff --git a/charts/gpu-operator/parent/charts/gpu-operator/templates/migstrategy-config.yaml b/charts/gpu-operator/parent/charts/gpu-operator/templates/migstrategy-config.yaml
@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: device-plugin-config
+  namespace: {{ .Release.Namespace }}
+data:
+  mixed: |-
+    version: v1
+    flags:
+      migStrategy: mixed
+      failOnInitError: true
+      nvidiaDriverRoot: "/run/nvidia/driver"
+      gdsEnabled: false
+      mofedEnabled: false
+      plugin:
+        passDeviceSpecs: true
+        deviceListStrategy:
+        - envvar
+        deviceIDStrategy: uuid
+        cdiAnnotationPrefix: cdi.k8s.io/
+        nvidiaCTKPath: "/usr/bin/nvidia-ctk"
+        containerDriverRoot: "/run/nvidia/driver"
+    resources:
+      gpus:
+      - pattern: "*"
+        name: nvidia.com/gpu
+  single: |-
+    version: v1
+    flags:
+      migStrategy: single
+      failOnInitError: true
+      nvidiaDriverRoot: "/run/nvidia/driver"
+      gdsEnabled: false
+      mofedEnabled: false
+      plugin:
+        passDeviceSpecs: true
+        deviceListStrategy:
+        - envvar
+        deviceIDStrategy: uuid
+        cdiAnnotationPrefix: cdi.k8s.io/
+        nvidiaCTKPath: "/usr/bin/nvidia-ctk"
+        containerDriverRoot: "/run/nvidia/driver"
+    resources:
+      gpus:
+      - pattern: "*"
+        name: nvidia.com/gpu
+  none: |-
+    version: v1
+    flags:
+      migStrategy: none
+      failOnInitError: true
+      nvidiaDriverRoot: "/run/nvidia/driver"
+      gdsEnabled: false
+      mofedEnabled: false
+      plugin:
+        passDeviceSpecs: true
+        deviceListStrategy:
+        - envvar
+        deviceIDStrategy: uuid
+        cdiAnnotationPrefix: cdi.k8s.io/
+        nvidiaCTKPath: "/usr/bin/nvidia-ctk"
+        containerDriverRoot: "/run/nvidia/driver"
+    resources:
+      gpus:
+      - pattern: "*"
+        name: nvidia.com/gpu
diff --git a/charts/gpu-operator/parent/values-schema.yaml b/charts/gpu-operator/parent/values-schema.yaml
@@ -18,12 +18,9 @@ gpu-operator:
     image: nvidia/k8s/container-toolkit
     version: v1.14.3-centos7
   migManager:
-    enabled: false
     config:
       name: "default-mig-parted-config"
       default: "all-disabled"
-  mig:
-    strategy: single
 
   node-feature-discovery:
     enableNodeFeatureApi: true
diff --git a/charts/gpu-operator/parent/values.schema.json b/charts/gpu-operator/parent/values.schema.json
@@ -69,9 +69,6 @@
                 "migManager": {
                     "type": "object",
                     "properties": {
-                        "enabled": {
-                            "type": "boolean"
-                        },
                         "config": {
                             "type": "object",
                             "properties": {
@@ -81,26 +78,6 @@
                             }
                         }
                     }
-                },
-                "mig": {
-                    "type": "object",
-                    "properties": {
-                        "strategy": {
-                            "type": "string",
-                            "enum": [
-                                "single",
-                                "mixed"
-                            ]
-                        }
-                    }
-                },
-                "node-feature-discovery": {
-                    "type": "object",
-                    "properties": {
-                        "enableNodeFeatureApi": {
-                            "type": "boolean"
-                        }
-                    }
                 }
             }
         }