feat: helm option to mount shared_fs checkpoints to master (#8741)

docs/release-notes/master-mount.rst

@@ -0,0 +1,9 @@
+:orphan:
+
+**Improvements**
+
+-  Helm: Add support for Downloading Checkpoints when using ``shared_fs``. Adds a ``mountToServer``
+   value under ``checkpointStorage``. By default, this parameter is set to ``false`` preserving the
+   current behavior. However when it's set to ``true`` and the storage type is ``shared_fs`` it
+   enables the hostpath mount on the server. This allows for the use of ``checkpoint.download()`` to
+   work with ``shared_fs`` on Determined starting from version ``0.27.0`` and later.

helm/charts/determined/templates/master-deployment.yaml

@@ -51,6 +51,10 @@ spec:
           - name: master-config
             mountPath: /etc/determined/
             readOnly: true
+          {{- if and (.Values.checkpointStorage.mountToServer) (eq .Values.checkpointStorage.type "shared_fs") }}
+          - name: checkpoint-storage
+            mountPath: /determined_shared_fs
+          {{ end }}
           {{- if .Values.tlsSecret }}
           - name: tls-secret
             mountPath: {{ include "determined.secretPath" . }}
@@ -113,6 +117,12 @@ spec:
         - name: master-config
           secret:
             secretName: determined-master-config-{{ .Release.Name }}
+        {{- if and (.Values.checkpointStorage.mountToServer) ( eq .Values.checkpointStorage.type "shared_fs") }}
+        - name: checkpoint-storage
+          hostPath:
+            path: {{ .Values.checkpointStorage.hostPath }}
+            type: Directory
+        {{ end }}
         {{- if .Values.tlsSecret }}
         - name: tls-secret
           secret:

helm/charts/determined/values.yaml

@@ -196,7 +196,8 @@ checkpointStorage:
   # system.
   type: shared_fs
   hostPath: /checkpoints
-
+  # By default, shared_fs is not mounted to the server pod. Change this to true to enable checkpoint downloads from the server.
+  mountToServer: false
   # For storing in GCS.
   # type: gcs
   # bucket: <bucket_name>