feature: minio support in fiab (#163)

In fiab, mlflow is configured without an object store. Therefore, artifacts (e.g., model) are stored locally within a container. Once training is over, then the artifacts get lost. With support for minio (an open-source object store), now artifacts are saved in an object store. Also, minio's endpoint is exposed via ingress so that workers outside a cluster can save artifacts. Note that tls for minio is not enabled because selfsigned cert used in fiab is likely to make mlflow's client complain about a verification issue. A caveat is that no persistent volume is configured in this minio setup, artifacts will get lost if minio pod crashes or terminates. The support for minio is for reference so that users can configure minio or aws s3 correctly for production.
cisco-open · Jul 1, 2022 · 28ffca1 · 28ffca1
1 parent 4a3c559
commit 28ffca1
Show file tree

Hide file tree

Showing 7 changed files with 59 additions and 8 deletions.
diff --git a/fiab/helm-chart/Chart.yaml b/fiab/helm-chart/Chart.yaml
@@ -25,3 +25,7 @@ dependencies:
   - name: mongodb
     version: 10.31.3
     repository: https://charts.bitnami.com/bitnami
+
+  - name: minio
+    version: 4.0.2
+    repository: https://charts.min.io
diff --git a/fiab/helm-chart/job/job-agent.yaml.mustache b/fiab/helm-chart/job/job-agent.yaml.mustache
@@ -42,5 +42,13 @@ spec:
               value: <% taskId %>
             - name: FLAME_TASK_KEY
               value: <% taskKey %>
+            {{ if ne .Values.s3Endpointurl "" }}
+            - name: MLFLOW_S3_ENDPOINT_URL
+              value: {{ .Values.s3EndpointUrl }}
+            {{ end }}
+            - name: AWS_ACCESS_KEY_ID
+              value: {{ .Values.accessKeyId }}
+            - name: AWS_SECRET_ACCESS_KEY
+              value: {{ .Values.secretAccessKey }}
       restartPolicy: Never
 <%={{ }}=%>
diff --git a/fiab/helm-chart/templates/controller-job-configmap.yaml b/fiab/helm-chart/templates/controller-job-configmap.yaml
@@ -22,6 +22,10 @@ data:
       apiserver: https://{{ .Values.frontDoorUrl.apiserver }}:443
       notifier: {{ .Values.frontDoorUrl.notifier }}:443
 
+    s3EndpointUrl: {{ .Values.mlflow.s3EndpointUrl }}
+    accessKeyId: {{ (index .Values.minio.users 0).accessKey }}
+    secretAccessKey: {{ (index .Values.minio.users 0).secretKey }}
+
     # enable this in case a selfsigned cert is in use (for test only)
     # this holds true in fiab setting
     insecure: true

diff --git a/fiab/helm-chart/templates/mlflow-deployment.yaml b/fiab/helm-chart/templates/mlflow-deployment.yaml
@@ -33,10 +33,17 @@ spec:
     spec:
       containers:
         - args:
-            - "mlflow server --backend-store-uri postgresql://mlflow:mlflow@postgres/mlflow --default-artifact-root file:./mlruns --host 0.0.0.0"
+            - "mlflow server --backend-store-uri postgresql://mlflow:mlflow@postgres/mlflow --default-artifact-root s3://mlruns --host 0.0.0.0"
           command: ["bash", "-c"]
           image: adacotechjp/mlflow:1.21.0
           name: {{ .Release.Name }}-mlflow
           ports:
-            - containerPort: {{ .Values.servicePort.mlflow }}
+            - containerPort: {{ .Values.mlflow.servicePort }}
+          env:
+            - name: MLFLOW_S3_ENDPOINT_URL
+              value: {{ .Values.mlflow.s3EndpointUrl }}
+            - name: AWS_ACCESS_KEY_ID
+              value: {{ (index .Values.minio.users 0).accessKey }}
+            - name: AWS_SECRET_ACCESS_KEY
+              value: {{ (index .Values.minio.users 0).secretKey }}
       restartPolicy: Always
diff --git a/fiab/helm-chart/templates/mlflow-ingress.yaml b/fiab/helm-chart/templates/mlflow-ingress.yaml
@@ -27,12 +27,12 @@ spec:
             service:
               name: {{ .Release.Name }}-mlflow
               port:
-                number: {{ .Values.servicePort.mlflow }}
+                number: {{ .Values.mlflow.servicePort }}
           pathType: Prefix
           path: /
         {{ else }}
         - backend:
             serviceName: {{ .Release.Name }}-mlflow
-            servicePort: {{ .Values.servicePort.mlflow }}
+            servicePort: {{ .Values.mlflow.servicePort }}
           path: /
         {{ end }}
diff --git a/fiab/helm-chart/templates/mlflow-service.yaml b/fiab/helm-chart/templates/mlflow-service.yaml
@@ -25,9 +25,9 @@ metadata:
   namespace: {{ .Release.Namespace }}
 spec:
   ports:
-    - name: "{{ .Values.servicePort.mlflow }}"
-      port: {{ .Values.servicePort.mlflow }}
-      targetPort: {{ .Values.servicePort.mlflow }}
+    - name: "{{ .Values.mlflow.servicePort }}"
+      port: {{ .Values.mlflow.servicePort }}
+      targetPort: {{ .Values.mlflow.servicePort }}
   selector:
     app: {{ .Release.Name }}-mlflow
   type: ClusterIP
diff --git a/fiab/helm-chart/values.yaml b/fiab/helm-chart/values.yaml
@@ -34,6 +34,31 @@ mongodb:
   #   create: true
   #######
 
+minio:
+  mode: standalone # other option: distributed
+  persistence:
+    enabled: false
+    size: 1Gi
+  buckets:
+    - name: mlruns
+      policy: none
+      purge: false
+  resources:
+    requests:
+      memory: 256Mi
+  rootUser: minio
+  rootPassword: minio123
+  users:
+    - accessKey: flame
+      secretKey: flame123
+      policy: readwrite
+  ingress:
+    enabled: true
+    ingressClassName: nginx
+    annotations: {}
+    hosts:
+      - minio.flame.test
+
 imageName: flame
 imageTag: latest
 workerImageName: flame
@@ -67,12 +92,15 @@ frontDoorUrl:
   notifier: notifier.flame.test
   mlflow: mlflow.flame.test
 
+mlflow:
+  s3EndpointUrl: http://minio.flame.test
+  servicePort: "5000"
+
 servicePort:
   apiserver: "10100"
   notifier: "10101"
   controller: "10102"
   agent: "10103"
-  mlflow: "5000"
   mqtt: "1883"
 
 replicas: 1