cloud-barista · itnpeople · Aug 23, 2022 · Aug 22, 2022
diff --git a/docs/design/entity.md b/docs/design/entity.md
@@ -22,6 +22,7 @@
     networkCni: "",
     label: "",
     installMonAgent: "",
+    k8sVersion: "",
     description: "",
     createdTime: "",
     nodes: [
@@ -75,6 +76,7 @@
 |installMonAgent    |모니터링 에이전트 설치 여부        |string | yes/no (no가 아니면 설치)              |
 |description        |description                 |string |                                     |
 |createdTime        |생성일자                      |string |                                     |
+|k8sVersion         |쿠버네티스버전                  |string |                                     |
 
 ### ClusterPhase
 > 프로비저닝 단계

diff --git a/src/core/app/const.go b/src/core/app/const.go
@@ -51,13 +51,14 @@ type Status struct {
 }
 
 type ClusterReq struct {
-	Name            string           `json:"name" example:"cluster-01"`
-	ControlPlane    []NodeSetReq     `json:"controlPlane"`
-	Worker          []NodeSetReq     `json:"worker"`
-	Config          ClusterConfigReq `json:"config"`
-	Label           string           `json:"label"`
-	InstallMonAgent string           `json:"installMonAgent" example:"no" default:"yes"`
-	Description     string           `json:"description"`
+	Name            string                 `json:"name" example:"cluster-01"`
+	ControlPlane    []NodeSetReq           `json:"controlPlane"`
+	Worker          []NodeSetReq           `json:"worker"`
+	Config          ClusterConfigReq       `json:"config"`
+	StorageClass    ClusterStorageClassReq `json:"storageclass"`
+	Label           string                 `json:"label"`
+	InstallMonAgent string                 `json:"installMonAgent" example:"no" default:"yes"`
+	Description     string                 `json:"description"`
 }
 
 type NodeReq struct {
@@ -80,3 +81,12 @@ type ClusterConfigKubernetesReq struct {
 	ServiceCidr      string     `json:"serviceCidr" example:"10.96.0.0/12"`
 	ServiceDnsDomain string     `json:"serviceDnsDomain" example:"cluster.local"`
 }
+
+type ClusterStorageClassReq struct {
+	Nfs ClusterStorageClassNfsReq `json:"nfs"`
+}
+
+type ClusterStorageClassNfsReq struct {
+	Server string `json:"server" example:"163.154.154.89"`
+	Path   string `json:"path" example:"/nfs/data"`
+}
diff --git a/src/core/model/types.go b/src/core/model/types.go
@@ -30,6 +30,7 @@ const (
 	SetupHaproxyFailedReason                  = ClusterReason("SetupHaproxyFailedReason")
 	InitControlPlaneFailedReason              = ClusterReason("InitControlPlaneFailedReason")
 	SetupNetworkCNIFailedReason               = ClusterReason("SetupNetworkCNIFailedReason")
+	SetupStorageClassFailedReason             = ClusterReason("SetupStorageClassFailedReason")
 	JoinControlPlaneFailedReason              = ClusterReason("JoinControlPlaneFailedReason")
 	JoinWorkerFailedReason                    = ClusterReason("JoinWorkerFailedReason")
 )

diff --git a/src/core/provision/machine.go b/src/core/provision/machine.go
@@ -123,6 +123,11 @@ func (self *Machine) bootstrap(networkCni app.NetworkCni, k8sVersion string) err
 		} else {
 			sourceFiles = append(sourceFiles, CNI_KILO_CRDS_FILE, CNI_KILO_KUBEADM_FILE, CNI_KILO_FLANNEL_FILE)
 		}
+
+		if _, err := self.executeSSH("mkdir -p %s/addons/%s", REMOTE_TARGET_PATH, "nfs"); err != nil {
+			return errors.New(fmt.Sprintf("Failed to create a addon directory. (node=%s, path='%s')", self.Name, "addons/"+"nfs"))
+		}
+		sourceFiles = append(sourceFiles, SC_NFS_RBAC_FILE, SC_NFS_CLASS_FILE, "addons/nfs/deploy_v4.0.16.sh")
 	}
 
 	//  - copy list-up files

diff --git a/src/core/provision/provisioner.go b/src/core/provision/provisioner.go
@@ -202,6 +202,28 @@ func (self *Provisioner) InstallNetworkCni() error {
 	return nil
 }
 
+func (self *Provisioner) InstallStorageClassNFS(storageReq app.ClusterStorageClassNfsReq) error {
+
+	storageYamls := []string{}
+	if storageReq.Server != "" {
+		storageYamls = append(storageYamls, SC_NFS_RBAC_FILE)
+		storageYamls = append(storageYamls, SC_NFS_CLASS_FILE)
+	}
+
+	for _, file := range storageYamls {
+		if _, err := self.Kubectl("apply -f %s/%s", REMOTE_TARGET_PATH, file); err != nil {
+			return err
+		}
+	}
+	if storageReq.Server != "" {
+		if _, err := self.leader.executeSSH("cd %s;./%s %s %s ", REMOTE_TARGET_PATH, "addons/nfs/deploy_v4.0.16.sh", storageReq.Path, storageReq.Server); err != nil {
+			return errors.New("Failed to setup storageCalss controla-plane.")
+		}
+	}
+
+	return nil
+}
+
 /* assign node labels */
 func (self *Provisioner) AssignNodeLabelAnnotation() error {
 

diff --git a/src/core/provision/types.go b/src/core/provision/types.go
@@ -11,6 +11,8 @@ const (
 	CNI_KILO_CRDS_FILE    = "addons/kilo/crds_v0.3.0.yaml"
 	CNI_KILO_KUBEADM_FILE = "addons/kilo/kilo-kubeadm-flannel_v0.3.0.yaml"
 	CNI_KILO_FLANNEL_FILE = "addons/kilo/kube-flannel_v0.14.0.yaml"
+	SC_NFS_RBAC_FILE      = "addons/nfs/rbac_v4.0.16.yaml"
+	SC_NFS_CLASS_FILE     = "addons/nfs/class_v4.0.16.yaml"
 )
 
 type Machine struct {

diff --git a/src/core/service/cluster.go b/src/core/service/cluster.go
@@ -257,6 +257,16 @@ func CreateCluster(namespace string, minorversion string, patchversion string, r
 	}
 	logger.Infof("[%s.%s] CNI installation has been completed.", namespace, clusterName)
 
+	// kubernetes provisioning : setting storageclass
+	if req.StorageClass.Nfs.Server != "" && req.StorageClass.Nfs.Path != "" {
+		if err = provisioner.InstallStorageClassNFS(req.StorageClass.Nfs); err != nil {
+			cluster.FailReason(model.SetupStorageClassFailedReason, fmt.Sprintf("Failed to install storageclass. (cause='%v')", err))
+			cleanUpCluster(*cluster, mcis)
+			return nil, errors.New(cluster.Status.Message)
+		}
+		logger.Infof("[%s.%s] Storageclass installation has been completed.", namespace, clusterName)
+	}
+
 	// save nodes metadata & update status
 	for _, node := range cluster.Nodes {
 		node.CreatedTime = lang.GetNowUTC()

diff --git a/src/docs/docs.go b/src/docs/docs.go
@@ -630,6 +630,9 @@ var doc = `{
                     "type": "string",
                     "example": "cluster-01"
                 },
+                "storageclass": {
+                    "$ref": "#/definitions/app.ClusterStorageClassReq"
+                },
                 "worker": {
                     "type": "array",
                     "items": {
@@ -638,6 +641,27 @@ var doc = `{
                 }
             }
         },
+        "app.ClusterStorageClassNfsReq": {
+            "type": "object",
+            "properties": {
+                "path": {
+                    "type": "string",
+                    "example": "/nfs/data"
+                },
+                "server": {
+                    "type": "string",
+                    "example": "163.154.154.89"
+                }
+            }
+        },
+        "app.ClusterStorageClassReq": {
+            "type": "object",
+            "properties": {
+                "nfs": {
+                    "$ref": "#/definitions/app.ClusterStorageClassNfsReq"
+                }
+            }
+        },
         "app.NodeReq": {
             "type": "object",
             "properties": {

diff --git a/src/docs/swagger.json b/src/docs/swagger.json
@@ -615,6 +615,9 @@
                     "type": "string",
                     "example": "cluster-01"
                 },
+                "storageclass": {
+                    "$ref": "#/definitions/app.ClusterStorageClassReq"
+                },
                 "worker": {
                     "type": "array",
                     "items": {
@@ -623,6 +626,27 @@
                 }
             }
         },
+        "app.ClusterStorageClassNfsReq": {
+            "type": "object",
+            "properties": {
+                "path": {
+                    "type": "string",
+                    "example": "/nfs/data"
+                },
+                "server": {
+                    "type": "string",
+                    "example": "163.154.154.89"
+                }
+            }
+        },
+        "app.ClusterStorageClassReq": {
+            "type": "object",
+            "properties": {
+                "nfs": {
+                    "$ref": "#/definitions/app.ClusterStorageClassNfsReq"
+                }
+            }
+        },
         "app.NodeReq": {
             "type": "object",
             "properties": {

diff --git a/src/docs/swagger.yaml b/src/docs/swagger.yaml
@@ -42,11 +42,27 @@ definitions:
       name:
         example: cluster-01
         type: string
+      storageclass:
+        $ref: '#/definitions/app.ClusterStorageClassReq'
       worker:
         items:
           $ref: '#/definitions/app.NodeSetReq'
         type: array
     type: object
+  app.ClusterStorageClassNfsReq:
+    properties:
+      path:
+        example: /nfs/data
+        type: string
+      server:
+        example: 163.154.154.89
+        type: string
+    type: object
+  app.ClusterStorageClassReq:
+    properties:
+      nfs:
+        $ref: '#/definitions/app.ClusterStorageClassNfsReq'
+    type: object
   app.NodeReq:
     properties:
       controlPlane:

diff --git a/src/scripts/addons/nfs/class_v4.0.16.yaml b/src/scripts/addons/nfs/class_v4.0.16.yaml
@@ -0,0 +1,7 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: nfs-client
+provisioner: k8s-sigs.io/nfs-subdir-external-provisioner # or choose another name, must match deployment's env PROVISIONER_NAME'
+parameters:
+  archiveOnDelete: "false"
diff --git a/src/scripts/addons/nfs/deploy_v4.0.16.sh b/src/scripts/addons/nfs/deploy_v4.0.16.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+cat << EOF > nfs-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nfs-client-provisioner
+  labels:
+    app: nfs-client-provisioner
+  # replace with namespace where provisioner is deployed
+  namespace: kube-system
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: nfs-client-provisioner
+  template:
+    metadata:
+      labels:
+        app: nfs-client-provisioner
+    spec:
+      serviceAccountName: nfs-client-provisioner
+      containers:
+        - name: nfs-client-provisioner
+          image: k8s.gcr.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
+          volumeMounts:
+            - name: nfs-client-root
+              mountPath: /persistentvolumes
+          env:
+            - name: PROVISIONER_NAME
+              value: k8s-sigs.io/nfs-subdir-external-provisioner
+            - name: NFS_SERVER
+              value: $2
+            - name: NFS_PATH
+              value: $1
+      volumes:
+        - name: nfs-client-root
+          nfs:
+            server: $2
+            path: $1
+EOF
+
+sudo kubectl apply -f nfs-deployment.yaml --kubeconfig=/etc/kubernetes/admin.conf
diff --git a/src/scripts/addons/nfs/rbac_v4.0.16.yaml b/src/scripts/addons/nfs/rbac_v4.0.16.yaml
@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nfs-client-provisioner
+  # replace with namespace where provisioner is deployed
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nfs-client-provisioner-runner
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: run-nfs-client-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: nfs-client-provisioner
+    # replace with namespace where provisioner is deployed
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: nfs-client-provisioner-runner
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: leader-locking-nfs-client-provisioner
+  # replace with namespace where provisioner is deployed
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: leader-locking-nfs-client-provisioner
+  # replace with namespace where provisioner is deployed
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: nfs-client-provisioner
+    # replace with namespace where provisioner is deployed
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: leader-locking-nfs-client-provisioner
+  apiGroup: rbac.authorization.k8s.io
diff --git a/src/scripts/bootstrap.sh b/src/scripts/bootstrap.sh
@@ -98,6 +98,7 @@ fi
 # kubeadm , kubelet, kubectl
 sudo apt-get install -y kubeadm=${K8S_VERSION} kubelet=${K8S_VERSION} kubectl=${K8S_VERSION}
 sudo apt-mark hold kubeadm kubelet kubectl
+sudo apt-get -y install nfs-common cifs-utils
 
 if [ "${CSP}" != "openstack" ]; then 
 	PUBLIC_IP='$(dig +short myip.opendns.com @resolver1.opendns.com)'