From b2de65246769486112cfc6b9549564479ae607ba Mon Sep 17 00:00:00 2001 From: daemon1024 Date: Thu, 22 Jun 2023 15:21:38 +0530 Subject: [PATCH] feat(install): Probe KubeArmor after installing to show enforcement is enabled or not Signed-off-by: daemon1024 --- install/install.go | 105 +++++++++++--------------------------------- k8s/env.go | 87 ++++++++++++++++++++++++++++++++++++ probe/probe.go | 107 ++++++++++++++++++++++----------------------- 3 files changed, 164 insertions(+), 135 deletions(-) create mode 100644 k8s/env.go diff --git a/install/install.go b/install/install.go index 54e71e6e..ea408432 100644 --- a/install/install.go +++ b/install/install.go @@ -16,12 +16,13 @@ import ( "time" "github.com/clarketm/json" + "github.com/fatih/color" "sigs.k8s.io/yaml" deployments "github.com/kubearmor/KubeArmor/deployments/get" "github.com/kubearmor/kubearmor-client/k8s" + "github.com/kubearmor/kubearmor-client/probe" - "golang.org/x/mod/semver" v1 "k8s.io/api/apps/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -115,7 +116,7 @@ func printMessage(msg string, flag bool) int { return 0 } -func checkPods(c *k8s.Client) int { +func checkPods(c *k8s.Client, o Options) { cursor := [4]string{"|", "/", "—", "\\"} fmt.Printf("😋 Checking if KubeArmor pods are running ...") stime := time.Now() @@ -140,7 +141,25 @@ func checkPods(c *k8s.Client) int { break } } - return 0 + kData, err := probe.ProbeRunningKubeArmorNodes(c, probe.Options{ + Namespace: o.Namespace, + }) + if err != nil || len(kData) == 0 { + return + } + enforcing := true + for _, k := range kData { + if k.ActiveLSM == "" || !k.ContainerSecurity { + enforcing = false + break + } + } + if enforcing { + fmt.Print(color.New(color.FgWhite, color.Bold).Sprint("\n\t🛡️ Your Cluster is Armored Up Now! \n")) + } else { + color.Yellow("\n\t⚠️ KubeArmor is running in Audit mode, only Observability is available no Enforcement. \n") + } + } func checkTerminatingPods(c *k8s.Client) int { @@ -176,7 +195,7 @@ func K8sInstaller(c *k8s.Client, o Options) error { animation = o.Animation var env string if o.Env.Auto { - env = AutoDetectEnvironment(c) + env = k8s.AutoDetectEnvironment(c) if env == "none" { return errors.New("unsupported environment or cluster not configured correctly") } @@ -278,7 +297,7 @@ func K8sInstaller(c *k8s.Client, o Options) error { } daemonset.Spec.Template.Spec.Containers[0].Image = o.KubearmorImage daemonset.Spec.Template.Spec.InitContainers[0].Image = o.InitImage - if o.Local == true { + if o.Local { daemonset.Spec.Template.Spec.Containers[0].ImagePullPolicy = "IfNotPresent" daemonset.Spec.Template.Spec.InitContainers[0].ImagePullPolicy = "IfNotPresent" } @@ -471,7 +490,7 @@ func K8sInstaller(c *k8s.Client, o Options) error { } if animation { - checkPods(c) + checkPods(c, o) } return nil } @@ -668,80 +687,6 @@ func K8sUninstaller(c *k8s.Client, o Options) error { return nil } -// AutoDetectEnvironment detect the environment for a given k8s context -func AutoDetectEnvironment(c *k8s.Client) (name string) { - env := "none" - - contextName := c.RawConfig.CurrentContext - clusterContext, exists := c.RawConfig.Contexts[contextName] - if !exists { - return env - } - - clusterName := clusterContext.Cluster - cluster := c.RawConfig.Clusters[clusterName] - nodes, _ := c.K8sClientset.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) - if len(nodes.Items) <= 0 { - return env - } - containerRuntime := nodes.Items[0].Status.NodeInfo.ContainerRuntimeVersion - nodeImage := nodes.Items[0].Status.NodeInfo.OSImage - - // Detecting Environment based on cluster name and context or OSImage - if clusterName == "minikube" || contextName == "minikube" { - env = "minikube" - return env - } - - if strings.HasPrefix(clusterName, "microk8s-") || contextName == "microk8s" { - env = "microk8s" - return env - } - - if strings.HasPrefix(clusterName, "gke_") { - env = "gke" - return env - } - - if strings.Contains(nodeImage, "Bottlerocket") { - env = "bottlerocket" - return env - } - - if strings.HasSuffix(clusterName, ".eksctl.io") || strings.HasSuffix(cluster.Server, "eks.amazonaws.com") { - env = "eks" - return env - } - - // Environment is Self Managed K8s, checking container runtime and it's version - - if strings.Contains(containerRuntime, "k3s") { - env = "k3s" - return env - } - - s := strings.Split(containerRuntime, "://") - runtime := s[0] - version := "v" + s[1] - - if runtime == "docker" && semver.Compare(version, "v18.9") >= 0 { - env = "docker" - return env - } - - if runtime == "cri-o" { - env = "oke" - return env - } - - if (runtime == "docker" && semver.Compare(version, "v19.3") >= 0) || runtime == "containerd" { - env = "generic" - return env - } - - return env -} - func writeToYAML(f *os.File, o interface{}) error { // Use "clarketm/json" to marshal so as to support zero values of structs with omitempty j, err := json.Marshal(o) diff --git a/k8s/env.go b/k8s/env.go new file mode 100644 index 00000000..2876a362 --- /dev/null +++ b/k8s/env.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2023 Authors of KubeArmor + +package k8s + +import ( + "context" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "golang.org/x/mod/semver" +) + +// AutoDetectEnvironment detect the environment for a given k8s context +func AutoDetectEnvironment(c *Client) (name string) { + env := "none" + + contextName := c.RawConfig.CurrentContext + clusterContext, exists := c.RawConfig.Contexts[contextName] + if !exists { + return env + } + + clusterName := clusterContext.Cluster + cluster := c.RawConfig.Clusters[clusterName] + nodes, _ := c.K8sClientset.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if len(nodes.Items) <= 0 { + return env + } + containerRuntime := nodes.Items[0].Status.NodeInfo.ContainerRuntimeVersion + nodeImage := nodes.Items[0].Status.NodeInfo.OSImage + + // Detecting Environment based on cluster name and context or OSImage + if clusterName == "minikube" || contextName == "minikube" { + env = "minikube" + return env + } + + if strings.HasPrefix(clusterName, "microk8s-") || contextName == "microk8s" { + env = "microk8s" + return env + } + + if strings.HasPrefix(clusterName, "gke_") { + env = "gke" + return env + } + + if strings.Contains(nodeImage, "Bottlerocket") { + env = "bottlerocket" + return env + } + + if strings.HasSuffix(clusterName, ".eksctl.io") || strings.HasSuffix(cluster.Server, "eks.amazonaws.com") { + env = "eks" + return env + } + + // Environment is Self Managed K8s, checking container runtime and it's version + + if strings.Contains(containerRuntime, "k3s") { + env = "k3s" + return env + } + + s := strings.Split(containerRuntime, "://") + runtime := s[0] + version := "v" + s[1] + + if runtime == "docker" && semver.Compare(version, "v18.9") >= 0 { + env = "docker" + return env + } + + if runtime == "cri-o" { + env = "oke" + return env + } + + if (runtime == "docker" && semver.Compare(version, "v19.3") >= 0) || runtime == "containerd" { + env = "generic" + return env + } + + return env +} diff --git a/probe/probe.go b/probe/probe.go index 9c58d675..da97360a 100644 --- a/probe/probe.go +++ b/probe/probe.go @@ -34,7 +34,6 @@ import ( "errors" - "github.com/kubearmor/kubearmor-client/install" "golang.org/x/sys/unix" ) @@ -75,7 +74,7 @@ func probeDaemonUninstaller(c *k8s.Client, o Options) error { // PrintProbeResult prints the result for the host and k8s probing kArmor does to check compatibility with KubeArmor func PrintProbeResult(c *k8s.Client, o Options) error { if runtime.GOOS != "linux" { - env := install.AutoDetectEnvironment(c) + env := k8s.AutoDetectEnvironment(c) if env == "none" { return errors.New("unsupported environment or cluster not configured correctly") } @@ -90,10 +89,17 @@ func PrintProbeResult(c *k8s.Client, o Options) error { if isKubeArmorRunning(c, o) { getKubeArmorDeployments(c, o) getKubeArmorContainers(c, o) - err := probeRunningKubeArmorNodes(c, o) + probeData, err := ProbeRunningKubeArmorNodes(c, o) if err != nil { log.Println("error occured when probing kubearmor nodes", err) } + for i, pd := range probeData { + _, err := boldWhite.Printf("Node %d : \n", i+1) + if err != nil { + color.Red(" Error") + } + printKubeArmorProbeOutput(pd) + } err = getAnnotatedPods(c) if err != nil { log.Println("error occured when getting annotated pods", err) @@ -226,7 +232,7 @@ func checkKernelHeaderPresent() bool { return false } -func execIntoPod(c *k8s.Client, podname, namespace, containername string, cmd string) (string, error) { +func execIntoPod(c *k8s.Client, podname, namespace, cmd string) (string, error) { buf := &bytes.Buffer{} errBuf := &bytes.Buffer{} request := c.K8sClientset.CoreV1().RESTClient(). @@ -243,7 +249,10 @@ func execIntoPod(c *k8s.Client, podname, namespace, containername string, cmd st TTY: true, }, scheme.ParameterCodec) exec, err := remotecommand.NewSPDYExecutor(c.Config, "POST", request.URL()) - err = exec.Stream(remotecommand.StreamOptions{ + if err != nil { + return "none", err + } + err = exec.StreamWithContext(context.TODO(), remotecommand.StreamOptions{ Stdout: buf, Stderr: errBuf, }) @@ -255,8 +264,8 @@ func execIntoPod(c *k8s.Client, podname, namespace, containername string, cmd st return buf.String(), nil } -func findFileInDir(c *k8s.Client, podname, namespace, containername string, cmd string) bool { - s, err := execIntoPod(c, podname, namespace, containername, cmd) +func findFileInDir(c *k8s.Client, podname, namespace, cmd string) bool { + s, err := execIntoPod(c, podname, namespace, cmd) if err != nil { return false } @@ -268,7 +277,7 @@ func findFileInDir(c *k8s.Client, podname, namespace, containername string, cmd } // Check for BTF Information or Kernel Headers Availability -func checkNodeKernelHeaderPresent(c *k8s.Client, o Options, nodeName string, kernelVersion string) bool { +func checkNodeKernelHeaderPresent(c *k8s.Client, o Options, nodeName string) bool { pods, err := c.K8sClientset.CoreV1().Pods(o.Namespace).List(context.Background(), metav1.ListOptions{ LabelSelector: "kubearmor-app=" + deployment.Karmorprobe, FieldSelector: "spec.nodeName=" + nodeName, @@ -277,9 +286,9 @@ func checkNodeKernelHeaderPresent(c *k8s.Client, o Options, nodeName string, ker return false } - if findFileInDir(c, pods.Items[0].Name, o.Namespace, pods.Items[0].Spec.Containers[0].Name, "find /sys/kernel/btf/ -name vmlinux") || - findFileInDir(c, pods.Items[0].Name, o.Namespace, pods.Items[0].Spec.Containers[0].Name, "find -L /usr/src -maxdepth 2 -path \"*$(uname -r)*\" -name \"Kconfig\"") || - findFileInDir(c, pods.Items[0].Name, o.Namespace, pods.Items[0].Spec.Containers[0].Name, "find -L /lib/modules/ -maxdepth 3 -path \"*$(uname -r)*\" -name \"Kconfig\"") { + if findFileInDir(c, pods.Items[0].Name, o.Namespace, "find /sys/kernel/btf/ -name vmlinux") || + findFileInDir(c, pods.Items[0].Name, o.Namespace, "find -L /usr/src -maxdepth 2 -path \"*$(uname -r)*\" -name \"Kconfig\"") || + findFileInDir(c, pods.Items[0].Name, o.Namespace, "find -L /lib/modules/ -maxdepth 3 -path \"*$(uname -r)*\" -name \"Kconfig\"") { return true } @@ -315,7 +324,7 @@ func getNodeLsmSupport(c *k8s.Client, o Options, nodeName string) (string, error return "none", err } - s, err := execIntoPod(c, pods.Items[0].Name, o.Namespace, pods.Items[0].Spec.Containers[0].Name, "cat "+srcPath) + s, err := execIntoPod(c, pods.Items[0].Name, o.Namespace, "cat "+srcPath) if err != nil { return "none", err } @@ -332,7 +341,7 @@ func probeNode(c *k8s.Client, o Options) { } fmt.Printf("\t Observability/Audit:") kernelVersion := item.Status.NodeInfo.KernelVersion - check2 := checkNodeKernelHeaderPresent(c, o, item.Name, kernelVersion) + check2 := checkNodeKernelHeaderPresent(c, o, item.Name) checkAuditSupport(kernelVersion, check2) lsm, err := getNodeLsmSupport(c, o, item.Name) if err != nil { @@ -380,7 +389,6 @@ func getKubeArmorDaemonset(c *k8s.Client, o Options) (bool, error) { } desired, ready, available := w.Status.DesiredNumberScheduled, w.Status.NumberReady, w.Status.NumberAvailable if desired != ready && desired != available { - data = append(data, []string{" ", "kubearmor ", "Desired: " + strconv.Itoa(int(desired)), "Ready: " + strconv.Itoa(int(ready)), "Available: " + strconv.Itoa(int(available))}) return false, nil } data = append(data, []string{" ", "kubearmor ", "Desired: " + strconv.Itoa(int(desired)), "Ready: " + strconv.Itoa(int(ready)), "Available: " + strconv.Itoa(int(available))}) @@ -437,42 +445,38 @@ func getKubeArmorContainers(c *k8s.Client, o Options) { } -func probeRunningKubeArmorNodes(c *k8s.Client, o Options) error { - - // // KubeArmor Nodes +// ProbeRunningKubeArmorNodes extracts data from running KubeArmor daemonset by executing into the container and reading /tmp/kubearmor.cfg +func ProbeRunningKubeArmorNodes(c *k8s.Client, o Options) ([]KubeArmorProbeData, error) { + // KubeArmor Nodes nodes, err := c.K8sClientset.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) if err != nil { - log.Println("error getting nodes", err) - return err + return []KubeArmorProbeData{}, fmt.Errorf("error occured when getting nodes %s", err.Error()) } - if len(nodes.Items) > 0 { + if len(nodes.Items) == 0 { + return []KubeArmorProbeData{}, fmt.Errorf("no nodes found") + } - for i, item := range nodes.Items { - _, err := boldWhite.Printf("Node %d : \n", i+1) - if err != nil { - color.Red(" Error while printing") - } - err = readDataFromKubeArmor(c, o, item.Name) - if err != nil { - return err - } + var dataList []KubeArmorProbeData + for _, item := range nodes.Items { + data, err := readDataFromKubeArmor(c, o, item.Name) + if err != nil { + return []KubeArmorProbeData{}, err } - } else { - fmt.Println("No kubernetes environment found") + dataList = append(dataList, data) } - return nil + + return dataList, nil } -func readDataFromKubeArmor(c *k8s.Client, o Options, nodeName string) error { +func readDataFromKubeArmor(c *k8s.Client, o Options, nodeName string) (KubeArmorProbeData, error) { srcPath := "/tmp/karmorProbeData.cfg" pods, err := c.K8sClientset.CoreV1().Pods(o.Namespace).List(context.Background(), metav1.ListOptions{ LabelSelector: "kubearmor-app=kubearmor", FieldSelector: "spec.nodeName=" + nodeName, }) if err != nil { - log.Println("error occured while getting kubeArmor pods", err) - return err + return KubeArmorProbeData{}, fmt.Errorf("error occured while getting kubeArmor pods %s", err.Error()) } reader, outStream := io.Pipe() cmdArr := []string{"cat", srcPath} @@ -491,13 +495,12 @@ func readDataFromKubeArmor(c *k8s.Client, o Options, nodeName string) error { TTY: false, }, scheme.ParameterCodec) exec, err := remotecommand.NewSPDYExecutor(c.Config, "POST", req.URL()) - if err != nil { - return err + return KubeArmorProbeData{}, err } go func() { defer outStream.Close() - err = exec.Stream(remotecommand.StreamOptions{ + err = exec.StreamWithContext(context.TODO(), remotecommand.StreamOptions{ Stdin: os.Stdin, Stdout: outStream, Stderr: os.Stderr, @@ -506,26 +509,19 @@ func readDataFromKubeArmor(c *k8s.Client, o Options, nodeName string) error { }() buf, err := io.ReadAll(reader) if err != nil { - log.Println("an error occured when reading file", err) - return err + return KubeArmorProbeData{}, fmt.Errorf("error occured while reading data from kubeArmor pod %s", err.Error()) } - err = printKubeArmorProbeOutput(buf) + var kd KubeArmorProbeData + var json = jsoniter.ConfigCompatibleWithStandardLibrary + err = json.Unmarshal(buf, &kd) if err != nil { - log.Println("an error occured when printing output", err) - return err + return KubeArmorProbeData{}, fmt.Errorf("error occured while parsing data from kubeArmor pod %s", err.Error()) } - return nil + return kd, nil } -func printKubeArmorProbeOutput(buf []byte) error { - var kd *KubeArmorProbeData - var json = jsoniter.ConfigCompatibleWithStandardLibrary +func printKubeArmorProbeOutput(kd KubeArmorProbeData) { var data [][]string - err := json.Unmarshal(buf, &kd) - if err != nil { - log.Println("an error occured when parsing file", err) - return err - } data = append(data, []string{" ", "OS Image:", green(kd.OSImage)}) data = append(data, []string{" ", "Kernel Version:", green(kd.KernelVersion)}) data = append(data, []string{" ", "Kubelet Version:", green(kd.KubeletVersion)}) @@ -537,7 +533,6 @@ func printKubeArmorProbeOutput(buf []byte) error { data = append(data, []string{" ", "Host Default Posture:", green(kd.HostDefaultPosture.FileAction) + itwhite("(File)"), green(kd.HostDefaultPosture.CapabilitiesAction) + itwhite("(Capabilities)"), green(kd.HostDefaultPosture.NetworkAction) + itwhite("(Network)")}) data = append(data, []string{" ", "Host Visibility:", green(kd.HostVisibility)}) renderOutputInTableWithNoBorders(data) - return nil } // sudo systemctl status kubearmor @@ -567,11 +562,13 @@ func probeSystemdMode() error { if err != nil { color.Red(" Error") } - err = printKubeArmorProbeOutput(buf) + var kd KubeArmorProbeData + var json = jsoniter.ConfigCompatibleWithStandardLibrary + err = json.Unmarshal(buf, &kd) if err != nil { - log.Println("an error occured when printing output", err) return err } + printKubeArmorProbeOutput(kd) return nil }