diff --git a/deployments/ek-dashboards/es-sts.yaml b/deployments/ek-dashboards/es-sts.yaml new file mode 100644 index 0000000..eafdc13 --- /dev/null +++ b/deployments/ek-dashboards/es-sts.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: es-cluster + namespace: kube-system +spec: + serviceName: elasticsearch + replicas: 3 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0 + resources: + limits: + cpu: 1000m + requests: + cpu: 100m + ports: + - containerPort: 9200 + name: rest + protocol: TCP + - containerPort: 9300 + name: inter-node + protocol: TCP + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + env: + - name: cluster.name + value: k8s-logs + - name: node.name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: discovery.seed_hosts + value: "es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch" + - name: cluster.initial_master_nodes + value: "es-cluster-0,es-cluster-1,es-cluster-2" + - name: ES_JAVA_OPTS + value: "-Xms512m -Xmx512m" + initContainers: + - name: fix-permissions + image: busybox + command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"] + securityContext: + privileged: true + volumeMounts: + - name: data + mountPath: /usr/share/elasticsearch/data + - name: increase-vm-max-map + image: busybox + command: ["sysctl", "-w", "vm.max_map_count=262144"] + securityContext: + privileged: true + - name: increase-fd-ulimit + image: busybox + command: ["sh", "-c", "ulimit -n 65536"] + securityContext: + privileged: true + volumeClaimTemplates: + - metadata: + name: data + labels: + app: elasticsearch + spec: + accessModes: [ "ReadWriteOnce" ] + # storageClassName: "" + resources: + requests: + storage: 5Gi diff --git a/deployments/ek-dashboards/es-svc.yaml b/deployments/ek-dashboards/es-svc.yaml new file mode 100644 index 0000000..dae96bb --- /dev/null +++ b/deployments/ek-dashboards/es-svc.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + namespace: kube-system + labels: + app: elasticsearch + namespace: kube-system +spec: + selector: + app: elasticsearch + clusterIP: None + ports: + - port: 9200 + name: rest + - port: 9300 + name: inter-node + diff --git a/deployments/ek-dashboards/kibana-deployment.yaml b/deployments/ek-dashboards/kibana-deployment.yaml new file mode 100644 index 0000000..7dd7faa --- /dev/null +++ b/deployments/ek-dashboards/kibana-deployment.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + labels: + app: kibana + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + template: + metadata: + labels: + app: kibana + spec: + containers: + - name: kibana + image: docker.elastic.co/kibana/kibana:7.2.0 + resources: + limits: + cpu: 1000m + requests: + cpu: 100m + env: + - name: ELASTICSEARCH_URL + value: http://elasticsearch:9200 + ports: + - containerPort: 5601 \ No newline at end of file diff --git a/deployments/ek-dashboards/kibana-svc.yaml b/deployments/ek-dashboards/kibana-svc.yaml new file mode 100644 index 0000000..f38f3a3 --- /dev/null +++ b/deployments/ek-dashboards/kibana-svc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: kibana-np + namespace: kube-system +spec: + selector: + app: kibana + type: NodePort + ports: + - port: 8080 + targetPort: 5601 + nodePort: 30000 \ No newline at end of file diff --git a/deployments/relay-deployment.yaml b/deployments/relay-deployment.yaml index ad9e401..a83f8dd 100644 --- a/deployments/relay-deployment.yaml +++ b/deployments/relay-deployment.yaml @@ -33,6 +33,13 @@ spec: containers: - name: kubearmor-relay-server image: kubearmor/kubearmor-relay-server:latest + env: + - name: ENABLE_DASHBOARDS + value: "true" + - name: ES_URL + value: "http://elasticsearch:9200" + - name: KUBEARMOR_SERVICE + value: "" imagePullPolicy: Always ports: - containerPort: 32767 diff --git a/docs/ek-dashboards/dash-1.png b/docs/ek-dashboards/dash-1.png new file mode 100644 index 0000000..014f95e Binary files /dev/null and b/docs/ek-dashboards/dash-1.png differ diff --git a/docs/ek-dashboards/dash-2.png b/docs/ek-dashboards/dash-2.png new file mode 100644 index 0000000..f375ea6 Binary files /dev/null and b/docs/ek-dashboards/dash-2.png differ diff --git a/docs/ek-dashboards/ek-dashboards.md b/docs/ek-dashboards/ek-dashboards.md new file mode 100644 index 0000000..f0e4bef --- /dev/null +++ b/docs/ek-dashboards/ek-dashboards.md @@ -0,0 +1,65 @@ +# kubearmor-elasticstack-logging + + +## Elastic Stack Visualisation + + +There would be 2 additional components along with the Relay server: + +1. **Elasticsearch** is a real-time, distributed, and scalable search engine which allows for full-text and structured search, as well as analytics. Relay server logs can be indexed and searched through which would be produced in large volumes of log data. + + +2. **Kibana** is a data visualization frontend and dashboard for Elasticsearch. Kibana allows user to explore the log data in a visual manner that is stored in the Elasticsearch instance with the help of a web interface. Users would also be allowed to build dashboards or view existing ones which would help to answer and quickly gain insight about the pods managed by KubeArmor: + +- Alert Metric +- Alert from Different Pods +- Alert from Different Namespace +- Alert based on Operations +- Policy and Action Summary +- NameSpace Matched Policy Count +- Namespace Severity Summary +- Alert Based on Tags + +Kibana will be a part of deployment , while ElasticSearch will be a part of StatefulSet that can run in any node + +``` +kubectl apply -f deployments/ek-dashboards +``` + +For the log data to be sent to elasticsearch, change the values of ```ENABLE_DASHBOARDS``` in ```deployments/relay-deployment.yaml>spec>template>spec>container>env``` to ```true``` , it should look like + +``` + ....... + + containers: + - name: kubearmor-relay-server + image: kubearmor/kubearmor-relay-server:latest + env: + - name: ENABLE_DASHBOARDS + value: "true" + ....... + +``` + +To View the DashBoards + +* Portforward the Kibana service +``` +kubectl port-forward deployment/kibana -n kube-system 5601:5601 +``` +* Open up a browser and go to [localhost:5601](localhost:5601) +* Go to sidebar and open ``Mangement`` -> ``Saved Objects`` -> ``Import`` + +Drag and drop the file from ```docs/ek-dashboards/export.ndjson``` + +* Go to ``Dashboard`` section , selct ``KA`` + +* The visalisations should be ready !! + +Here are some example visulisation with [multiubuntu](https://github.com/kubearmor/KubeArmor/blob/main/examples/multiubuntu.md) and [wordpress-mysql](https://github.com/kubearmor/KubeArmor/blob/main/examples/wordpress-mysql.md) example + +![Dash Board 2](./dash-2.png) +![Dash Board 1](./dash-1.png) + + + diff --git a/docs/ek-dashboards/export.ndjson b/docs/ek-dashboards/export.ndjson new file mode 100644 index 0000000..3165c3a --- /dev/null +++ b/docs/ek-dashboards/export.ndjson @@ -0,0 +1,9 @@ +{"attributes":{"fields":"[{\"name\":\"Action\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Action.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Action\",\"subType\":\"multi\"},{\"name\":\"ClusterName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ClusterName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ClusterName\",\"subType\":\"multi\"},{\"name\":\"ContainerID\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ContainerID.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ContainerID\",\"subType\":\"multi\"},{\"name\":\"ContainerImage\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ContainerImage.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ContainerImage\",\"subType\":\"multi\"},{\"name\":\"ContainerName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ContainerName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ContainerName\",\"subType\":\"multi\"},{\"name\":\"Data\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Data.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Data\",\"subType\":\"multi\"},{\"name\":\"Enforcer\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Enforcer.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Enforcer\",\"subType\":\"multi\"},{\"name\":\"HostName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"HostName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"HostName\",\"subType\":\"multi\"},{\"name\":\"HostPID\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"HostPPID\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Labels\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Labels.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Labels\",\"subType\":\"multi\"},{\"name\":\"Message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Message\",\"subType\":\"multi\"},{\"name\":\"NamespaceName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"NamespaceName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"NamespaceName\",\"subType\":\"multi\"},{\"name\":\"Operation\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Operation.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Operation\",\"subType\":\"multi\"},{\"name\":\"PID\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"PPID\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"ParentProcessName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":1,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ParentProcessName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ParentProcessName\",\"subType\":\"multi\"},{\"name\":\"PodName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"PodName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"PodName\",\"subType\":\"multi\"},{\"name\":\"PolicyName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"PolicyName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"PolicyName\",\"subType\":\"multi\"},{\"name\":\"ProcessName\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"ProcessName.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"ProcessName\",\"subType\":\"multi\"},{\"name\":\"Resource\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Resource.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Resource\",\"subType\":\"multi\"},{\"name\":\"Result\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Result.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Result\",\"subType\":\"multi\"},{\"name\":\"Severity\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":2,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Severity.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Severity\",\"subType\":\"multi\"},{\"name\":\"Source\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Source.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Source\",\"subType\":\"multi\"},{\"name\":\"Timestamp\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"Type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"Type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"parent\":\"Type\",\"subType\":\"multi\"},{\"name\":\"UID\",\"type\":\"number\",\"esTypes\":[\"long\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"UpdatedTime\",\"type\":\"date\",\"esTypes\":[\"date\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]","timeFieldName":"UpdatedTime","title":"alert"},"id":"7b01e390-e6d4-11ed-b89b-4f76d1177612","migrationVersion":{"index-pattern":"6.5.0"},"references":[],"type":"index-pattern","updated_at":"2023-05-06T02:06:17.865Z","version":"WzQ4LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Alert based on Operations","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Alert based on Operations\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"series\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"Operation.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"NamespaceName.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"cc7b5190-e8b2-11ed-8b33-8379e48ef322","migrationVersion":{"visualization":"7.2.0"},"references":[{"id":"7b01e390-e6d4-11ed-b89b-4f76d1177612","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-06T02:06:17.865Z","version":"WzQ5LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Alert from Different Namespace","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Alert from Different Namespace\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"NamespaceName.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"2cc01810-e8b3-11ed-8b33-8379e48ef322","migrationVersion":{"visualization":"7.2.0"},"references":[{"id":"7b01e390-e6d4-11ed-b89b-4f76d1177612","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-06T02:06:17.865Z","version":"WzUwLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"NameSpace Matched Policy Count ","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"NameSpace Matched Policy Count \",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"NamespaceName.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"PolicyName.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"0f1ac9f0-e8b2-11ed-8b33-8379e48ef322","migrationVersion":{"visualization":"7.2.0"},"references":[{"id":"7b01e390-e6d4-11ed-b89b-4f76d1177612","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-06T02:06:17.865Z","version":"WzUxLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"alias\":null,\"negate\":true,\"type\":\"phrase\",\"key\":\"PolicyName.keyword\",\"value\":\"DefaultPosture\",\"params\":{\"query\":\"DefaultPosture\"},\"disabled\":false,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"PolicyName.keyword\":{\"query\":\"DefaultPosture\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Policy and Action Summary","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Policy and Action Summary\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"PolicyName.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"Action.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"f651d580-e8bb-11ed-8b33-8379e48ef322","migrationVersion":{"visualization":"7.2.0"},"references":[{"id":"7b01e390-e6d4-11ed-b89b-4f76d1177612","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"7b01e390-e6d4-11ed-b89b-4f76d1177612","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-06T02:06:17.865Z","version":"WzUyLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Alert from Different Pods ","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Alert from Different Pods \",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"PodName.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"00eebf70-e8b3-11ed-8b33-8379e48ef322","migrationVersion":{"visualization":"7.2.0"},"references":[{"id":"7b01e390-e6d4-11ed-b89b-4f76d1177612","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-06T02:06:17.865Z","version":"WzUzLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Alert Metric","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Alert Metric\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}},\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"bucket\":{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"alerts\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"NamespaceName.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"1b9b6500-e8ca-11ed-8b33-8379e48ef322","migrationVersion":{"visualization":"7.2.0"},"references":[{"id":"7b01e390-e6d4-11ed-b89b-4f76d1177612","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-06T02:06:17.865Z","version":"WzU0LDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Namespace Severity Summary ","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Namespace Severity Summary \",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"dimensions\":{\"x\":{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"},\"y\":[{\"accessor\":2,\"format\":{\"id\":\"number\"},\"params\":{},\"aggType\":\"count\"}],\"series\":[{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\"}},\"params\":{},\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"NamespaceName.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"\\\" \\\"\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"Severity.keyword\",\"order\":\"desc\",\"size\":5,\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"ce153e90-e8ca-11ed-8b33-8379e48ef322","migrationVersion":{"visualization":"7.2.0"},"references":[{"id":"7b01e390-e6d4-11ed-b89b-4f76d1177612","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-06T02:06:17.865Z","version":"WzU1LDFd"} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":30,\"w\":24,\"h\":15,\"i\":\"1\"},\"panelIndex\":\"1\",\"version\":\"7.2.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{\"vis\":{\"colors\":{\"wordpress-mysql\":\"#F2C96D\",\"multiubuntu\":\"#629E51\"}}},\"gridData\":{\"x\":24,\"y\":15,\"w\":24,\"h\":15,\"i\":\"2\"},\"panelIndex\":\"2\",\"version\":\"7.2.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":45,\"w\":24,\"h\":15,\"i\":\"3\"},\"panelIndex\":\"3\",\"version\":\"7.2.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":24,\"y\":30,\"w\":24,\"h\":15,\"i\":\"4\"},\"panelIndex\":\"4\",\"version\":\"7.2.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"vis\":{\"colors\":{\"ubuntu-2-deployment-78fcd99688-zmqhg\":\"#EF843C\"}}},\"gridData\":{\"x\":0,\"y\":15,\"w\":24,\"h\":15,\"i\":\"5\"},\"panelIndex\":\"5\",\"version\":\"7.2.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{},\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":15,\"i\":\"7\"},\"panelIndex\":\"7\",\"version\":\"7.2.0\",\"panelRefName\":\"panel_5\"},{\"gridData\":{\"x\":24,\"y\":45,\"w\":24,\"h\":15,\"i\":\"8\"},\"version\":\"7.2.0\",\"panelIndex\":\"8\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"}]","timeRestore":false,"title":"KA","version":1},"id":"5d818630-e8ce-11ed-8b33-8379e48ef322","migrationVersion":{"dashboard":"7.0.0"},"references":[{"id":"cc7b5190-e8b2-11ed-8b33-8379e48ef322","name":"panel_0","type":"visualization"},{"id":"2cc01810-e8b3-11ed-8b33-8379e48ef322","name":"panel_1","type":"visualization"},{"id":"0f1ac9f0-e8b2-11ed-8b33-8379e48ef322","name":"panel_2","type":"visualization"},{"id":"f651d580-e8bb-11ed-8b33-8379e48ef322","name":"panel_3","type":"visualization"},{"id":"00eebf70-e8b3-11ed-8b33-8379e48ef322","name":"panel_4","type":"visualization"},{"id":"1b9b6500-e8ca-11ed-8b33-8379e48ef322","name":"panel_5","type":"visualization"},{"id":"ce153e90-e8ca-11ed-8b33-8379e48ef322","name":"panel_6","type":"visualization"}],"type":"dashboard","updated_at":"2023-05-06T02:06:17.865Z","version":"WzU2LDFd"} \ No newline at end of file diff --git a/relay-server/elasticsearch/adapter.go b/relay-server/elasticsearch/adapter.go new file mode 100644 index 0000000..2e06013 --- /dev/null +++ b/relay-server/elasticsearch/adapter.go @@ -0,0 +1,208 @@ +package elasticsearch + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log" + "strings" + "sync/atomic" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/dustin/go-humanize" + "github.com/elastic/go-elasticsearch/v7" + "github.com/elastic/go-elasticsearch/v7/esutil" + "github.com/google/uuid" + kg "github.com/kubearmor/kubearmor-relay-server/relay-server/log" + "github.com/kubearmor/kubearmor-relay-server/relay-server/server" +) + +var ( + countSuccessful uint64 + countEntered uint64 + start time.Time +) + +// ElasticsearchClient Structure +type ElasticsearchClient struct { + kaClient *server.LogClient + esClient *elasticsearch.Client + cancel context.CancelFunc + bulkIndexer esutil.BulkIndexer + ctx context.Context + alertCh chan interface{} +} + +// NewElasticsearchClient creates a new Elasticsearch client with the given Elasticsearch URL +// and kubearmor LogClient with endpoint. It has a retry mechanism for certain HTTP status codes and a backoff function for retry delays. +// It then creates a new NewBulkIndexer with the esClient +func NewElasticsearchClient(esURL, Endpoint string) (*ElasticsearchClient, error) { + retryBackoff := backoff.NewExponentialBackOff() + cfg := elasticsearch.Config{ + Addresses: []string{esURL}, + + // Retry on 429 TooManyRequests statuses + RetryOnStatus: []int{502, 503, 504, 429}, + + // Configure the backoff function + RetryBackoff: func(i int) time.Duration { + if i == 1 { + retryBackoff.Reset() + } + return retryBackoff.NextBackOff() + }, + MaxRetries: 5, + } + + esClient, err := elasticsearch.NewClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create Elasticsearch client: %v", err) + } + bi, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{ + Client: esClient, // The Elasticsearch client + FlushBytes: 1000000, // The flush threshold in bytes [1mb] + FlushInterval: 30 * time.Second, // The periodic flush interval [30 secs] + }) + if err != nil { + log.Fatalf("Error creating the indexer: %s", err) + } + alertCh := make(chan interface{}, 10000) + kaClient := server.NewClient(Endpoint) + return &ElasticsearchClient{kaClient: kaClient, bulkIndexer: bi, esClient: esClient, alertCh: alertCh}, nil +} + +// bulkIndex takes an interface and index name and adds the data to the Elasticsearch bulk indexer. +// The bulk indexer flushes after the FlushBytes or FlushInterval thresholds are reached. +// The method generates a UUID as the document ID and includes success and failure callbacks for each item added to the bulk indexer. +func (ecl *ElasticsearchClient) bulkIndex(a interface{}, index string) { + countEntered++ + data, err := json.Marshal(a) + if err != nil { + log.Fatalf("Error marshaling data: %s", err) + } + + err = ecl.bulkIndexer.Add( + ecl.ctx, + esutil.BulkIndexerItem{ + Index: index, + Action: "index", + DocumentID: uuid.New().String(), + Body: bytes.NewReader(data), + OnSuccess: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem) { + atomic.AddUint64(&countSuccessful, 1) + }, + OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, res esutil.BulkIndexerResponseItem, err error) { + if err != nil { + log.Printf("ERROR: %s", err) + } else { + log.Printf("ERROR: %s: %s", res.Error.Type, res.Error.Reason) + } + }, + }, + ) + + if err != nil { + log.Fatalf("Error adding items to bulk indexer: %s", err) + } +} + +// Start starts the Elasticsearch client by performing a health check on the gRPC server +// and starting goroutines to consume messages from the alert channel and bulk index them. +// The method starts a goroutine for each stream and waits for messages to be received. +// Additional goroutines consume alert from the alert channel and bulk index them. +func (ecl *ElasticsearchClient) Start() error { + start = time.Now() + client := ecl.kaClient + ecl.ctx, ecl.cancel = context.WithCancel(context.Background()) + + // do healthcheck + if ok := client.DoHealthCheck(); !ok { + return fmt.Errorf("failed to check the liveness of the gRPC server") + } + kg.Printf("Checked the liveness of the gRPC server") + + client.WgServer.Add(1) + go func() { + defer client.WgServer.Done() + for client.Running { + res, err := client.AlertStream.Recv() + if err != nil { + kg.Warnf("Failed to receive an alert (%s)", client.Server) + break + } + tel, _ := json.Marshal(res) + fmt.Printf("%s\n", string(tel)) + ecl.alertCh <- res + } + }() + + for i := 0; i < 5; i++ { + go func() { + for { + select { + case alert := <-ecl.alertCh: + ecl.bulkIndex(alert, "alert") + case <-ecl.ctx.Done(): + close(ecl.alertCh) + return + } + } + }() + } + return nil +} + +// Stop stops the Elasticsearch client and performs necessary cleanup operations. +// It stops the Kubearmor Relay client, closes the BulkIndexer and cancels the context. +func (ecl *ElasticsearchClient) Stop() error { + logClient := ecl.kaClient + logClient.Running = false + time.Sleep(2 * time.Second) + + //Destoy KubeArmor Relay Client + if err := logClient.DestroyClient(); err != nil { + return fmt.Errorf("failed to destroy the kubearmor relay gRPC client (%s)", err.Error()) + } + kg.Printf("Destroyed kubearmor relay gRPC client") + + //Close BulkIndexer + if err := ecl.bulkIndexer.Close(ecl.ctx); err != nil { + kg.Errf("Unexpected error: %s", err) + } + + ecl.cancel() + + kg.Printf("Stopped kubearmor receiver") + time.Sleep(2 * time.Second) + ecl.PrintBulkStats() + return nil +} + +// PrintBulkStats prints data on the bulk indexing process, including the number of indexed documents, +// the number of errors, and the indexing rate , after elasticsearch client stops +func (ecl *ElasticsearchClient) PrintBulkStats() { + biStats := ecl.bulkIndexer.Stats() + println(strings.Repeat("▔", 80)) + + dur := time.Since(start) + + if biStats.NumFailed > 0 { + fmt.Printf( + "Indexed [%s] documents with [%s] errors in %s (%s docs/sec)", + humanize.Comma(int64(biStats.NumFlushed)), + humanize.Comma(int64(biStats.NumFailed)), + dur.Truncate(time.Millisecond), + humanize.Comma(int64(1000.0/float64(dur/time.Millisecond)*float64(biStats.NumFlushed))), + ) + } else { + log.Printf( + "Sucessfuly indexed [%s] documents in %s (%s docs/sec)", + humanize.Comma(int64(biStats.NumFlushed)), + dur.Truncate(time.Millisecond), + humanize.Comma(int64(1000.0/float64(dur/time.Millisecond)*float64(biStats.NumFlushed))), + ) + } + println(strings.Repeat("▔", 80)) +} diff --git a/relay-server/go.mod b/relay-server/go.mod index 18cf18f..fbb1bf2 100644 --- a/relay-server/go.mod +++ b/relay-server/go.mod @@ -10,6 +10,9 @@ replace ( ) require ( + github.com/cenkalti/backoff/v4 v4.2.1 + github.com/dustin/go-humanize v1.0.1 + github.com/elastic/go-elasticsearch/v7 v7.17.7 github.com/google/uuid v1.3.0 github.com/kubearmor/KubeArmor/protobuf v0.0.0-20230426155201-4a0d0af2a5d6 go.uber.org/zap v1.19.0 diff --git a/relay-server/go.sum b/relay-server/go.sum index 19f1e4b..03f79de 100644 --- a/relay-server/go.sum +++ b/relay-server/go.sum @@ -622,6 +622,8 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= @@ -651,6 +653,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elastic/go-elasticsearch/v7 v7.17.7 h1:pcYNfITNPusl+cLwLN6OLmVT+F73Els0nbaWOmYachs= +github.com/elastic/go-elasticsearch/v7 v7.17.7/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= diff --git a/relay-server/main.go b/relay-server/main.go index 3eeeff2..8840e02 100644 --- a/relay-server/main.go +++ b/relay-server/main.go @@ -9,6 +9,8 @@ import ( "os/signal" "syscall" + "github.com/kubearmor/kubearmor-relay-server/relay-server/elasticsearch" + kg "github.com/kubearmor/kubearmor-relay-server/relay-server/log" "github.com/kubearmor/kubearmor-relay-server/relay-server/server" ) @@ -50,6 +52,14 @@ func main() { gRPCPortPtr := flag.String("gRPCPort", "32767", "gRPC port") flag.Parse() + //get env + enableEsDashboards := os.Getenv("ENABLE_DASHBOARDS") + esUrl := os.Getenv("ES_URL") + endPoint := os.Getenv("KUBEARMOR_SERVICE") + if endPoint == "" { + endPoint = "localhost:32767" + } + // == // // create a relay server @@ -68,6 +78,20 @@ func main() { go relayServer.GetFeedsFromNodes() kg.Print("Started to receive log feeds from each node") + // == // + + // check and start an elasticsearch client + if enableEsDashboards == "true" { + esCl, err := elasticsearch.NewElasticsearchClient(esUrl, endPoint) + if err != nil { + kg.Warnf("Failed to start a Elasticsearch Client") + } + go esCl.Start() + defer esCl.Stop() + } + + // == // + // listen for interrupt signals sigChan := GetOSSigChannel() <-sigChan diff --git a/relay-server/server/relayServer.go b/relay-server/server/relayServer.go index 9258a00..a2fc38a 100644 --- a/relay-server/server/relayServer.go +++ b/relay-server/server/relayServer.go @@ -273,8 +273,8 @@ type LogClient struct { // flag Running bool - // server - server string + // Server + Server string // connection conn *grpc.ClientConn @@ -283,13 +283,13 @@ type LogClient struct { client pb.LogServiceClient // messages - msgStream pb.LogService_WatchMessagesClient + MsgStream pb.LogService_WatchMessagesClient // alerts - alertStream pb.LogService_WatchAlertsClient + AlertStream pb.LogService_WatchAlertsClient // logs - logStream pb.LogService_WatchLogsClient + LogStream pb.LogService_WatchLogsClient // wait group WgServer sync.WaitGroup @@ -305,9 +305,9 @@ func NewClient(server string) *LogClient { // == // - lc.server = server + lc.Server = server - lc.conn, err = grpc.Dial(lc.server, grpc.WithInsecure()) + lc.conn, err = grpc.Dial(lc.Server, grpc.WithInsecure()) if err != nil { kg.Warnf("Failed to connect to KubeArmor's gRPC service (%s)", server) return nil @@ -328,7 +328,7 @@ func NewClient(server string) *LogClient { msgIn := pb.RequestMessage{} msgIn.Filter = "all" - lc.msgStream, err = lc.client.WatchMessages(context.Background(), &msgIn) + lc.MsgStream, err = lc.client.WatchMessages(context.Background(), &msgIn) if err != nil { kg.Warnf("Failed to call WatchMessages (%s) err=%s\n", server, err.Error()) return nil @@ -339,7 +339,7 @@ func NewClient(server string) *LogClient { alertIn := pb.RequestMessage{} alertIn.Filter = "policy" - lc.alertStream, err = lc.client.WatchAlerts(context.Background(), &alertIn) + lc.AlertStream, err = lc.client.WatchAlerts(context.Background(), &alertIn) if err != nil { kg.Warnf("Failed to call WatchAlerts (%s) err=%s\n", server, err.Error()) return nil @@ -350,7 +350,7 @@ func NewClient(server string) *LogClient { logIn := pb.RequestMessage{} logIn.Filter = "system" - lc.logStream, err = lc.client.WatchLogs(context.Background(), &logIn) + lc.LogStream, err = lc.client.WatchLogs(context.Background(), &logIn) if err != nil { kg.Warnf("Failed to call WatchLogs (%s)\n err=%s", server, err.Error()) return nil @@ -373,7 +373,7 @@ func (lc *LogClient) DoHealthCheck() bool { nonce := pb.NonceMessage{Nonce: randNum} res, err := lc.client.HealthCheck(context.Background(), &nonce) if err != nil { - kg.Warnf("Failed to check the liveness of KubeArmor's gRPC service (%s)", lc.server) + kg.Warnf("Failed to check the liveness of KubeArmor's gRPC service (%s)", lc.Server) return false } @@ -395,8 +395,8 @@ func (lc *LogClient) WatchMessages() error { for lc.Running { var res *pb.Message - if res, err = lc.msgStream.Recv(); err != nil { - kg.Warnf("Failed to receive a message (%s)", lc.server) + if res, err = lc.MsgStream.Recv(); err != nil { + kg.Warnf("Failed to receive a message (%s)", lc.Server) break } @@ -420,7 +420,7 @@ func (lc *LogClient) WatchMessages() error { MsgLock.RUnlock() } - kg.Print("Stopped watching messages from " + lc.server) + kg.Print("Stopped watching messages from " + lc.Server) return nil } @@ -435,8 +435,8 @@ func (lc *LogClient) WatchAlerts() error { for lc.Running { var res *pb.Alert - if res, err = lc.alertStream.Recv(); err != nil { - kg.Warnf("Failed to receive an alert (%s)", lc.server) + if res, err = lc.AlertStream.Recv(); err != nil { + kg.Warnf("Failed to receive an alert (%s)", lc.Server) break } @@ -460,7 +460,7 @@ func (lc *LogClient) WatchAlerts() error { AlertLock.RUnlock() } - kg.Print("Stopped watching alerts from " + lc.server) + kg.Print("Stopped watching alerts from " + lc.Server) return nil } @@ -475,8 +475,8 @@ func (lc *LogClient) WatchLogs() error { for lc.Running { var res *pb.Log - if res, err = lc.logStream.Recv(); err != nil { - kg.Warnf("Failed to receive a log (%s)", lc.server) + if res, err = lc.LogStream.Recv(); err != nil { + kg.Warnf("Failed to receive a log (%s)", lc.Server) break } @@ -499,7 +499,7 @@ func (lc *LogClient) WatchLogs() error { LogLock.RUnlock() } - kg.Print("Stopped watching logs from " + lc.server) + kg.Print("Stopped watching logs from " + lc.Server) return nil }