Use Helm chart for Kafka cluster setup (#477)

* Remove trailing whitespace in CRDs in Helm chart * Add Chart manifest for Kafka cluster * Move Kafka cluster & user manifests to chart templates * Add basic parameterisation for Kafka cluster values * Add enable toggles for plaintext & TLS listeners Also change structure of Helm values to be nested for plaintext & TLS listener config. * Parameterise listener types & change to internal by default * Add Helm vars for broker probe timeouts * Add Helm vars for topic config * Add Helm vars for broker storage config * Add Helm vars for Zookeeper probe timeouts * Add Helm vars for Zookeeper storage config * Separate metrics config to separate Helm template * Reorder Ansible Kafka vars for logical grouping * Use feature gate toggle in Kafka role instead of duplicating logic * Use Kafka cluster Helm chart in Ansible * Fix whitespace chomping issues in cluster template * Add Python requirements for Ansible playbooks * Add Ansible Galaxy requirements for playbooks * Disable Grafana installation in Ansible ecosystem * Remove unused value Variables set in one role are not visible to others. Inheritence is hierarchical and does not include siblings. * Force reinstallation of Kafka Helm chart in case of changes to the chart itself This is mostly a convenience when developing, as we would otherwise expect published chart versions which should not be changing (patches are new versions). * Allow passing Helm values to Kafka chart via Ansible * Allow disabling Kafka metrics in Helm chart * Allow disabling KafkaUser CR creation in Helm chart
SeldonIO · Oct 10, 2022 · c482de1 · c482de1
1 parent 6f2bf4a
commit c482de1
Show file tree

Hide file tree

Showing 11 changed files with 172 additions and 87 deletions.
diff --git a/ansible/playbooks/vars/default.yaml b/ansible/playbooks/vars/default.yaml
@@ -9,7 +9,9 @@ seldon_monitoring_prometheus_operator_values:
 
 # Strimzi Kafka Configuration
 strimzi_kafka_namespace: seldon-mesh
+strimzi_kafka_create_cluster: false
+strimzi_kafka_add_feature_gates: true
 strimzi_kafka_install_prometheus_operator: false
 strimzi_kafka_create_prometheus_instance: false
-strimzi_kafka_create_cluster: false
+strimzi_kafka_install_grafana: false
 strimzi_kafka_grafana_prometheus_url: "http://seldon-monitoring-prometheus.{{ seldon_monitoring_namespace }}.svc:9090"
diff --git a/ansible/requirements.txt b/ansible/requirements.txt
@@ -0,0 +1,4 @@
+ansible>=4.2.0,<5.0.0
+docker>=5.0.0,<6.0.0
+openshift>=0.12.0,<0.13.0
+passlib>=1.7.4,<2.0.0
diff --git a/ansible/requirements.yml b/ansible/requirements.yml
@@ -0,0 +1,6 @@
+collections:
+  - name: https://github.com/SeldonIO/ansible-k8s-collection.git
+    type: git
+    version: ddad72dcff53dd8ca05ca477f74df7992fd489fc
+  - name: kubernetes.core
+    version: 2.3.0
diff --git a/ansible/roles/ecosystem/defaults/main.yaml b/ansible/roles/ecosystem/defaults/main.yaml
@@ -2,7 +2,9 @@ repo_root_dir: "{{ inventory_dir }}/.."
 
 seldon_mesh_namespace: seldon-mesh
 kafka_namespace: seldon-mesh
-strimzi_kafka_namespace: kafka
+
+kafka_wait_timeout: "300s"
+kafka_cluster_values: {}
 
 # Installation on/off flags
 install_prometheus: true

diff --git a/ansible/roles/ecosystem/tasks/kafka.yaml b/ansible/roles/ecosystem/tasks/kafka.yaml
@@ -3,17 +3,14 @@
     name: seldonio.k8s.strimzi_kafka
   when: install_kafka | bool
 
-- name: Enable KRaft
-  shell:
-      " kubectl set env deployment/strimzi-cluster-operator STRIMZI_FEATURE_GATES=+UseStrimziPodSets,+UseKRaft -n {{ strimzi_kafka_namespace }} "
-  tags: kraft
-  when: ecosystem_enable_kraft | bool
-
-- name: Create Kafka Cluster
-  kubernetes.core.k8s:
+- name: "Create Kafka Cluster"
+  kubernetes.core.helm:
     state: present
+    force: yes
     namespace: "{{ kafka_namespace }}"
-    template: "{{ repo_root_dir }}/{{ item }}"
-  with_items:
-    - kafka/strimzi/cluster.yaml
+    name: "seldon-core-v2-kafka"
+    chart_ref: "{{ repo_root_dir }}/kafka/strimzi"
+    values: "{{ kafka_cluster_values }}"
+    wait: yes
+    wait_timeout: "{{ kafka_wait_timeout }}"
   when: ecosystem_configure_kafka | bool
diff --git a/k8s/helm-charts/seldon-core-v2-setup/values.yaml b/k8s/helm-charts/seldon-core-v2-setup/values.yaml
@@ -62,7 +62,7 @@ security:
           crtPath: /tmp/certs/ddc/tls.crt
           caPath: /tmp/certs/ddc/ca.crt
           serverCaPath: /tmp/certs/dds/ca.crt
-  
+
 opentelemetry:
   endpoint: seldon-collector:4317
   enable: true
@@ -153,7 +153,7 @@ scheduler:
     registry: docker.io
     repository: seldonio/seldon-scheduler
     tag: latest
-      
+
 serverConfig:
   rclone:
     image:

diff --git a/kafka/strimzi/Chart.yaml b/kafka/strimzi/Chart.yaml
@@ -0,0 +1,14 @@
+apiVersion: v2
+type: application
+
+version: 0.1.0
+appVersion: "0.2.0-dev"
+name: "seldon-core-v2-kafka"
+description: "A Kafka cluster suitable for Seldon Core v2"
+maintainers:
+  - name: Seldon Technologies Ltd
+    email: hello@seldon.io
+    url: www.seldon.io
+home: https://docs.seldon.io/projects/seldon-core-v2
+sources:
+  - https://github.com/SeldonIO/seldon-core-v2/kafka/strimzi
diff --git a/kafka/strimzi/templates/cluster.yaml b/kafka/strimzi/templates/cluster.yaml
@@ -0,0 +1,74 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: {{ .Values.cluster.name }}
+spec:
+  entityOperator:
+    userOperator: {}
+  kafka:
+    version: {{ .Values.cluster.version }}
+    replicas: {{ .Values.broker.replicas }}
+    listeners:
+      {{- if .Values.broker.plaintext.enabled }}
+      - name: plain
+        port: {{ .Values.broker.plaintext.port }}
+        type: {{ .Values.broker.plaintext.listenerType }}
+        tls: false
+      {{- end }}
+      {{- if .Values.broker.tls.enabled }}
+      - name: tls
+        port: {{ .Values.broker.tls.port }}
+        type: {{ .Values.broker.tls.listenerType }}
+        tls: true
+        authentication:
+          type: tls
+      {{- end }}
+    readinessProbe:
+      initialDelaySeconds: {{ .Values.broker.readiness.initialDelaySeconds }}
+      timeoutSeconds: {{ .Values.broker.readiness.timeoutSeconds }}
+    livenessProbe:
+      initialDelaySeconds: {{ .Values.broker.liveness.initialDelaySeconds }}
+      timeoutSeconds: {{ .Values.broker.liveness.timeoutSeconds }}
+    config:
+      auto.create.topics.enable: {{ .Values.topic.autoCreate }}
+      offsets.topic.replication.factor: {{ .Values.topic.offsetReplicationFactor }}
+      transaction.state.log.replication.factor: {{ .Values.topic.txStateReplicationFactor }}
+      transaction.state.log.min.isr: {{ .Values.topic.txStateMinISR }}
+      default.replication.factor: {{ .Values.topic.defaultReplicationFactor }}
+      min.insync.replicas: {{ .Values.topic.minISR }}
+      inter.broker.protocol.version: {{ .Values.broker.interBrokerProtocolVersion }}
+    template:
+      pod:
+        tmpDirSizeLimit: {{ .Values.broker.tmpDirSizeLimit }}
+    storage:
+      type: jbod
+      volumes:
+      - id: 0
+        type: persistent-claim
+        size: {{ .Values.broker.pvcSize }}
+        deleteClaim: false
+    metricsConfig:
+      type: jmxPrometheusExporter
+      valueFrom:
+        configMapKeyRef:
+          name: kafka-metrics
+          key: kafka-metrics-config.yml
+  # zookeeper settings should not be use in case of STRIMZI_FEATURE_GATES=+UseStrimziPodSets,+UseKRaft (raft)
+  # to enable raft run:
+  # `kubectl set env deployment/strimzi-cluster-operator STRIMZI_FEATURE_GATES=+UseStrimziPodSets,+UseKRaft -n kafka`
+  # which is the default with ansible install
+  zookeeper:
+    replicas: {{ .Values.zookeeper.replicas }}
+    readinessProbe:
+      initialDelaySeconds: {{ .Values.zookeeper.readiness.initialDelaySeconds }}
+      timeoutSeconds: {{ .Values.zookeeper.readiness.timeoutSeconds }}
+    livenessProbe:
+      initialDelaySeconds: {{ .Values.zookeeper.liveness.initialDelaySeconds }}
+      timeoutSeconds: {{ .Values.zookeeper.liveness.timeoutSeconds }}
+    storage:
+      type: persistent-claim
+      size: {{ .Values.zookeeper.pvcSize }}
+      deleteClaim: false
+  kafkaExporter:
+    topicRegex: ".*"
+    groupRegex: ".*"
diff --git a/kafka/strimzi/cluster.yaml → kafka/strimzi/templates/metrics.yaml b/kafka/strimzi/cluster.yaml → kafka/strimzi/templates/metrics.yaml
@@ -1,74 +1,4 @@
-apiVersion: kafka.strimzi.io/v1beta2
-kind: Kafka
-metadata:
-  name: seldon
-spec:
-  entityOperator:
-    userOperator: {}
-  kafka:
-    version: 3.2.0
-    replicas: 3
-    listeners:
-      - name: plain
-        port: 9092
-        type: loadbalancer
-        tls: false
-      - name: tls
-        port: 9093
-        type: loadbalancer
-        tls: true
-        authentication:
-          type: tls
-    readinessProbe:
-      initialDelaySeconds: 15
-      timeoutSeconds: 5
-    livenessProbe:
-      initialDelaySeconds: 15
-      timeoutSeconds: 5
-    config:
-      auto.create.topics.enable: true
-      offsets.topic.replication.factor: 1
-      transaction.state.log.replication.factor: 1
-      transaction.state.log.min.isr: 1
-      default.replication.factor: 1
-      min.insync.replicas: 1
-      inter.broker.protocol.version: "3.2"
-    template:
-      pod:
-        tmpDirSizeLimit: 100Mi
-    storage:
-      type: jbod
-      volumes:
-      - id: 0
-        type: persistent-claim
-        size: 200Gi
-        deleteClaim: false
-    metricsConfig:
-      type: jmxPrometheusExporter
-      valueFrom:
-        configMapKeyRef:
-          name: kafka-metrics
-          key: kafka-metrics-config.yml
-  # zookeeper settings should not be use in case of STRIMZI_FEATURE_GATES=+UseStrimziPodSets,+UseKRaft (raft)
-  # to enable raft run:
-  # `kubectl set env deployment/strimzi-cluster-operator STRIMZI_FEATURE_GATES=+UseStrimziPodSets,+UseKRaft -n kafka`
-  # which is the default with ansible install
-  zookeeper:
-    replicas: 1
-    readinessProbe:
-      initialDelaySeconds: 15
-      timeoutSeconds: 5
-    livenessProbe:
-      initialDelaySeconds: 15
-      timeoutSeconds: 5
-    storage:
-      type: persistent-claim
-      size: 100Gi
-      deleteClaim: false
-  kafkaExporter:
-    topicRegex: ".*"
-    groupRegex: ".*"
----
+{{- if .Values.metrics.enabled }}
 kind: ConfigMap
 apiVersion: v1
 metadata:
@@ -231,4 +161,4 @@ data:
       labels:
         replicaId: "$2"
         memberType: "$3"
-
+{{- end }}
diff --git a/kafka/strimzi/user.yaml → kafka/strimzi/templates/user.yaml b/kafka/strimzi/user.yaml → kafka/strimzi/templates/user.yaml
@@ -1,3 +1,4 @@
+{{- if .Values.broker.tls.enabled }}
 apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaUser
 metadata:
@@ -7,3 +8,4 @@ metadata:
 spec:
   authentication:
     type: tls
+{{- end }}
diff --git a/kafka/strimzi/values.yaml b/kafka/strimzi/values.yaml
@@ -0,0 +1,54 @@
+cluster:
+  name: "seldon"
+  version: "3.2.0"
+
+metrics:
+  enabled: true
+
+broker:
+  replicas: 3
+
+  plaintext:
+    enabled: true
+    port: 9092
+    listenerType: internal
+
+  tls:
+    enabled: true
+    port: 9093
+    listenerType: internal
+
+  readiness:
+    initialDelaySeconds: 15
+    timeoutSeconds: 5
+
+  liveness:
+    initialDelaySeconds: 15
+    timeoutSeconds: 5
+
+  interBrokerProtocolVersion: "3.2"
+
+  tmpDirSizeLimit: 100Mi
+
+  pvcSize: 100Gi
+
+topic:
+  autoCreate: true
+  offsetReplicationFactor: 1
+  txStateReplicationFactor: 1
+  txStateMinISR: 1
+  defaultReplicationFactor: 1
+  minISR: 1
+
+zookeeper:
+  replicas: 1
+
+  readiness:
+    initialDelaySeconds: 15
+    timeoutSeconds: 5
+
+  liveness:
+    initialDelaySeconds: 15
+    timeoutSeconds: 5
+
+    pvcSize: 100Gi