diff --git a/README.md b/README.md index ce2009dc..3f7b35e9 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ One option is to keep kubernets-kafka as a git submodule and edit the relative p Have a look at: * [./prometheus](./prometheus/) * [./linkedin-burrow](./linkedin-burrow/) + * [./consumers-prometheus](./consumers-prometheus/) * [or plain JMX](https://github.com/Yolean/kubernetes-kafka/pull/96) * what's happening in the [monitoring](https://github.com/Yolean/kubernetes-kafka/labels/monitoring) label. * Note that this repo is intentionally light on [automation](https://github.com/Yolean/kubernetes-kafka/labels/automation). We think every SRE team must build the operational knowledge first. diff --git a/consumers-prometheus/kafka-minion-service.yaml b/consumers-prometheus/kafka-minion-service.yaml new file mode 100644 index 00000000..23678753 --- /dev/null +++ b/consumers-prometheus/kafka-minion-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: metrics-minion + namespace: kafka + labels: &labels + app: kafka-minion + type: openmetrics +spec: + selector: *labels + ports: + - name: http + port: 8080 diff --git a/consumers-prometheus/kafka-minion.yaml b/consumers-prometheus/kafka-minion.yaml new file mode 100644 index 00000000..e1aa8721 --- /dev/null +++ b/consumers-prometheus/kafka-minion.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metrics-minion + namespace: kafka + labels: &labels + app: kafka-minion + type: openmetrics +spec: + replicas: 1 + selector: + matchLabels: *labels + template: + metadata: + labels: *labels + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: /metrics + spec: + containers: + - name: kafka-minion + image: solsson/kafka-consumers-prometheus@sha256:a005aa02581fa46884b4a4499f7b4d56e889770c64e7e84d0a9ad195935fa59c + env: + - name: TELEMETRY_HOST + value: 0.0.0.0 + - name: TELEMETRY_PORT + value: "8080" + - name: EXPORTER_IGNORE_SYSTEM_TOPICS + value: "true" + - name: EXPORTER_METRICS_PREFIX + value: kafka_minion + - name: LOG_LEVEL + value: info + - name: KAFKA_BROKERS + value: kafka-0.broker:9092, kafka-1.broker:9092, kafka-2.broker:9092 + - name: KAFKA_CONSUMER_OFFSETS_TOPIC_NAME + value: __consumer_offsets + ports: + - name: http + containerPort: 8080 + readinessProbe: + httpGet: + port: http + path: /readycheck + livenessProbe: + httpGet: + port: http + path: /healthcheck diff --git a/consumers-prometheus/kustomization.yaml b/consumers-prometheus/kustomization.yaml new file mode 100644 index 00000000..d74b09d0 --- /dev/null +++ b/consumers-prometheus/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- kafka-minion-service.yaml +- kafka-minion.yaml diff --git a/prometheus/50-kafka-jmx-exporter-patch.yml b/prometheus/50-kafka-jmx-exporter-patch.yml index 179cbb5e..13b10181 100644 --- a/prometheus/50-kafka-jmx-exporter-patch.yml +++ b/prometheus/50-kafka-jmx-exporter-patch.yml @@ -1,4 +1,4 @@ -# meant to be applied using +# meant to be applied using kustomize, or with pre-1.14 kubectl: # kubectl --namespace kafka patch statefulset kafka --patch "$(cat prometheus/50-kafka-jmx-exporter-patch.yml )" apiVersion: apps/v1 kind: StatefulSet diff --git a/prometheus/README.md b/prometheus/README.md index 64629108..0ac318cd 100644 --- a/prometheus/README.md +++ b/prometheus/README.md @@ -1,10 +1,27 @@ # Export metrics to Prometheus -Kafka uses JMX to expose metrics, as is already [enabled](https://github.com/Yolean/kubernetes-kafka/pull/96) for broker pods. There's many ways to use JMX. For example [Kafka Manager](../yahoo-kafka-manager/) uses it to display current broker traffic. +JMX is already [enabled](https://github.com/Yolean/kubernetes-kafka/pull/96) for broker pods (TODO extract to kustomization). There's many ways to use JMX. For example [Kafka Manager](../yahoo-kafka-manager/) uses it to display current broker traffic. -At Yolean we use Prometheus. This folder adds a sidecar to the broker pods that exports selected JMX metrics over HTTP in Prometheus format. To add a container to an existing pod we must use the `patch`command: +This folder adds a sidecar to the broker pods that exports selected JMX metrics over HTTP in Prometheus format. To add a container to an existing pod we must use the `patch`command: + +Using kubectl 1.14+ + +``` +kubectl --namespace kafka apply -k prometheus/ +``` + +Using pre-1.14 kubectl: ``` kubectl --namespace kafka apply -f prometheus/10-metrics-config.yml kubectl --namespace kafka patch statefulset kafka --patch "$(cat prometheus/50-kafka-jmx-exporter-patch.yml )" ``` + +## Consumer lag monitoring + +See [Burrow](../linkedin-burrow) +or [Kafka Minion](../consumers-prometheus/) + +## Prometheus Operator + +Use the [prometheus-operator](../variants/prometheus-operator/) kustomization. diff --git a/prometheus/kustomization.yaml b/prometheus/kustomization.yaml new file mode 100644 index 00000000..81c00fed --- /dev/null +++ b/prometheus/kustomization.yaml @@ -0,0 +1,9 @@ +bases: +# This kustomization needs to depend on one with kafka in it, to add the sidecar, +# but it needs to be the kafka from the chosen variant, as ../kafka here would override other kustomizations +#- ../kafka +#- ../variants/scale-1 +resources: +- 10-metrics-config.yml +patchesStrategicMerge: +- 50-kafka-jmx-exporter-patch.yml diff --git a/variants/prometheus-operator-example/additional-scrape-configs.yaml b/variants/prometheus-operator-example/additional-scrape-configs.yaml new file mode 100644 index 00000000..657a19b0 --- /dev/null +++ b/variants/prometheus-operator-example/additional-scrape-configs.yaml @@ -0,0 +1,12 @@ + +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: k8s + namespace: monitoring +spec: + additionalScrapeConfigs: + # github.com/kubernetes-sigs/kustomize/blob/master/examples/kvSourceGoPlugin.md is clearly WIP + name: additional-scrape-configs-5m4c7m6mc9 + # See https://github.com/prometheus/prometheus/pull/4131, and upon disagreement see https://github.com/prometheus/prometheus/issues/4484 + key: pods-discovery-by-prometheus-io-annotations.yaml diff --git a/variants/prometheus-operator-example/alertmanager-main-scale-1.yaml b/variants/prometheus-operator-example/alertmanager-main-scale-1.yaml new file mode 100644 index 00000000..38da8bdf --- /dev/null +++ b/variants/prometheus-operator-example/alertmanager-main-scale-1.yaml @@ -0,0 +1,7 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: main + namespace: monitoring +spec: + replicas: 1 diff --git a/variants/prometheus-operator-example/k8s-kafka-rbac.yaml b/variants/prometheus-operator-example/k8s-kafka-rbac.yaml new file mode 100644 index 00000000..e13c4e3c --- /dev/null +++ b/variants/prometheus-operator-example/k8s-kafka-rbac.yaml @@ -0,0 +1,32 @@ +# Allows the "k8s" prometheus from Prometheus Operator contrib to do service discovery iin the kafka namespace +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus-k8s + namespace: kafka +rules: +- apiGroups: + - "" + resources: + - services + - endpoints + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prometheus-k8s + namespace: kafka +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus-k8s +subjects: +- kind: ServiceAccount + name: prometheus-k8s + namespace: monitoring diff --git a/variants/prometheus-operator-example/k8s-kafka-servicemonitor.yaml b/variants/prometheus-operator-example/k8s-kafka-servicemonitor.yaml new file mode 100644 index 00000000..35b5b416 --- /dev/null +++ b/variants/prometheus-operator-example/k8s-kafka-servicemonitor.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: broker-monitoring + namespace: kafka + labels: + app: kafka +spec: + publishNotReadyAddresses: true + ports: + - name: fromjmx + port: 5556 + selector: + app: kafka +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kafka + namespace: monitoring + labels: + k8s-app: kafka +spec: + namespaceSelector: + matchNames: + - kafka + selector: + matchLabels: + app: kafka + endpoints: + # https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 120s + scrapeTimeout: 119s + port: fromjmx + scheme: http + path: /metrics diff --git a/variants/prometheus-operator-example/k8s-minion-servicemonitor.yaml b/variants/prometheus-operator-example/k8s-minion-servicemonitor.yaml new file mode 100644 index 00000000..79bbfa22 --- /dev/null +++ b/variants/prometheus-operator-example/k8s-minion-servicemonitor.yaml @@ -0,0 +1,22 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kafka-metrics-minion + namespace: monitoring + labels: + k8s-app: kafka-metrics-minion +spec: + namespaceSelector: + matchNames: + - kafka + selector: + matchLabels: + app: kafka-minion + type: openmetrics + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + interval: 30s + scrapeTimeout: 30s + port: http + scheme: http + path: /metrics diff --git a/variants/prometheus-operator-example/kustomization.yaml b/variants/prometheus-operator-example/kustomization.yaml new file mode 100644 index 00000000..a71b6fed --- /dev/null +++ b/variants/prometheus-operator-example/kustomization.yaml @@ -0,0 +1,31 @@ +bases: +# With prometheus-operator only you must add your own Prometheus and Alertmanager resources +#- github.com/coreos/prometheus-operator?ref=de9a6e1 +- github.com/coreos/kube-prometheus?ref=3a64636 +- ../../consumers-prometheus +# The ../../prometheus base must be edited to point to the chosen kafka base +# Actually to apply the sidecar with apply -k it has to be included with the kafka variant; can't be its own kustomization because you'll get +# either "failed to find an object with apps_v1_StatefulSet|kafka to apply the patch" or "id 'apps_v1_StatefulSet|kafka|~P|zoo|~S' already used" +#- ../../prometheus +resources: +- k8s-kafka-rbac.yaml +# with base ../../prometheus +#- k8s-kafka-servicemonitor.yaml +# with base ../../consumers-prometheus +- k8s-minion-servicemonitor.yaml +patchesStrategicMerge: +- prometheus-k8s-scale-1.yaml +- prometheus-k8s-2.9.2.yaml +- alertmanager-main-scale-1.yaml +- prometheus-k8s-nodeport.yaml +- additional-scrape-configs.yaml +secretGenerator: +- name: additional-scrape-configs + namespace: monitoring + #kvSources: + #- name: pods-discovery-by-prometheus-io-annotations.yaml + #pluginType: builtin + #args: + #- scrape-configs/pods-discovery-by-prometheus-io-annotations.yaml + files: + - scrape-configs/pods-discovery-by-prometheus-io-annotations.yaml diff --git a/variants/prometheus-operator-example/prometheus-k8s-2.9.2.yaml b/variants/prometheus-operator-example/prometheus-k8s-2.9.2.yaml new file mode 100644 index 00000000..ad928e9b --- /dev/null +++ b/variants/prometheus-operator-example/prometheus-k8s-2.9.2.yaml @@ -0,0 +1,8 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: k8s + namespace: monitoring +spec: + baseImage: quay.io/prometheus/prometheus + version: v2.7.2 diff --git a/variants/prometheus-operator-example/prometheus-k8s-nodeport.yaml b/variants/prometheus-operator-example/prometheus-k8s-nodeport.yaml new file mode 100644 index 00000000..3ef4ab1b --- /dev/null +++ b/variants/prometheus-operator-example/prometheus-k8s-nodeport.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus-k8s + namespace: monitoring +spec: + type: NodePort + ports: + - port: 9090 + nodePort: 32490 diff --git a/variants/prometheus-operator-example/prometheus-k8s-scale-1.yaml b/variants/prometheus-operator-example/prometheus-k8s-scale-1.yaml new file mode 100644 index 00000000..939ddb73 --- /dev/null +++ b/variants/prometheus-operator-example/prometheus-k8s-scale-1.yaml @@ -0,0 +1,7 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: k8s + namespace: monitoring +spec: + replicas: 1 diff --git a/variants/prometheus-operator-example/scrape-configs/pods-discovery-by-prometheus-io-annotations.yaml b/variants/prometheus-operator-example/scrape-configs/pods-discovery-by-prometheus-io-annotations.yaml new file mode 100644 index 00000000..4d812001 --- /dev/null +++ b/variants/prometheus-operator-example/scrape-configs/pods-discovery-by-prometheus-io-annotations.yaml @@ -0,0 +1,35 @@ +# Example scrape config for pods +# +# The relabeling allows the actual pod scrape endpoint to be configured via the +# following annotations: +# +# * `prometheus.io/scrape`: Only scrape pods that have a value of `true` +# * `prometheus.io/path`: If the metrics path is not `/metrics` override this. +# * `prometheus.io/port`: Scrape the pod on the indicated port instead of the +# pod's declared ports (default is a port-free target if none are declared). +- job_name: 'kubernetes-pods' + + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name