Add test tools for kafka authentication

* Add test stacks for SSL, SASL/PLAIN, SASL/SCRAM, SASL over TLS authentication

clients/cmd/promtail/promtail-kafka-sasl-plain.yaml

@@ -0,0 +1,26 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: kafka-sasl-plain
+    kafka:
+      use_incoming_timestamp: false
+      brokers:
+        - localhost:29092
+      authentication:
+        type: sasl
+        sasl_config:
+          mechanism: PLAIN
+          user: kafkaadmin
+          password: kafkaadmin-pass
+          use_tls: false
+      group_id: kafka_group
+      topics:
+        - foo
+        - ^promtail.*
+      labels:
+        job: kafka-sasl-plain

clients/cmd/promtail/promtail-kafka-sasl-scram.yaml

@@ -0,0 +1,26 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: kafka-sasl-plain
+    kafka:
+      use_incoming_timestamp: false
+      brokers:
+        - localhost:29092
+      authentication:
+        type: sasl
+        sasl_config:
+          mechanism: SCRAM-SHA-512
+          user: kafkaadmin
+          password: kafkaadmin-pass
+          use_tls: false
+      group_id: kafka_group
+      topics:
+        - foo
+        - ^promtail.*
+      labels:
+        job: kafka-sasl-plain

clients/cmd/promtail/promtail-kafka-sasl-ssl.yaml

@@ -0,0 +1,28 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: kafka-sasl-plain
+    kafka:
+      use_incoming_timestamp: false
+      brokers:
+        - localhost:29092
+      authentication:
+        type: sasl
+        sasl_config:
+          mechanism: PLAIN
+          user: kafkaadmin
+          password: kafkaadmin-pass
+          use_tls: true
+          ca_file: ../../../tools/kafka/secrets/promtail-kafka-ca.pem
+          insecure_skip_verify: true
+      group_id: kafka_group
+      topics:
+        - foo
+        - ^promtail.*
+      labels:
+        job: kafka-sasl-plain

clients/cmd/promtail/promtail-kafka-ssl.yaml

@@ -0,0 +1,27 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+clients:
+  - url: http://localhost:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: kafka-mtls
+    kafka:
+      use_incoming_timestamp: false
+      brokers:
+        - localhost:29092
+      authentication:
+        type: ssl
+        tls_config:
+          ca_file: ../../../tools/kafka/secrets/promtail-kafka-ca.pem
+          cert_file: ../../../tools/kafka/secrets/kafka.consumer.keystore.cer.pem
+          key_file: ../../../tools/kafka/secrets/kafka.consumer.keystore.key.pem
+          server_name: localhost
+          insecure_skip_verify: true
+      group_id: kafka_mtls_group
+      topics:
+        - foo
+        - ^promtail.*
+      labels:
+        job: kafka-mtls

tools/kafka/.gitignore

@@ -0,0 +1,8 @@
+*.crt
+*.jks
+*_creds
+*.key
+*.pem
+*.csr
+*.srl
+*.p12

tools/kafka/README.md

@@ -15,6 +15,16 @@ To discover available brokers you can use the `make print-brokers`.
 
 Finally to stop the compose stack use `make stop-kafka`. This will result in all topics being lost with their messages.
 
+## Running secure kafka locally
+
+To test authentication, you need to start the Kafka container which is configured with authentication.
+
+You can also use `make start-kafka` in appropriate directory like `sasl-scram` you need.
+
+In addition, you need to create certificates using `make create-certs` when using SSL/TLS.
+
+If you don't need to authenticate, you should use the tools in `plain` directory.
+
 ## Working with Topic
 
 In Kafka before sending messages you need to create and select the topic you want to use for the exchange.

tools/kafka/Makefile → tools/kafka/plain/Makefile

@@ -4,7 +4,7 @@ TOPIC ?= promtail
 RF ?= 1
 PARTS ?= 3
 
-BROKER_LIST := $(shell ./broker-list.sh $(HOST_IP))
+BROKER_LIST := $(shell ../broker-list.sh $(HOST_IP))
 DOCKER_RUN := docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e HOST_IP=$(HOST_IP) -i -t wurstmeister/kafka /bin/bash -c
 
 start-kafka:

tools/kafka/sasl-plain/Makefile

@@ -0,0 +1,16 @@
+
+HOST_IP ?= host.docker.internal
+TOPIC ?= promtail
+RF ?= 1
+PARTS ?= 3
+
+BROKER_LIST := $(shell ../broker-list.sh $(HOST_IP))
+
+start-kafka:
+	docker-compose up -d
+
+stop-kafka:
+	docker-compose down
+
+print-brokers:
+	@echo $(BROKER_LIST)

tools/kafka/sasl-plain/conf/kafka.jaas.conf

@@ -0,0 +1,19 @@
+KafkaServer {
+    org.apache.kafka.common.security.plain.PlainLoginModule required
+    username="kafkaadmin"
+    password="kafkaadmin-pass"
+    user_kafkaadmin="kafkaadmin-pass"
+   ;
+};
+KafkaClient {
+    org.apache.kafka.common.security.plain.PlainLoginModule required
+    username="kafkaadmin"
+    password="kafkaadmin-pass"
+    ;
+};
+Client {
+    org.apache.zookeeper.server.auth.DigestLoginModule required
+    username="super"
+    password="adminsecret"
+    ;
+};

tools/kafka/sasl-plain/conf/zookeeper.jaas.conf

@@ -0,0 +1,4 @@
+Server {
+       org.apache.zookeeper.server.auth.DigestLoginModule required
+       user_super="adminsecret";
+};

tools/kafka/sasl-plain/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '2'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    ports:
+      - "22181:22181"
+    environment:
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_CLIENT_PORT: 22181
+      KAFKA_OPTS: -Djava.security.auth.login.config=/etc/kafka/secrets/zookeeper.jaas.conf
+        -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
+        -Dzookeeper.requireClientAuthScheme=sasl
+    volumes:
+      - ./conf:/etc/kafka/secrets
+
+  kafka:
+    image: confluentinc/cp-kafka:6.2.1
+    depends_on:
+      - zookeeper
+    ports:
+      - "29092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:22181
+      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka:9092
+      KAFKA_SECURITY_INTER_BROKER_PROTOCOL: SASL_PLAINTEXT
+      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
+      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
+      KAFKA_OPTS: -Djava.security.auth.login.config=/etc/kafka/secrets/kafka.jaas.conf
+    volumes:
+      - ./conf:/etc/kafka/secrets
+      - /var/run/docker.sock:/var/run/docker.sock

tools/kafka/sasl-scram/Makefile

@@ -0,0 +1,22 @@
+
+HOST_IP ?= host.docker.internal
+TOPIC ?= promtail
+RF ?= 1
+PARTS ?= 3
+
+BROKER_LIST := $(shell ../broker-list.sh $(HOST_IP))
+
+start-kafka:
+	docker-compose up -d zookeeper
+	docker-compose exec zookeeper kafka-configs \
+		--zookeeper localhost:22181 \
+		--alter \
+		--add-config 'SCRAM-SHA-512=[iterations=8192,password=kafkaadmin-pass],SCRAM-SHA-512=[password=kafkaadmin-pass]' \
+		--entity-type users --entity-name kafkaadmin
+	docker-compose up -d kafka
+
+stop-kafka:
+	docker-compose down
+
+print-brokers:
+	@echo $(BROKER_LIST)

tools/kafka/sasl-scram/conf/kafka.jaas.conf

@@ -0,0 +1,18 @@
+KafkaServer {
+    org.apache.kafka.common.security.scram.ScramLoginModule required
+    username="kafkaadmin"
+    password="kafkaadmin-pass"
+    ;
+};
+KafkaClient {
+    org.apache.kafka.common.security.scram.ScramLoginModule required
+    username="kafkaadmin"
+    password="kafkaadmin-pass"
+    ;
+};
+Client {
+    org.apache.zookeeper.server.auth.DigestLoginModule required
+    username="super"
+    password="adminsecret"
+    ;
+};

tools/kafka/sasl-scram/conf/zookeeper.jaas.conf

@@ -0,0 +1,10 @@
+Server {
+       org.apache.zookeeper.server.auth.DigestLoginModule required
+       user_super="adminsecret";
+};
+Client {
+    org.apache.zookeeper.server.auth.DigestLoginModule required
+    username="super"
+    password="adminsecret"
+    ;
+};

tools/kafka/sasl-scram/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '2'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    ports:
+      - "22181:22181"
+    environment:
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_CLIENT_PORT: 22181
+      KAFKA_OPTS: -Djava.security.auth.login.config=/etc/kafka/secrets/zookeeper.jaas.conf
+        -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
+        -Dzookeeper.requireClientAuthScheme=sasl
+    volumes:
+      - ./conf:/etc/kafka/secrets
+
+  kafka:
+    image: confluentinc/cp-kafka:6.2.1
+    depends_on:
+      - zookeeper
+    ports:
+      - "29092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:22181
+      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka:9092
+      KAFKA_SECURITY_INTER_BROKER_PROTOCOL: SASL_PLAINTEXT
+      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: SCRAM-SHA-512
+      KAFKA_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
+      KAFKA_OPTS: -Djava.security.auth.login.config=/etc/kafka/secrets/kafka.jaas.conf
+    volumes:
+      - ./conf:/etc/kafka/secrets
+      - /var/run/docker.sock:/var/run/docker.sock

tools/kafka/sasl-ssl/Makefile

@@ -0,0 +1,19 @@
+
+HOST_IP ?= host.docker.internal
+TOPIC ?= promtail
+RF ?= 1
+PARTS ?= 3
+
+BROKER_LIST := $(shell ../broker-list.sh $(HOST_IP))
+
+create-certs:
+	bash ../secrets/create-certs.sh
+
+start-kafka:
+	docker-compose up -d
+
+stop-kafka:
+	docker-compose down
+
+print-brokers:
+	@echo $(BROKER_LIST)

tools/kafka/sasl-ssl/conf/kafka.jaas.conf

@@ -0,0 +1,19 @@
+KafkaServer {
+    org.apache.kafka.common.security.plain.PlainLoginModule required
+    username="kafkaadmin"
+    password="kafkaadmin-pass"
+    user_kafkaadmin="kafkaadmin-pass"
+   ;
+};
+KafkaClient {
+    org.apache.kafka.common.security.plain.PlainLoginModule required
+    username="kafkaadmin"
+    password="kafkaadmin-pass"
+    ;
+};
+Client {
+    org.apache.zookeeper.server.auth.DigestLoginModule required
+    username="super"
+    password="adminsecret"
+    ;
+};

tools/kafka/sasl-ssl/conf/zookeeper.jaas.conf

@@ -0,0 +1,4 @@
+Server {
+       org.apache.zookeeper.server.auth.DigestLoginModule required
+       user_super="adminsecret";
+};