k8s-int-test-build: zk-less druid cluster and http based segment/task managment

.travis.yml

@@ -554,8 +554,11 @@ jobs:
       jdk: openjdk8
       services: &integration_test_services_k8s
         - docker
-      env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest'
-      before_script: integration-tests/script/build_run_k8s_cluster.sh
+      env: CONFIG_FILE='k8s_run_config_file.json' IT_TEST='-Dit.test=ITNestedQueryPushDownTest' POD_NAME=int-test POD_NAMESPACE=default
+      before_script:
+        - integration-tests/script/setup_k8s_cluster.sh
+        - integration-tests/script/setup_druid_operator_on_k8s.sh
+        - integration-tests/script/setup_druid_on_k8s.sh
       script: &run_integration_test_k8s
         - ${MVN} verify -pl integration-tests -P int-tests-config-file ${IT_TEST} ${MAVEN_SKIP}
       after_script: integration-tests/script/stop_k8s_cluster.sh

docs/development/extensions-core/kubernetes.md

@@ -39,9 +39,7 @@ This extension works together with HTTP based segment and task management in Dru
 `druid.indexer.runner.type=httpRemote`
 `druid.discovery.type=k8s`
 
-For Node Discovery, Each Druid process running inside a pod "announces" itself by adding few "labels" and "annotations" in the pod spec. So, to add those...
-- Druid process needs to be aware of pod name and namespace which it reads from environment variables `POD_NAME` and `POD_NAMESPACE`. These variable names can be changed, see configuration below. But in the end, each pod needs to have pod name and namespace added as environment variables.
-- Label/Annotation path in the pod spec must exist, which is easily satisfied if there is at least one label/annotation in the pod spec already. This limitation may be removed in future.
+For Node Discovery, Each Druid process running inside a pod "announces" itself by adding few "labels" and "annotations" in the pod spec. Druid process needs to be aware of pod name and namespace which it reads from environment variables `POD_NAME` and `POD_NAMESPACE`. These variable names can be changed, see configuration below. But in the end, each pod needs to have self pod name and namespace added as environment variables.
 
 Additionally, this extension has following configuration.
 
@@ -57,3 +55,34 @@ Additionally, this extension has following configuration.
 |`druid.discovery.k8s.renewDeadline`|`Duration`|Lease renewal period used by Leader.|PT17S|No|
 |`druid.discovery.k8s.retryPeriod`|`Duration`|Retry wait used by Leader Election algorithm on failed operations.|PT5S|No|
 
+### Gotchas
+
+- Label/Annotation path in each pod spec MUST EXIST, which is easily satisfied if there is at least one label/annotation in the pod spec already. This limitation may be removed in future.
+- Druid Pods need permissions to be able to add labels to self-pod, List and Watch other Pods, create ConfigMap for leader election. Assuming, "default" service account is used by Druid pods, you might need to add following or something similar Kubernetes Role and Role Binding.
+
+```
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: druid-cluster
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - configmaps
+  verbs:
+  - '*'
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: druid-cluster
+subjects:
+- kind: ServiceAccount
+  name: default
+roleRef:
+  kind: Role
+  name: druid-cluster
+  apiGroup: rbac.authorization.k8s.io
+```

integration-tests/k8s/role-and-binding.yaml

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: druid-cluster
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - configmaps
+  verbs:
+  - '*'
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: druid-cluster
+subjects:
+- kind: ServiceAccount
+  name: default
+roleRef:
+  kind: Role
+  name: druid-cluster
+  apiGroup: rbac.authorization.k8s.io

integration-tests/tiny-cluster.yaml → integration-tests/k8s/tiny-cluster.yaml

@@ -26,6 +26,12 @@ spec:
   podLabels:
     environment: stage
     release: alpha
+  podAnnotations:
+    dummy: k8s_extn_needs_atleast_one_annotation
+  readinessProbe:
+    httpGet:
+      path: /status/health
+      port: 8088
   securityContext:
     fsGroup: 0
     runAsUser: 0
@@ -60,10 +66,15 @@ spec:
     </Configuration>
   common.runtime.properties: |
 
-    # Zookeeper
-    druid.zk.service.host=tiny-cluster-zk-0.tiny-cluster-zk
-    druid.zk.paths.base=/druid
-    druid.zk.service.compress=false
+    #
+    # Zookeeper-less Druid Cluster
+    #
+    druid.zk.service.enabled=false
+    druid.discovery.type=k8s
+    druid.discovery.k8s.clusterIdentifier=druid-it
+    druid.serverview.type=http
+    druid.coordinator.loadqueuepeon.type=http
+    druid.indexer.runner.type=httpRemote
 
     # Metadata Store
     druid.metadata.storage.type=derby
@@ -79,13 +90,30 @@ spec:
     #
     # Extensions
     #
-    druid.extensions.loadList=["druid-avro-extensions","druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches"]
+    druid.extensions.loadList=["druid-avro-extensions","druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-kubernetes-extensions"]
 
     #
     # Service discovery
     #
     druid.selectors.indexing.serviceName=druid/overlord
     druid.selectors.coordinator.serviceName=druid/coordinator
+
+    druid.indexer.logs.type=file
+    druid.indexer.logs.directory=/druid/data/task-logs
+    druid.indexer.task.baseDir=/druid/data/task-base
+
+    druid.lookup.enableLookupSyncOnStartup=false
+
+  env:
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+
   nodes:
     brokers:
       # Optionally specify for running broker as Deployment
@@ -174,7 +202,6 @@ spec:
         druid.coordinator.asOverlord.enabled=true
         druid.coordinator.asOverlord.overlordService=druid/overlord
         druid.indexer.queue.startDelay=PT30S
-        druid.indexer.runner.type=local
       extra.jvm.options: |-
         -Xmx800m
         -Xms800m
@@ -237,16 +264,16 @@ spec:
 
     routers:
       nodeType: "router"
-      druid.port: 8888
+      druid.port: 8088
       services:
         - spec:
             type: NodePort
             ports:
               - name: router-service-port
                 nodePort: 30400
-                port: 8888
+                port: 8088
                 protocol: TCP
-                targetPort: 8888
+                targetPort: 8088
           selector:
             nodeSpecUniqueStr: druid-tiny-cluster-routers
           metadata:
@@ -258,7 +285,7 @@ spec:
       replicas: 1
       runtime.properties: |
         druid.service=druid/router
-        druid.plaintextPort=8888
+        druid.plaintextPort=8088
 
         # HTTP proxy
         druid.router.http.numConnections=50
@@ -317,4 +344,4 @@ spec:
         requests:
           memory: "3G"
         limits:
-          memory: "3G"
+          memory: "3G"

integration-tests/script/setup_druid_on_k8s.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+export KUBECTL="sudo /usr/local/bin/kubectl"
+
+# setup client keystore
+cd integration-tests
+./docker/tls/generate-client-certs-and-keystores.sh
+rm -rf docker/client_tls
+cp -r client_tls docker/client_tls
+cd ..
+
+# Build Docker images for pods
+mvn -B -ff -q dependency:go-offline \
+      install \
+      -Pdist,bundle-contrib-exts \
+      -Pskip-static-checks,skip-tests \
+      -Dmaven.javadoc.skip=true
+
+docker build -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced .
+
+# This tmp dir is used for MiddleManager pod and Historical Pod to cache segments.
+mkdir tmp
+chmod 777 tmp
+
+$KUBECTL apply -f integration-tests/k8s/role-and-binding.yaml
+sed -i "s|REPLACE_VOLUMES|`pwd`|g" integration-tests/k8s/tiny-cluster.yaml
+$KUBECTL apply -f integration-tests/k8s/tiny-cluster.yaml
+
+# Wait a bit
+sleep 120
+
+## Debug And FastFail
+
+$KUBECTL get pod
+$KUBECTL get svc
+

integration-tests/script/setup_druid_operator_on_k8s.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+export DRUID_OPERATOR_VERSION=0.0.3
+export KUBECTL="sudo /usr/local/bin/kubectl"
+
+
+# Prepare For Druid-Operator
+git clone https://github.com/druid-io/druid-operator.git
+cd druid-operator
+git checkout -b druid-operator-$DRUID_OPERATOR_VERSION druid-operator-$DRUID_OPERATOR_VERSION
+cd ..
+sed -i "s|REPLACE_IMAGE|druidio/druid-operator:$DRUID_OPERATOR_VERSION|g" druid-operator/deploy/operator.yaml
+
+# Deploy Druid Operator and Druid CR spec
+$KUBECTL create -f druid-operator/deploy/service_account.yaml
+$KUBECTL create -f druid-operator/deploy/role.yaml
+$KUBECTL create -f druid-operator/deploy/role_binding.yaml
+$KUBECTL create -f druid-operator/deploy/crds/druid.apache.org_druids_crd.yaml
+$KUBECTL create -f druid-operator/deploy/operator.yaml
+
+echo "Setup Druid Operator on K8S Done!"