From 3d6749b20836622b3497c9eae1e8c477a0fa3c2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Miderb=C3=A4ck?= Date: Tue, 19 Dec 2017 15:09:46 +0100 Subject: [PATCH] Filtering out non-running containers/tasks/pods (#209) --- src/orchestration/KubernetesClient.js | 16 +- src/orchestration/LocalDockerClient.js | 10 +- src/orchestration/SwarmDockerClient.js | 17 ++- .../test-data/KubernetesClient.spec.data.json | 123 +++++++++++++++ .../LocalDockerClient.spec.data.json | 50 ++++++ .../SwarmDockerClient.spec.data.json | 143 ++++++++++++++++++ 6 files changed, 348 insertions(+), 11 deletions(-) diff --git a/src/orchestration/KubernetesClient.js b/src/orchestration/KubernetesClient.js index 73407728..a397368d 100644 --- a/src/orchestration/KubernetesClient.js +++ b/src/orchestration/KubernetesClient.js @@ -43,12 +43,18 @@ class KubernetesClient { */ static async listEngines() { const pods = await kubeHttpGet(`/api/v1/pods?labelSelector=${Config.discoveryLabel}`); - - const engineInfoEntries = pods.items.map((pod) => { + const runningPods = pods.items.filter( + (pod) => { + if (pod.status.phase.toLowerCase() === 'running') { + logger.debug(`Valid engine pod info received: ${JSON.stringify(pod)}`); + return true; + } + logger.info(`Discarding non-running engine pod: ${JSON.stringify(pod)}`); + return false; + }); + const engineInfoEntries = runningPods.map((pod) => { const labels = pod.metadata.labels; - const engine = { - ip: pod.status.podIP, - }; + const engine = { ip: pod.status.podIP }; const key = pod.metadata.uid; return { key, engine, kubernetes: pod, labels }; }); diff --git a/src/orchestration/LocalDockerClient.js b/src/orchestration/LocalDockerClient.js index a2f36ccd..e157ad58 100644 --- a/src/orchestration/LocalDockerClient.js +++ b/src/orchestration/LocalDockerClient.js @@ -41,7 +41,15 @@ class LocalDockerClient { return new Promise((resolve, reject) => { LocalDockerClient.docker.listContainers({ filters: { label: [Config.discoveryLabel] } }, (err, containers) => { if (!err) { - const engineInfoEntries = containers.map((local) => { + const runningContainers = containers.filter((container) => { + if (container.State.toLowerCase() === 'running') { + logger.debug(`Valid engine container info received: ${JSON.stringify(container)}`); + return true; + } + logger.info(`Discarding non-running engine container: ${JSON.stringify(container)}`); + return false; + }); + const engineInfoEntries = runningContainers.map((local) => { const labels = local.Labels; const engine = { ip: getIpAddress(local), diff --git a/src/orchestration/SwarmDockerClient.js b/src/orchestration/SwarmDockerClient.js index dfa01af6..501745dd 100644 --- a/src/orchestration/SwarmDockerClient.js +++ b/src/orchestration/SwarmDockerClient.js @@ -30,13 +30,20 @@ function getIpAddress(task) { function getTasks(docker, discoveryLabel) { return new Promise((resolve, reject) => { - docker.listTasks({ filters: '{ "desired-state": ["running"] }' }, (err, tasks) => { + docker.listTasks({}, (err, tasks) => { if (!err) { // We do filtering on the discovery label here, but this should be possible to do by - // specifying a filter on labels above. - const filteredTasks = tasks.filter( - task => discoveryLabel in task.Spec.ContainerSpec.Labels); - resolve(filteredTasks); + // specifying the filter in the listTasks() call above. + const labeledTasks = tasks.filter(task => discoveryLabel in task.Spec.ContainerSpec.Labels); + const runningTasks = labeledTasks.filter((task) => { + if (task.Status.State.toLowerCase() === 'running') { + logger.debug(`Valid engine container task received: ${JSON.stringify(task)}`); + return true; + } + logger.info(`Discarding non-running engine task: ${JSON.stringify(task)}`); + return false; + }); + resolve(runningTasks); } else { logger.error('Error when listing Docker Swarm tasks', err); reject(err); diff --git a/test/test-data/KubernetesClient.spec.data.json b/test/test-data/KubernetesClient.spec.data.json index 99ceaf0f..a3a8a2f8 100644 --- a/test/test-data/KubernetesClient.spec.data.json +++ b/test/test-data/KubernetesClient.spec.data.json @@ -256,6 +256,129 @@ ], "qosClass": "BestEffort" } + }, + { + "metadata": { + "name": "engine-deployment-4234688684-pm222", + "generateName": "engine-deployment-4234688684-", + "namespace": "default", + "selfLink": "/api/v1/namespaces/default/pods/engine-deployment-4234688684-pm222", + "uid": "89fff5bc-872b-11e7-b00a-00155d014987", + "resourceVersion": "113232", + "creationTimestamp": "2017-08-22T11:17:51Z", + "labels": { + "qix-engine": "prod", + "pod-template-hash": "4163433257", + "service": "engine", + "qix-engine-api-port": "9076", + "qix-engine-metrics-port": "9090" + }, + "annotations": { + "kubernetes.io/created-by": "{\"kind\":\"SerializedReference\",\"apiVersion\":\"v1\",\"reference\":{\"kind\":\"ReplicaSet\",\"namespace\":\"default\",\"name\":\"engine-deployment-4163433257\",\"uid\":\"88cb5d22-872b-11e7-b00a-00155d014987\",\"apiVersion\":\"extensions\",\"resourceVersion\":\"113210\"}}\n" + }, + "ownerReferences": [ + { + "apiVersion": "extensions/v1beta1", + "kind": "ReplicaSet", + "name": "engine-deployment-4234688684", + "uid": "88fffd22-872b-11e7-b00a-00155d014987", + "controller": true, + "blockOwnerDeletion": true + } + ] + }, + "spec": { + "volumes": [ + { + "name": "default-token-4l9dx", + "secret": { + "secretName": "default-token-4l9dx", + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "engine", + "image": "qlikea/engine:12.42.0", + "ports": [ + { + "containerPort": 9076, + "protocol": "TCP" + }, + { + "containerPort": 9098, + "protocol": "TCP" + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "default-token-4l9dx", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "IfNotPresent" + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "serviceAccountName": "default", + "serviceAccount": "default", + "nodeName": "minikube", + "securityContext": {}, + "imagePullSecrets": [ + { + "name": "dockerhub" + } + ], + "schedulerName": "default-scheduler" + }, + "status": { + "phase": "Paused", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2017-08-22T11:17:51Z" + }, + { + "type": "Ready", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2017-08-22T11:17:53Z" + }, + { + "type": "PodScheduled", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2017-08-22T11:17:51Z" + } + ], + "startTime": "2017-08-22T11:17:51Z", + "containerStatuses": [ + { + "name": "engine", + "state": { + "running": { + "startedAt": "2017-08-22T11:17:52Z" + } + }, + "lastState": {}, + "ready": true, + "restartCount": 0, + "image": "qlikea/engine:12.42.0", + "imageID": "docker-pullable://qlikea/engine@sha256:331d299079cb758403099dd0cfbd34a2970f3a2d15504d4994fd8425558116ce", + "containerID": "docker://063f46fbc0a77a4cf2e82a2856ceee222a1a0738dfbf33701a49af0a06e2d203" + } + ], + "qosClass": "BestEffort" + } } ] }, diff --git a/test/test-data/LocalDockerClient.spec.data.json b/test/test-data/LocalDockerClient.spec.data.json index b90e6d55..12d10432 100644 --- a/test/test-data/LocalDockerClient.spec.data.json +++ b/test/test-data/LocalDockerClient.spec.data.json @@ -99,6 +99,56 @@ } }, "Mounts": [] + }, + { + "Id": "c506c9b17ee150673dd5818c4ae21f09618f3581c0f19e1c1a24450c6c2eb421", + "Names": [ + "/mira_engine3_1" + ], + "Image": "qlikea/engine:12.38.0", + "ImageID": "sha256:8f00a39a3f0151ae7d80744eafd6ee71474df010349d44c9cf455a76baf59b1f", + "Command": "./entrypoint.sh", + "Created": 1502866132, + "Ports": [ + { + "IP": "0.0.0.0", + "PrivatePort": 9076, + "PublicPort": 9376, + "Type": "tcp" + } + ], + "Labels": { + "foo": "foo2", + "bar": "bar2", + "qix-engine": "prod", + "qix-engine-api-port": "9276", + "qix-engine-metrics-port": "9090" + }, + "State": "paused", + "Status": "Paused About a minute", + "HostConfig": { + "NetworkMode": "mira_default" + }, + "NetworkSettings": { + "Networks": { + "mira_default": { + "IPAMConfig": null, + "Links": null, + "Aliases": null, + "NetworkID": "0ff1add8bc3c672e38b3cfbd6d3d10e824297550f67428508151bde8b19d7916", + "EndpointID": "c64e002b80cd25fb8e402a082157991f569624029f02da957550f14b68560479", + "Gateway": "172.19.0.1", + "IPAddress": "172.19.0.3", + "IPPrefixLen": 16, + "IPv6Gateway": "", + "GlobalIPv6Address": "", + "GlobalIPv6PrefixLen": 0, + "MacAddress": "02:42:ac:13:00:02", + "DriverOpts": null + } + } + }, + "Mounts": [] } ], "miraOutput": [ diff --git a/test/test-data/SwarmDockerClient.spec.data.json b/test/test-data/SwarmDockerClient.spec.data.json index 76473235..f9d34b75 100644 --- a/test/test-data/SwarmDockerClient.spec.data.json +++ b/test/test-data/SwarmDockerClient.spec.data.json @@ -285,6 +285,149 @@ ] } ] + }, + { + "ID": "eqlp1haskhf876h2a9i91hd7im", + "Version": { + "Index": 780 + }, + "CreatedAt": "2017-08-24T09:13:38.429799819Z", + "UpdatedAt": "2017-08-24T09:13:52.511982557Z", + "Labels": {}, + "Spec": { + "ContainerSpec": { + "Image": "qlikea/engine:12.42.0", + "Labels": { + "com.docker.stack.namespace": "mira-stack", + "qix-engine": "prod", + "foo": "foo2", + "bar": "bar2", + "qix-engine-api-port": "9076", + "qix-engine-metrics-port": "9090" + }, + "Privileges": { + "CredentialSpec": null, + "SELinuxContext": null + } + }, + "Resources": {}, + "Placement": { + "Constraints": [ + "node.role == manager" + ] + }, + "Networks": [ + { + "Target": "vtj9ni6c1mkv7v4ak4j3iqh9g", + "Aliases": [ + "engine1" + ] + } + ], + "ForceUpdate": 0 + }, + "ServiceID": "t1s73kzykq9oenl414moh4htr", + "Slot": 1, + "NodeID": "4j4kqo4xx8ihfrkmuebvps11m", + "Status": { + "Timestamp": "2017-08-24T09:13:52.463896476Z", + "State": "paused", + "Message": "paused", + "ContainerStatus": { + "ContainerID": "8262d7dfcda867547affaeccac013d401b72448e4a621d9e300736d9e5b1643f", + "PID": 41650 + }, + "PortStatus": {} + }, + "DesiredState": "running", + "NetworksAttachments": [ + { + "Network": { + "ID": "e67gn9n85r0q4tvjwth7d3tay", + "Version": { + "Index": 6 + }, + "CreatedAt": "2017-08-23T13:48:18.402425147Z", + "UpdatedAt": "2017-08-23T13:48:18.418368808Z", + "Spec": { + "Name": "ingress", + "Labels": {}, + "DriverConfiguration": {}, + "Ingress": true, + "IPAMOptions": { + "Driver": {}, + "Configs": [ + { + "Subnet": "10.255.0.0/16", + "Gateway": "10.255.0.1" + } + ] + }, + "Scope": "swarm" + }, + "DriverState": { + "Name": "overlay", + "Options": { + "com.docker.network.driver.overlay.vxlanid_list": "4096" + } + }, + "IPAMOptions": { + "Driver": { + "Name": "default" + }, + "Configs": [ + { + "Subnet": "10.255.0.0/16", + "Gateway": "10.255.0.1" + } + ] + } + }, + "Addresses": [ + "10.255.0.6/16" + ] + }, + { + "Network": { + "ID": "vtj9ni6c1mkv7v4ak4j3iqh9g", + "Version": { + "Index": 569 + }, + "CreatedAt": "2017-08-23T14:06:39.512982339Z", + "UpdatedAt": "2017-08-23T14:06:39.519284155Z", + "Spec": { + "Name": "mira-stack_default", + "Labels": { + "com.docker.stack.namespace": "mira-stack" + }, + "DriverConfiguration": { + "Name": "overlay" + }, + "Scope": "swarm" + }, + "DriverState": { + "Name": "overlay", + "Options": { + "com.docker.network.driver.overlay.vxlanid_list": "4098" + } + }, + "IPAMOptions": { + "Driver": { + "Name": "default" + }, + "Configs": [ + { + "Subnet": "10.0.1.0/24", + "Gateway": "10.0.1.1" + } + ] + } + }, + "Addresses": [ + "10.0.1.16/24" + ] + } + ] } ], "miraOutput": [