From c477e29c3f517bf7d6b8344a289cc2c09d4852d8 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Thu, 3 Feb 2022 06:50:41 -0700 Subject: [PATCH] Add tooling for local k3d clusters under tools/dev (#5272) * add k3d folder with loki-distributed env * add readme * shellcheck create_cluster.sh * lint jsonnet * Update grafana image to latest --- tools/dev/k3d/Makefile | 27 ++++ tools/dev/k3d/README.md | 31 +++++ tools/dev/k3d/chartfile.yaml | 22 +++ tools/dev/k3d/charts/.gitignore | 2 + .../loki-distributed/main.jsonnet | 131 ++++++++++++++++++ .../environments/loki-distributed/spec.json | 14 ++ .../values/default/config.libsonnet | 67 +++++++++ .../values/default/values.libsonnet | 56 ++++++++ tools/dev/k3d/jsonnetfile.json | 60 ++++++++ tools/dev/k3d/jsonnetfile.lock.json | 126 +++++++++++++++++ tools/dev/k3d/lib/grafana/grafana.libsonnet | 66 +++++++++ tools/dev/k3d/lib/jaeger/jaeger.libsonnet | 67 +++++++++ tools/dev/k3d/lib/k.libsonnet | 1 + .../k3d/lib/prometheus/prometheus.libsonnet | 40 ++++++ tools/dev/k3d/lib/promtail/promtail.libsonnet | 24 ++++ tools/dev/k3d/scripts/create_cluster.sh | 34 +++++ tools/dev/k3d/vendor/.gitignore | 2 + 17 files changed, 770 insertions(+) create mode 100644 tools/dev/k3d/Makefile create mode 100644 tools/dev/k3d/README.md create mode 100644 tools/dev/k3d/chartfile.yaml create mode 100644 tools/dev/k3d/charts/.gitignore create mode 100644 tools/dev/k3d/environments/loki-distributed/main.jsonnet create mode 100644 tools/dev/k3d/environments/loki-distributed/spec.json create mode 100644 tools/dev/k3d/environments/loki-distributed/values/default/config.libsonnet create mode 100644 tools/dev/k3d/environments/loki-distributed/values/default/values.libsonnet create mode 100644 tools/dev/k3d/jsonnetfile.json create mode 100644 tools/dev/k3d/jsonnetfile.lock.json create mode 100644 tools/dev/k3d/lib/grafana/grafana.libsonnet create mode 100644 tools/dev/k3d/lib/jaeger/jaeger.libsonnet create mode 100644 tools/dev/k3d/lib/k.libsonnet create mode 100644 tools/dev/k3d/lib/prometheus/prometheus.libsonnet create mode 100644 tools/dev/k3d/lib/promtail/promtail.libsonnet create mode 100755 tools/dev/k3d/scripts/create_cluster.sh create mode 100644 tools/dev/k3d/vendor/.gitignore diff --git a/tools/dev/k3d/Makefile b/tools/dev/k3d/Makefile new file mode 100644 index 000000000000..7da5303b5183 --- /dev/null +++ b/tools/dev/k3d/Makefile @@ -0,0 +1,27 @@ +.PHONY: loki-distributed down add-repos update-repos prepare build-latest-image + +IMAGE_TAG := $(shell ../../../tools/image-tag) +REGISTRY_PORT ?= 45629 + +loki-distributed: prepare + $(CURDIR)/scripts/create_cluster.sh "loki-distributed" + +down: + k3d cluster delete loki-distributed + +add-repos: + helm repo add --force-update prometheus-community https://prometheus-community.github.io/helm-charts + helm repo add --force-update grafana https://grafana.github.io/helm-charts + helm repo add --force-update minio https://helm.min.io + +update-repos: add-repos + helm repo update + tk tool charts vendor + jb update + +prepare: update-repos build-latest-image + +build-latest-image: + make -C $(CURDIR)/../../.. loki-image + docker tag grafana/loki:$(IMAGE_TAG) k3d-grafana:$(REGISTRY_PORT)/loki:latest + docker push k3d-grafana:$(REGISTRY_PORT)/loki:latest diff --git a/tools/dev/k3d/README.md b/tools/dev/k3d/README.md new file mode 100644 index 000000000000..04ddcbbf13ed --- /dev/null +++ b/tools/dev/k3d/README.md @@ -0,0 +1,31 @@ +# Deploy Loki to k3d for Local Development + +## Pre-requisites + +In order to use the make targets in this directory, make sure you have the following tools installed: +* [kubectl](https://kubernetes.io/docs/tasks/tools/) +* [k3d](https://k3d.io/v4.4.8/) +* [tanka](https://github.com/grafana/tanka) +* [jsonnet](https://jsonnet.org/) +* [jq](https://stedolan.github.io/jq/) + +**Note**: in case docker is unable to resolve the local k3d registry hostname, add the following entry to the `/etc/hosts` file: +``` +127.0.0.1 k3d-grafana +``` + +## Spinning Up An Environment + +Each environment has it's own make target. To bring up `loki-distributed`, for example, run: + +```bash +make loki-distributed +``` + +## Tearing Down An Environment + +The `down` make target will tear down all environments. + +```bash +make down +``` diff --git a/tools/dev/k3d/chartfile.yaml b/tools/dev/k3d/chartfile.yaml new file mode 100644 index 000000000000..2dc4421c49ba --- /dev/null +++ b/tools/dev/k3d/chartfile.yaml @@ -0,0 +1,22 @@ +directory: charts +repositories: +- name: grafana + url: https://grafana.github.io/grafana/helm-charts +- name: prometheus-community + url: https://prometheus-community.github.io/helm-charts +- name: minio + url: https://helm.min.io +requires: +- chart: grafana/promtail + version: 3.8.1 +- chart: grafana/grafana + version: 6.20.1 +- chart: prometheus-community/prometheus + version: 15.0.1 +- chart: minio/minio + version: 8.0.10 +- chart: grafana/loki-distributed + version: 0.42.0 +- chart: grafana/loki-simple-scalable + version: 0.2.0 +version: 1 diff --git a/tools/dev/k3d/charts/.gitignore b/tools/dev/k3d/charts/.gitignore new file mode 100644 index 000000000000..d6b7ef32c847 --- /dev/null +++ b/tools/dev/k3d/charts/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tools/dev/k3d/environments/loki-distributed/main.jsonnet b/tools/dev/k3d/environments/loki-distributed/main.jsonnet new file mode 100644 index 000000000000..d436823396f7 --- /dev/null +++ b/tools/dev/k3d/environments/loki-distributed/main.jsonnet @@ -0,0 +1,131 @@ +local k = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet'; +local tanka = import 'github.com/grafana/jsonnet-libs/tanka-util/main.libsonnet'; +local spec = (import './spec.json').spec; + +local jaeger = import 'jaeger/jaeger.libsonnet'; +local grafana = import 'grafana/grafana.libsonnet'; +local prometheus = import 'prometheus/prometheus.libsonnet'; +local promtail = import 'promtail/promtail.libsonnet'; + +local helm = tanka.helm.new(std.thisFile) { + template(name, chart, conf={}):: + std.native('helmTemplate')(name, chart, conf { calledFrom: std.thisFile }), +}; +local clusterName = 'loki-distributed'; +local normalizedClusterName = std.strReplace(clusterName, '-', '_'); + +grafana + prometheus + promtail + jaeger { + local gatewayName = self.loki.service_loki_distributed_gateway.metadata.name, + local gatewayHost = '%s' % gatewayName, + local gatewayUrl = 'http://%s' % gatewayHost, + local jaegerQueryName = self.jaeger.query_service.metadata.name, + local jaegerQueryUrl = 'http://%s' % jaegerQueryName, + local jaegerAgentName = self.jaeger.agent_service.metadata.name, + local jaegerAgentUrl = 'http://%s' % jaegerAgentName, + local prometheusServerName = self.prometheus.service_prometheus_server.metadata.name, + local prometheusUrl = 'http://%s' % prometheusServerName, + local namespace = spec.namespace, + + _config+:: { + clusterName: clusterName, + gatewayName: gatewayName, + gatewayHost: gatewayHost, + gelUrl: gatewayUrl, + jaegerAgentName: jaegerAgentName, + jaegerAgentPort: 6831, + namespace: namespace, + adminToken: 'gel-admin-token', + + grafana: { + datasources: [ + { + name: 'Prometheus', + type: 'prometheus', + access: 'proxy', + url: prometheusUrl, + }, + { + name: 'Jaeger', + type: 'jaeger', + access: 'proxy', + url: jaegerQueryUrl, + uid: 'jaeger_uid', + }, + { + name: 'Loki', + type: 'loki', + access: 'proxy', + url: gatewayUrl, + jsonData: { + derivedFields: [ + { + datasourceUid: 'jaeger_uid', + matcherRegex: 'traceID=(\\w+)', + name: 'TraceID', + url: '$${__value.raw}', + }, + ], + }, + }, + ], + }, + }, + + minio: helm.template('minio', '../../charts/minio', { + namespace: $._config.namespace, + values: { + accessKey: 'loki', + rootUser: 'loki', + secretKey: 'supersecret', + rootPassword: 'supersecret', + buckets: [ + { + name: 'loki-data', + policy: 'public', + purge: false, + }, + ], + persistence: { + enabled: true, + 'storage-class': 'local-path', + size: '10Gi', + }, + }, + }), + + local config = import './values/default/config.libsonnet', + local values = (import './values/default/values.libsonnet').lokiValues(k.util.manifestYaml(config)), + + loki: helm.template($._config.clusterName, '../../charts/loki-distributed', { + namespace: $._config.namespace, + values: values { + local registry = 'k3d-grafana:45629', + loki+: { + image: { + registry: registry, + repository: 'loki', + tag: 'latest', + pullPolicy: 'Always', + }, + }, + }, + }) + { + ['deployment_loki_distributed_%s' % [name]]+: + k.apps.v1.deployment.mapContainers($._addJaegerEnvVars) + + k.apps.v1.deployment.spec.template.metadata.withAnnotations($._prometheusAnnotations) + for name in [ + 'compactor', + 'distributor', + 'gateway', + 'query_frontend', + ] + } + { + ['stateful_set_loki_distributed_%s' % [name]]+: + k.apps.v1.statefulSet.mapContainers($._addJaegerEnvVars) + + k.apps.v1.statefulSet.spec.template.metadata.withAnnotations($._prometheusAnnotations) + for name in [ + 'ingester', + 'querier', + ] + }, +} diff --git a/tools/dev/k3d/environments/loki-distributed/spec.json b/tools/dev/k3d/environments/loki-distributed/spec.json new file mode 100644 index 000000000000..f1bc10758cff --- /dev/null +++ b/tools/dev/k3d/environments/loki-distributed/spec.json @@ -0,0 +1,14 @@ +{ + "apiVersion": "tanka.dev/v1alpha1", + "kind": "Environment", + "metadata": { + "name": "environments/loki-distributed", + "namespace": "environments/loki-distributed/main.jsonnet" + }, + "spec": { + "apiServer": "https://0.0.0.0:42281", + "namespace": "k3d-loki-distributed", + "resourceDefaults": {}, + "expectVersions": {} + } +} diff --git a/tools/dev/k3d/environments/loki-distributed/values/default/config.libsonnet b/tools/dev/k3d/environments/loki-distributed/values/default/config.libsonnet new file mode 100644 index 000000000000..c400c03e3524 --- /dev/null +++ b/tools/dev/k3d/environments/loki-distributed/values/default/config.libsonnet @@ -0,0 +1,67 @@ +{ + auth_enabled: false, + common: { + path_prefix: '/var/loki', + replication_factor: 3, + ring: { + kvstore: { + store: 'memberlist', + }, + }, + storage: { + s3: { + endpoint: 'minio:9000', + insecure: true, + bucketnames: 'loki-data', + access_key_id: ' loki', + secret_access_key: ' supersecret', + s3forcepathstyle: true, + }, + }, + }, + server: { + http_listen_port: 3100, + }, + ingester: { + lifecycler: { + final_sleep: '0s', + }, + }, + memberlist: { + join_members: [ + '{{ include "loki.fullname" . }}-memberlist', + ], + }, + schema_config: { + configs: [ + { + from: '2020-05-15', + store: 'boltdb-shipper', + object_store: 'filesystem', + schema: 'v11', + index: { + prefix: 'index_', + period: '24h', + }, + }, + ], + }, + limits_config: { + enforce_metric_name: false, + reject_old_samples: true, + reject_old_samples_max_age: '168h', + retention_period: '24h', + }, + frontend: { + log_queries_longer_than: '5s', + compress_responses: true, + }, + frontend_worker: { + frontend_address: '{{ include "loki.queryFrontendFullname" . }}:9095', + parallelism: 6, + match_max_concurrent: true, + }, + querier: { + max_concurrent: 6, + }, +} diff --git a/tools/dev/k3d/environments/loki-distributed/values/default/values.libsonnet b/tools/dev/k3d/environments/loki-distributed/values/default/values.libsonnet new file mode 100644 index 000000000000..b1db4ad9a920 --- /dev/null +++ b/tools/dev/k3d/environments/loki-distributed/values/default/values.libsonnet @@ -0,0 +1,56 @@ +{ + lokiValues: function(configStr) { + loki: { + config: configStr, + }, + ingester: { + replicas: 3, + persistence: { + enabled: true, + }, + podAnnotations: { + 'prometheus.io/scrape': 'true', + 'prometheus.io/path': '/metrics', + 'prometheus.io/port': '3100', + }, + }, + distributor: { + replicas: 3, + podAnnotations: { + 'prometheus.io/scrape': 'true', + 'prometheus.io/path': '/metrics', + 'prometheus.io/port': '3100', + }, + }, + querier: { + replicas: 3, + podAnnotations: { + 'prometheus.io/scrape': 'true', + 'prometheus.io/path': '/metrics', + 'prometheus.io/port': '3100', + }, + }, + queryFrontend: { + replicas: 3, + podAnnotations: { + 'prometheus.io/scrape': 'true', + 'prometheus.io/path': '/metrics', + 'prometheus.io/port': '3100', + }, + }, + gateway: { + replicas: 1, + }, + compactor: { + enabled: true, + persistence: { + enabled: true, + }, + podAnnotations: { + 'prometheus.io/scrape': 'true', + 'prometheus.io/path': '/metrics', + 'prometheus.io/port': '3100', + }, + }, + }, +} diff --git a/tools/dev/k3d/jsonnetfile.json b/tools/dev/k3d/jsonnetfile.json new file mode 100644 index 000000000000..bf9d414cb765 --- /dev/null +++ b/tools/dev/k3d/jsonnetfile.json @@ -0,0 +1,60 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "ksonnet-util" + } + }, + "version": "master" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "tanka-util" + } + }, + "version": "master" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/k8s-libsonnet.git", + "subdir": "1.20" + } + }, + "version": "main" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "grafana" + } + }, + "version": "master" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "enterprise-metrics" + } + }, + "version": "master" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "loki-simple-scalable" + } + }, + "version": "create-ssd-lib" + } + ], + "legacyImports": true +} diff --git a/tools/dev/k3d/jsonnetfile.lock.json b/tools/dev/k3d/jsonnetfile.lock.json new file mode 100644 index 000000000000..a3ffb0f515de --- /dev/null +++ b/tools/dev/k3d/jsonnetfile.lock.json @@ -0,0 +1,126 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/cortex-jsonnet.git", + "subdir": "cortex" + } + }, + "version": "acbe9614c51e2020930901d1b9f3ef33c527409c", + "sum": "maEEiq7pOZVlyQgwFdjujsDP3f4q5wGLna8X89s0DQg=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "consul" + } + }, + "version": "5a128df878434da37969b811e99bb9cd0a3779e3", + "sum": "Po3c1Ic96ngrJCtOazic/7OsLkoILOKZWXWyZWl+od8=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "enterprise-metrics" + } + }, + "version": "5a128df878434da37969b811e99bb9cd0a3779e3", + "sum": "zBjtNmnhfGlOHX0yOQIkaN8qX0LymfUkbj5yEG9EakQ=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "etcd-operator" + } + }, + "version": "5a128df878434da37969b811e99bb9cd0a3779e3", + "sum": "fQC1HY2iSTGfd4UfUE0nomf6NcRT+6xaVTDgwwRCrQc=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "grafana" + } + }, + "version": "5a128df878434da37969b811e99bb9cd0a3779e3", + "sum": "7rANfqY8ERvoABHbwoGsdGpUeHxxYCSVOcM4Eky4QtQ=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "jaeger-agent-mixin" + } + }, + "version": "5a128df878434da37969b811e99bb9cd0a3779e3", + "sum": "DsdBoqgx5kE3zc6fMYnfiGjW2+9Mx2OXFieWm1oFHgY=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "ksonnet-util" + } + }, + "version": "5a128df878434da37969b811e99bb9cd0a3779e3", + "sum": "JDsc/bUs5Yv1RkGKcm0hMteqCKZqemxA3qP6eiEATr8=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "loki-simple-scalable" + } + }, + "version": "0dfdeb96f42f2050ba923b92dfdbf090d26569f4", + "sum": "788HNReqyKDNH7ARqwepctP7bqBAfawarD4jgK+2ugQ=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "memcached" + } + }, + "version": "5a128df878434da37969b811e99bb9cd0a3779e3", + "sum": "dTOeEux3t9bYSqP2L/uCuLo/wUDpCKH4w+4OD9fePUk=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/jsonnet-libs.git", + "subdir": "tanka-util" + } + }, + "version": "5a128df878434da37969b811e99bb9cd0a3779e3", + "sum": "AGgjH6IJe/1qwNtxFIiG8V1uyOJZascEydQsNrfPQ4c=" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/docsonnet.git", + "subdir": "doc-util" + } + }, + "version": "fc3f9bca2dff836b0e924a993bdf11bc51af78d4", + "sum": "JUBWG9ybm0TlY3uCWrNoQS00BcfPlCvuK9jPFU0NIj8=" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/k8s-libsonnet.git", + "subdir": "1.20" + } + }, + "version": "f8efa81cf15257bd151b97e31599e20b2ba5311b", + "sum": "GSPYEqrqDbHKCZ/uFqx3MBmofDhE9Bh+2wP5k8Z39RY=" + } + ], + "legacyImports": false +} diff --git a/tools/dev/k3d/lib/grafana/grafana.libsonnet b/tools/dev/k3d/lib/grafana/grafana.libsonnet new file mode 100644 index 000000000000..3530cdb94c6e --- /dev/null +++ b/tools/dev/k3d/lib/grafana/grafana.libsonnet @@ -0,0 +1,66 @@ +local k = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet'; +local tanka = import 'github.com/grafana/jsonnet-libs/tanka-util/main.libsonnet'; +local spec = (import './spec.json').spec; +local helm = tanka.helm.new(std.thisFile) { + template(name, chart, conf={}):: + std.native('helmTemplate')(name, chart, conf { calledFrom: std.thisFile }), +}; +{ + _config+:: { + jaegerAgentName: error 'please provide $._config.jaegerAgentName', + jaegerAgentPort: 6831, + namespace: error 'plase provide $._config.namespace', + provisioningDir: '/etc/grafana/provisioning', + lokiUrl: error 'please provide $._config.lokiUrl', + grafana: { + datasources: [], + }, + }, + + _images+:: { + grafana: { + repository: 'grafana/grafana', + tag: 'latest', + pullPolicy: 'IfNotPresent', + }, + }, + + grafana: helm.template('grafana', '../../charts/grafana', { + namespace: $._config.namespace, + values: { + image: $._images.grafana, + testFramework: { + enabled: false, + }, + env: { + GF_AUTH_ANONYMOUS_ENABLED: true, + GF_AUTH_ANONYMOUS_ORG_ROLE: 'Admin', + GF_FEATURE_TOGGLES_ENABLE: 'ngalert', + JAEGER_AGENT_PORT: 6831, + JAEGER_AGENT_HOST: $._config.jaegerAgentName, + }, + podAnnotations: { + 'prometheus.io/scrape': 'true', + 'prometheus.io/port': '3000', + }, + datasources: { + 'datasources.yaml': { + apiVersion: 1, + datasources: $._config.grafana.datasources, + }, + }, + 'grafana.ini': { + 'tracing.jaeger': { + always_included_tag: 'app=grafana', + sampler_type: 'const', + sampler_param: 1, + }, + paths: { + provisioning: $._config.provisioningDir, + }, + }, + }, + kubeVersion: 'v1.18.0', + noHooks: false, + }), +} diff --git a/tools/dev/k3d/lib/jaeger/jaeger.libsonnet b/tools/dev/k3d/lib/jaeger/jaeger.libsonnet new file mode 100644 index 000000000000..3b684cdb017d --- /dev/null +++ b/tools/dev/k3d/lib/jaeger/jaeger.libsonnet @@ -0,0 +1,67 @@ +local k = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet'; +{ + local deployment = k.apps.v1.deployment, + local container = k.core.v1.container, + local port = k.core.v1.containerPort, + local service = k.core.v1.service, + local servicePort = k.core.v1.servicePort, + local envVar = if std.objectHasAll(k.core.v1, 'envVar') then k.core.v1.envVar else k.core.v1.container.envType, + local queryServiceName = self.jaeger.query_service.metadata.name, + + _config+:: { + jagerAgentName: error 'must provide $._config.jaegerAgentName', + jaeger: { + agentPort: 6831, + queryPort: 16686, + queryGrpcPort: 16685, + agentConfigsPort: 5778, + collectorZipkinHttpPort: 9411, + }, + }, + + _addJaegerEnvVars:: function(c) c { + env: [ + envVar.new('JAEGER_AGENT_HOST', $._config.jaegerAgentName), + envVar.new('JAEGER_AGENT_PORT', '6831'), + envVar.new('JAEGER_SAMPLER_TYPE', 'const'), + envVar.new('JAEGER_SAMPLER_PARAM', '1'), + envVar.new('JAEGER_TAGS', 'app=gel'), + ], + }, + + jaeger: { + deployment: deployment.new(name='jaeger', replicas=1, containers=[ + container.new('jaeger', 'jaegertracing/all-in-one') + + container.withPorts([ + port.newNamed($._config.jaeger.queryGrpcPort, 'query-grpc'), + port.newNamed($._config.jaeger.agentConfigsPort, 'agent-configs'), + port.newNamed($._config.jaeger.collectorZipkinHttpPort, 'zipkin'), + port.newNamed($._config.jaeger.queryPort, 'query'), + port.newNamedUDP($._config.jaeger.agentPort, 'agent'), + port.newNamedUDP(5775, 'zipkin-thrift'), + port.newNamedUDP(6832, 'jaeger-thrift'), + ]) + + container.withEnv([ + envVar.new('COLLECTOR_ZIPKIN_HTTP_PORT', '%d' % [$._config.jaeger.collectorZipkinHttpPort]), + envVar.new('JAEGER_AGENT_HOST', queryServiceName), + envVar.new('JAEGER_AGENT_PORT', '%d' % [$._config.jaeger.agentPort]), + ]) + container.mixin.readinessProbe.httpGet.withPath('/') + + container.mixin.readinessProbe.httpGet.withPort(14269) + + container.mixin.readinessProbe.withInitialDelaySeconds(5), + ]), + + query_service: service.new('jaeger-query', { + name: 'jaeger', + }, [ + servicePort.newNamed('query', 80, $._config.jaeger.queryPort), + servicePort.newNamed('query-gprc', $._config.jaeger.queryGrpcPort, $._config.jaeger.queryGrpcPort), + ]), + + agent_service: service.new('jaeger-agent', { + name: 'jaeger', + }, [ + servicePort.newNamed('agent-compat', $._config.jaeger.agentPort, $._config.jaeger.agentPort) + servicePort.withProtocol('UDP'), + servicePort.newNamed('agent-configs', $._config.jaeger.agentConfigsPort, $._config.jaeger.agentConfigsPort), + ]), + }, +} diff --git a/tools/dev/k3d/lib/k.libsonnet b/tools/dev/k3d/lib/k.libsonnet new file mode 100644 index 000000000000..942b20141b7e --- /dev/null +++ b/tools/dev/k3d/lib/k.libsonnet @@ -0,0 +1 @@ +import 'github.com/jsonnet-libs/k8s-libsonnet/1.20/main.libsonnet' diff --git a/tools/dev/k3d/lib/prometheus/prometheus.libsonnet b/tools/dev/k3d/lib/prometheus/prometheus.libsonnet new file mode 100644 index 000000000000..0d6df3c701dc --- /dev/null +++ b/tools/dev/k3d/lib/prometheus/prometheus.libsonnet @@ -0,0 +1,40 @@ +local k = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet'; +local tanka = import 'github.com/grafana/jsonnet-libs/tanka-util/main.libsonnet'; +local helm = tanka.helm.new(std.thisFile) { + template(name, chart, conf={}):: + std.native('helmTemplate')(name, chart, conf { calledFrom: std.thisFile }), +}; +{ + local envVar = k.core.v1.envVar, + _config+:: { + jaegerAgentName: error 'please provide $._config.jaegerAgentName', + jaegerAgentPort: 6831, + namespace: error 'plase prvoide $._config.namespace', + }, + + _prometheusAnnotations:: { 'prometheus.io/scrape': 'true', 'prometheus.io/port': '3100' }, + + prometheus: helm.template('prometheus', '../../charts/prometheus', { + namespace: $._config.namespace, + values: { + server: { + env: [ + envVar.new('JAEGER_AGENT_HOST', $._config.jaegerAgentName), + envVar.new('JAEGER_AGENT_PORT', '%d' % $._config.jaegerAgentPort), + envVar.new('JAEGER_SAMPLER_TYPE', 'const'), + envVar.new('JAEGER_SAMPLER_PARAM', '1'), + envVar.new('JAEGER_TAGS', 'app=prometheus'), + ], + }, + alertmanager: { + enabled: false, + }, + pushgateway: { + enabled: false, + }, + }, + kubeVersion: 'v1.18.0', + noHooks: false, + }), + +} diff --git a/tools/dev/k3d/lib/promtail/promtail.libsonnet b/tools/dev/k3d/lib/promtail/promtail.libsonnet new file mode 100644 index 000000000000..86eea835b606 --- /dev/null +++ b/tools/dev/k3d/lib/promtail/promtail.libsonnet @@ -0,0 +1,24 @@ +local k = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet'; +local tanka = import 'github.com/grafana/jsonnet-libs/tanka-util/main.libsonnet'; +local helm = tanka.helm.new(std.thisFile) { + template(name, chart, conf={}):: + std.native('helmTemplate')(name, chart, conf { calledFrom: std.thisFile }), +}; +{ + _config+:: { + namespace: error 'please provide $._config.namespace', + gatewayHost: error 'please provide $._config.gatewayAddress', + }, + + promtail: helm.template('promtail', '../../charts/promtail', { + namespace: $._config.namespace, + values: { + extraArgs: ['--config.expand-env=true'], + config: { + lokiAddress: 'http://%s/loki/api/v1/push' % $._config.gatewayHost, + }, + }, + kubeVersion: 'v1.18.0', + noHooks: false, + }), +} diff --git a/tools/dev/k3d/scripts/create_cluster.sh b/tools/dev/k3d/scripts/create_cluster.sh new file mode 100755 index 000000000000..86b94bc0a3a9 --- /dev/null +++ b/tools/dev/k3d/scripts/create_cluster.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +current_dir="$(cd "$(dirname "$0")" && pwd)" +k3d_dir="$(cd "${current_dir}/.." && pwd)" + +cluster_name="$1" +namespace="k3d-${cluster_name}" +environment="${k3d_dir}/environments/${cluster_name}" +registry_port="${REGISTRY_PORT:-45629}" + +if k3d registry list | grep -q -m 1 k3d-grafana; then + k3d registry create k3d-grafana \ + --port "${registry_port}" +fi + +if k3d cluster list | grep -q -m 1 "${cluster_name}"; then + k3d cluster create "${cluster_name}" \ + --servers 1 \ + --agents 3 \ + --registry-use "k3d-grafana:${registry_port}" \ + --wait +fi + +tk env set "${environment}" \ + --server="https://0.0.0.0:$(k3d node list -o json | jq -r ".[] | select(.name == \"k3d-${cluster_name}-serverlb\") | .portMappings.\"6443\"[] | .HostPort")" \ + --namespace="${namespace}" + +kubectl config set-context "${namespace}" + +if kubectl get namespaces | grep -q -m 1 "${namespace}"; then + kubectl create namespace "${namespace}" || true +fi + +tk apply "${environment}" diff --git a/tools/dev/k3d/vendor/.gitignore b/tools/dev/k3d/vendor/.gitignore new file mode 100644 index 000000000000..d6b7ef32c847 --- /dev/null +++ b/tools/dev/k3d/vendor/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore