Add tooling for local k3d clusters under tools/dev (#5272)

* add k3d folder with loki-distributed env

* add readme

* shellcheck create_cluster.sh

* lint jsonnet

* Update grafana image to latest

tools/dev/k3d/Makefile

@@ -0,0 +1,27 @@
+.PHONY: loki-distributed down add-repos update-repos prepare build-latest-image
+
+IMAGE_TAG := $(shell ../../../tools/image-tag)
+REGISTRY_PORT ?= 45629
+
+loki-distributed: prepare
+	$(CURDIR)/scripts/create_cluster.sh "loki-distributed"
+
+down:
+	k3d cluster delete loki-distributed
+
+add-repos:
+	helm repo add --force-update prometheus-community https://prometheus-community.github.io/helm-charts
+	helm repo add --force-update grafana https://grafana.github.io/helm-charts
+	helm repo add --force-update minio https://helm.min.io
+
+update-repos: add-repos
+	helm repo update
+	tk tool charts vendor
+	jb update
+
+prepare: update-repos build-latest-image
+
+build-latest-image:
+	make -C $(CURDIR)/../../.. loki-image
+	docker tag grafana/loki:$(IMAGE_TAG) k3d-grafana:$(REGISTRY_PORT)/loki:latest
+	docker push k3d-grafana:$(REGISTRY_PORT)/loki:latest

tools/dev/k3d/README.md

@@ -0,0 +1,31 @@
+# Deploy Loki to k3d for Local Development
+
+## Pre-requisites 
+
+In order to use the make targets in this directory, make sure you have the following tools installed:
+* [kubectl](https://kubernetes.io/docs/tasks/tools/)
+* [k3d](https://k3d.io/v4.4.8/)
+* [tanka](https://github.com/grafana/tanka)
+* [jsonnet](https://jsonnet.org/)
+* [jq](https://stedolan.github.io/jq/)
+
+**Note**: in case docker is unable to resolve the local k3d registry hostname, add the following entry to the `/etc/hosts` file:
+```
+127.0.0.1 k3d-grafana
+```
+
+## Spinning Up An Environment
+
+Each environment has it's own make target. To bring up `loki-distributed`, for example, run:
+
+```bash
+make loki-distributed
+```
+
+## Tearing Down An Environment
+
+The `down` make target will tear down all environments.
+
+```bash
+make down
+```

tools/dev/k3d/chartfile.yaml

@@ -0,0 +1,22 @@
+directory: charts
+repositories:
+- name: grafana
+  url: https://grafana.github.io/grafana/helm-charts
+- name: prometheus-community
+  url: https://prometheus-community.github.io/helm-charts
+- name: minio
+  url: https://helm.min.io
+requires:
+- chart: grafana/promtail
+  version: 3.8.1
+- chart: grafana/grafana
+  version: 6.20.1
+- chart: prometheus-community/prometheus
+  version: 15.0.1
+- chart: minio/minio
+  version: 8.0.10
+- chart: grafana/loki-distributed
+  version: 0.42.0
+- chart: grafana/loki-simple-scalable
+  version: 0.2.0
+version: 1

tools/dev/k3d/charts/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

tools/dev/k3d/environments/loki-distributed/main.jsonnet

@@ -0,0 +1,131 @@
+local k = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet';
+local tanka = import 'github.com/grafana/jsonnet-libs/tanka-util/main.libsonnet';
+local spec = (import './spec.json').spec;
+
+local jaeger = import 'jaeger/jaeger.libsonnet';
+local grafana = import 'grafana/grafana.libsonnet';
+local prometheus = import 'prometheus/prometheus.libsonnet';
+local promtail = import 'promtail/promtail.libsonnet';
+
+local helm = tanka.helm.new(std.thisFile) {
+  template(name, chart, conf={})::
+    std.native('helmTemplate')(name, chart, conf { calledFrom: std.thisFile }),
+};
+local clusterName = 'loki-distributed';
+local normalizedClusterName = std.strReplace(clusterName, '-', '_');
+
+grafana + prometheus + promtail + jaeger {
+  local gatewayName = self.loki.service_loki_distributed_gateway.metadata.name,
+  local gatewayHost = '%s' % gatewayName,
+  local gatewayUrl = 'http://%s' % gatewayHost,
+  local jaegerQueryName = self.jaeger.query_service.metadata.name,
+  local jaegerQueryUrl = 'http://%s' % jaegerQueryName,
+  local jaegerAgentName = self.jaeger.agent_service.metadata.name,
+  local jaegerAgentUrl = 'http://%s' % jaegerAgentName,
+  local prometheusServerName = self.prometheus.service_prometheus_server.metadata.name,
+  local prometheusUrl = 'http://%s' % prometheusServerName,
+  local namespace = spec.namespace,
+
+  _config+:: {
+    clusterName: clusterName,
+    gatewayName: gatewayName,
+    gatewayHost: gatewayHost,
+    gelUrl: gatewayUrl,
+    jaegerAgentName: jaegerAgentName,
+    jaegerAgentPort: 6831,
+    namespace: namespace,
+    adminToken: 'gel-admin-token',
+
+    grafana: {
+      datasources: [
+        {
+          name: 'Prometheus',
+          type: 'prometheus',
+          access: 'proxy',
+          url: prometheusUrl,
+        },
+        {
+          name: 'Jaeger',
+          type: 'jaeger',
+          access: 'proxy',
+          url: jaegerQueryUrl,
+          uid: 'jaeger_uid',
+        },
+        {
+          name: 'Loki',
+          type: 'loki',
+          access: 'proxy',
+          url: gatewayUrl,
+          jsonData: {
+            derivedFields: [
+              {
+                datasourceUid: 'jaeger_uid',
+                matcherRegex: 'traceID=(\\w+)',
+                name: 'TraceID',
+                url: '$${__value.raw}',
+              },
+            ],
+          },
+        },
+      ],
+    },
+  },
+
+  minio: helm.template('minio', '../../charts/minio', {
+    namespace: $._config.namespace,
+    values: {
+      accessKey: 'loki',
+      rootUser: 'loki',
+      secretKey: 'supersecret',
+      rootPassword: 'supersecret',
+      buckets: [
+        {
+          name: 'loki-data',
+          policy: 'public',
+          purge: false,
+        },
+      ],
+      persistence: {
+        enabled: true,
+        'storage-class': 'local-path',
+        size: '10Gi',
+      },
+    },
+  }),
+
+  local config = import './values/default/config.libsonnet',
+  local values = (import './values/default/values.libsonnet').lokiValues(k.util.manifestYaml(config)),
+
+  loki: helm.template($._config.clusterName, '../../charts/loki-distributed', {
+    namespace: $._config.namespace,
+    values: values {
+      local registry = 'k3d-grafana:45629',
+      loki+: {
+        image: {
+          registry: registry,
+          repository: 'loki',
+          tag: 'latest',
+          pullPolicy: 'Always',
+        },
+      },
+    },
+  }) + {
+    ['deployment_loki_distributed_%s' % [name]]+:
+      k.apps.v1.deployment.mapContainers($._addJaegerEnvVars) +
+      k.apps.v1.deployment.spec.template.metadata.withAnnotations($._prometheusAnnotations)
+    for name in [
+      'compactor',
+      'distributor',
+      'gateway',
+      'query_frontend',
+    ]
+  } + {
+    ['stateful_set_loki_distributed_%s' % [name]]+:
+      k.apps.v1.statefulSet.mapContainers($._addJaegerEnvVars) +
+      k.apps.v1.statefulSet.spec.template.metadata.withAnnotations($._prometheusAnnotations)
+    for name in [
+      'ingester',
+      'querier',
+    ]
+  },
+}

tools/dev/k3d/environments/loki-distributed/spec.json

@@ -0,0 +1,14 @@
+{
+  "apiVersion": "tanka.dev/v1alpha1",
+  "kind": "Environment",
+  "metadata": {
+    "name": "environments/loki-distributed",
+    "namespace": "environments/loki-distributed/main.jsonnet"
+  },
+  "spec": {
+    "apiServer": "https://0.0.0.0:42281",
+    "namespace": "k3d-loki-distributed",
+    "resourceDefaults": {},
+    "expectVersions": {}
+  }
+}

tools/dev/k3d/environments/loki-distributed/values/default/config.libsonnet

@@ -0,0 +1,67 @@
+{
+  auth_enabled: false,
+  common: {
+    path_prefix: '/var/loki',
+    replication_factor: 3,
+    ring: {
+      kvstore: {
+        store: 'memberlist',
+      },
+    },
+    storage: {
+      s3: {
+        endpoint: 'minio:9000',
+        insecure: true,
+        bucketnames: 'loki-data',
+        access_key_id: ' loki',
+        secret_access_key: ' supersecret',
+        s3forcepathstyle: true,
+      },
+    },
+  },
+  server: {
+    http_listen_port: 3100,
+  },
+  ingester: {
+    lifecycler: {
+      final_sleep: '0s',
+    },
+  },
+  memberlist: {
+    join_members: [
+      '{{ include "loki.fullname" . }}-memberlist',
+    ],
+  },
+  schema_config: {
+    configs: [
+      {
+        from: '2020-05-15',
+        store: 'boltdb-shipper',
+        object_store: 'filesystem',
+        schema: 'v11',
+        index: {
+          prefix: 'index_',
+          period: '24h',
+        },
+      },
+    ],
+  },
+  limits_config: {
+    enforce_metric_name: false,
+    reject_old_samples: true,
+    reject_old_samples_max_age: '168h',
+    retention_period: '24h',
+  },
+  frontend: {
+    log_queries_longer_than: '5s',
+    compress_responses: true,
+  },
+  frontend_worker: {
+    frontend_address: '{{ include "loki.queryFrontendFullname" . }}:9095',
+    parallelism: 6,
+    match_max_concurrent: true,
+  },
+  querier: {
+    max_concurrent: 6,
+  },
+}

tools/dev/k3d/environments/loki-distributed/values/default/values.libsonnet

@@ -0,0 +1,56 @@
+{
+  lokiValues: function(configStr) {
+    loki: {
+      config: configStr,
+    },
+    ingester: {
+      replicas: 3,
+      persistence: {
+        enabled: true,
+      },
+      podAnnotations: {
+        'prometheus.io/scrape': 'true',
+        'prometheus.io/path': '/metrics',
+        'prometheus.io/port': '3100',
+      },
+    },
+    distributor: {
+      replicas: 3,
+      podAnnotations: {
+        'prometheus.io/scrape': 'true',
+        'prometheus.io/path': '/metrics',
+        'prometheus.io/port': '3100',
+      },
+    },
+    querier: {
+      replicas: 3,
+      podAnnotations: {
+        'prometheus.io/scrape': 'true',
+        'prometheus.io/path': '/metrics',
+        'prometheus.io/port': '3100',
+      },
+    },
+    queryFrontend: {
+      replicas: 3,
+      podAnnotations: {
+        'prometheus.io/scrape': 'true',
+        'prometheus.io/path': '/metrics',
+        'prometheus.io/port': '3100',
+      },
+    },
+    gateway: {
+      replicas: 1,
+    },
+    compactor: {
+      enabled: true,
+      persistence: {
+        enabled: true,
+      },
+      podAnnotations: {
+        'prometheus.io/scrape': 'true',
+        'prometheus.io/path': '/metrics',
+        'prometheus.io/port': '3100',
+      },
+    },
+  },
+}