From e8a9555346e99c0bc4664b7fd8fc9c36336ae53f Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sat, 2 Feb 2019 02:17:55 +0200 Subject: [PATCH] Add GKE Istio Gateway and Prometheus definitions --- .gitignore | 2 + artifacts/canaries/canary.yaml | 2 +- artifacts/canaries/deployment.yaml | 2 +- artifacts/gke/istio-gateway.yaml | 27 ++ artifacts/gke/istio-prometheus.yaml | 443 ++++++++++++++++++++++++++++ 5 files changed, 474 insertions(+), 2 deletions(-) create mode 100644 artifacts/gke/istio-gateway.yaml create mode 100644 artifacts/gke/istio-prometheus.yaml diff --git a/.gitignore b/.gitignore index 95cf6da5e..6a78657de 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out .DS_Store + +artifacts/gcloud/ \ No newline at end of file diff --git a/artifacts/canaries/canary.yaml b/artifacts/canaries/canary.yaml index f4d88afbd..70c86c072 100644 --- a/artifacts/canaries/canary.yaml +++ b/artifacts/canaries/canary.yaml @@ -25,7 +25,7 @@ spec: - public-gateway.istio-system.svc.cluster.local # Istio virtual service host names (optional) hosts: - - app.iowa.weavedx.com + - app.istio.weavedx.com canaryAnalysis: # schedule interval (default 60s) interval: 10s diff --git a/artifacts/canaries/deployment.yaml b/artifacts/canaries/deployment.yaml index de686bbc0..57ed8a419 100644 --- a/artifacts/canaries/deployment.yaml +++ b/artifacts/canaries/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.3.0 + image: quay.io/stefanprodan/podinfo:1.4.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/artifacts/gke/istio-gateway.yaml b/artifacts/gke/istio-gateway.yaml new file mode 100644 index 000000000..79c016156 --- /dev/null +++ b/artifacts/gke/istio-gateway.yaml @@ -0,0 +1,27 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: public-gateway + namespace: istio-system +spec: + selector: + istio: ingressgateway + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" + tls: + httpsRedirect: true + - port: + number: 443 + name: https + protocol: HTTPS + hosts: + - "*" + tls: + mode: SIMPLE + privateKey: /etc/istio/ingressgateway-certs/tls.key + serverCertificate: /etc/istio/ingressgateway-certs/tls.crt diff --git a/artifacts/gke/istio-prometheus.yaml b/artifacts/gke/istio-prometheus.yaml new file mode 100644 index 000000000..ad9dbdcf4 --- /dev/null +++ b/artifacts/gke/istio-prometheus.yaml @@ -0,0 +1,443 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus + labels: + app: prometheus +rules: + - apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + - nodes/proxy + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: + - configmaps + verbs: ["get"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: prometheus + labels: + app: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: istio-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: istio-system + labels: + app: prometheus +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus + namespace: istio-system + labels: + app: prometheus +data: + prometheus.yml: |- + global: + scrape_interval: 15s + scrape_configs: + + - job_name: 'istio-mesh' + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - istio-system + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-telemetry;prometheus + + + # Scrape config for envoy stats + - job_name: 'envoy-stats' + metrics_path: /stats/prometheus + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: '.*-envoy-prom' + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:15090 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod_name + + metric_relabel_configs: + # Exclude some of the envoy metrics that have massive cardinality + # This list may need to be pruned further moving forward, as informed + # by performance and scalability testing. + - source_labels: [ cluster_name ] + regex: '(outbound|inbound|prometheus_stats).*' + action: drop + - source_labels: [ tcp_prefix ] + regex: '(outbound|inbound|prometheus_stats).*' + action: drop + - source_labels: [ listener_address ] + regex: '(.+)' + action: drop + - source_labels: [ http_conn_manager_listener_prefix ] + regex: '(.+)' + action: drop + - source_labels: [ http_conn_manager_prefix ] + regex: '(.+)' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_tls.*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_tcp_downstream.*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_http_(stats|admin).*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_cluster_(lb|retry|bind|internal|max|original).*' + action: drop + + + - job_name: 'istio-policy' + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - istio-system + + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-policy;http-monitoring + + - job_name: 'istio-telemetry' + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - istio-system + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-telemetry;http-monitoring + + - job_name: 'pilot' + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - istio-system + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-pilot;http-monitoring + + - job_name: 'galley' + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - istio-system + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-galley;http-monitoring + + # scrape config for API servers + - job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - default + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kubernetes;https + + # scrape config for nodes (kubelet) + - job_name: 'kubernetes-nodes' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + + # Scrape config for Kubelet cAdvisor. + # + # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics + # (those whose names begin with 'container_') have been removed from the + # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to + # retrieve those metrics. + # + # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor + # HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics" + # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with + # the --cadvisor-port=0 Kubelet flag). + # + # This job is not necessary and should be removed in Kubernetes 1.6 and + # earlier versions, or it will cause the metrics to be scraped twice. + - job_name: 'kubernetes-cadvisor' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + # scrape config for service endpoints. + - job_name: 'kubernetes-service-endpoints' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name + + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: # If first two labels are present, pod should be scraped by the istio-secure job. + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_sidecar_istio_io_status] + action: drop + regex: (.+) + - source_labels: [__meta_kubernetes_pod_annotation_istio_mtls] + action: drop + regex: (true) + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod_name + + - job_name: 'kubernetes-pods-istio-secure' + scheme: https + tls_config: + ca_file: /etc/istio-certs/root-cert.pem + cert_file: /etc/istio-certs/cert-chain.pem + key_file: /etc/istio-certs/key.pem + insecure_skip_verify: true # prometheus does not support secure naming. + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + # sidecar status annotation is added by sidecar injector and + # istio_workload_mtls_ability can be specifically placed on a pod to indicate its ability to receive mtls traffic. + - source_labels: [__meta_kubernetes_pod_annotation_sidecar_istio_io_status, __meta_kubernetes_pod_annotation_istio_mtls] + action: keep + regex: (([^;]+);([^;]*))|(([^;]*);(true)) + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__] # Only keep address that is host:port + action: keep # otherwise an extra target with ':443' is added for https scheme + regex: ([^:]+):(\d+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod_name +--- + +# Source: istio/charts/prometheus/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: istio-system + annotations: + prometheus.io/scrape: 'true' + labels: + name: prometheus +spec: + selector: + app: prometheus + ports: + - name: http-prometheus + protocol: TCP + port: 9090 + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: istio-system + labels: + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + annotations: + sidecar.istio.io/inject: "false" + scheduler.alpha.kubernetes.io/critical-pod: "" + spec: + serviceAccountName: prometheus + containers: + - name: prometheus + image: "docker.io/prom/prometheus:v2.7.1" + imagePullPolicy: IfNotPresent + args: + - '--storage.tsdb.retention=6h' + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - containerPort: 9090 + name: http + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + resources: + requests: + cpu: 10m + + volumeMounts: + - name: config-volume + mountPath: /etc/prometheus + - mountPath: /etc/istio-certs + name: istio-certs + volumes: + - name: config-volume + configMap: + name: prometheus + - name: istio-certs + secret: + defaultMode: 420 + optional: true + secretName: istio.default