From 19bfef48cbad57468591e8214c4a5f390091f1e1 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Fri, 19 Apr 2024 09:41:16 +0200 Subject: [PATCH] feat: Add pattern ingester support in SSD mode (#12685) --- docs/sources/setup/install/helm/reference.md | 16 +++++++++++++++- pkg/loki/loki.go | 2 +- production/helm/loki/CHANGELOG.md | 10 ++++------ production/helm/loki/Chart.yaml | 2 +- production/helm/loki/README.md | 2 +- production/helm/loki/values.yaml | 10 ++++++---- 6 files changed, 28 insertions(+), 14 deletions(-) diff --git a/docs/sources/setup/install/helm/reference.md b/docs/sources/setup/install/helm/reference.md index 7a2e697f331d..525dc05e4b9a 100644 --- a/docs/sources/setup/install/helm/reference.md +++ b/docs/sources/setup/install/helm/reference.md @@ -5211,7 +5211,7 @@ null "replication_factor": 3 }, "compactor": {}, - "config": "{{- if .Values.enterprise.enabled}}\n{{- tpl .Values.enterprise.config . }}\n{{- else }}\nauth_enabled: {{ .Values.loki.auth_enabled }}\n{{- end }}\n\n{{- with .Values.loki.server }}\nserver:\n {{- toYaml . | nindent 2}}\n{{- end}}\n\nmemberlist:\n{{- if .Values.loki.memberlistConfig }}\n {{- toYaml .Values.loki.memberlistConfig | nindent 2 }}\n{{- else }}\n{{- if .Values.loki.extraMemberlistConfig}}\n{{- toYaml .Values.loki.extraMemberlistConfig | nindent 2}}\n{{- end }}\n join_members:\n - {{ include \"loki.memberlist\" . }}\n {{- with .Values.migrate.fromDistributed }}\n {{- if .enabled }}\n - {{ .memberlistService }}\n {{- end }}\n {{- end }}\n{{- end }}\n\n{{- with .Values.loki.ingester }}\ningester:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- if .Values.loki.commonConfig}}\ncommon:\n{{- toYaml .Values.loki.commonConfig | nindent 2}}\n storage:\n {{- include \"loki.commonStorageConfig\" . | nindent 4}}\n{{- end}}\n\n{{- with .Values.loki.limits_config }}\nlimits_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\nruntime_config:\n file: /etc/loki/runtime-config/runtime-config.yaml\n\n{{- with .Values.chunksCache }}\n{{- if .enabled }}\nchunk_store_config:\n chunk_cache_config:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached:\n batch_size: {{ .batchSize }}\n parallelism: {{ .parallelism }}\n memcached_client:\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-chunks-cache.{{ $.Release.Namespace }}.svc\n consistent_hash: true\n timeout: {{ .timeout }}\n max_idle_conns: 72\n{{- end }}\n{{- end }}\n\n{{- if .Values.loki.schemaConfig }}\nschema_config:\n{{- toYaml .Values.loki.schemaConfig | nindent 2}}\n{{- end }}\n\n{{- if .Values.loki.useTestSchema }}\nschema_config:\n{{- toYaml .Values.loki.testSchemaConfig | nindent 2}}\n{{- end }}\n\n{{ include \"loki.rulerConfig\" . }}\n\n{{- if or .Values.tableManager.retention_deletes_enabled .Values.tableManager.retention_period }}\ntable_manager:\n retention_deletes_enabled: {{ .Values.tableManager.retention_deletes_enabled }}\n retention_period: {{ .Values.tableManager.retention_period }}\n{{- end }}\n\nquery_range:\n align_queries_with_step: true\n {{- with .Values.loki.query_range }}\n {{- tpl (. | toYaml) $ | nindent 4 }}\n {{- end }}\n {{- if .Values.resultsCache.enabled }}\n {{- with .Values.resultsCache }}\n cache_results: true\n results_cache:\n cache:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached_client:\n consistent_hash: true\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-results-cache.{{ $.Release.Namespace }}.svc\n timeout: {{ .timeout }}\n update_interval: 1m\n {{- end }}\n {{- end }}\n\n{{- with .Values.loki.storage_config }}\nstorage_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.query_scheduler }}\nquery_scheduler:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.compactor }}\ncompactor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.analytics }}\nanalytics:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.querier }}\nquerier:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.index_gateway }}\nindex_gateway:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend }}\nfrontend:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend_worker }}\nfrontend_worker:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.distributor }}\ndistributor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\ntracing:\n enabled: {{ .Values.loki.tracing.enabled }}\n", + "config": "{{- if .Values.enterprise.enabled}}\n{{- tpl .Values.enterprise.config . }}\n{{- else }}\nauth_enabled: {{ .Values.loki.auth_enabled }}\n{{- end }}\n\n{{- with .Values.loki.server }}\nserver:\n {{- toYaml . | nindent 2}}\n{{- end}}\n\npattern_ingester:\n enabled: {{ .Values.loki.pattern_ingester.enabled }}\n\nmemberlist:\n{{- if .Values.loki.memberlistConfig }}\n {{- toYaml .Values.loki.memberlistConfig | nindent 2 }}\n{{- else }}\n{{- if .Values.loki.extraMemberlistConfig}}\n{{- toYaml .Values.loki.extraMemberlistConfig | nindent 2}}\n{{- end }}\n join_members:\n - {{ include \"loki.memberlist\" . }}\n {{- with .Values.migrate.fromDistributed }}\n {{- if .enabled }}\n - {{ .memberlistService }}\n {{- end }}\n {{- end }}\n{{- end }}\n\n{{- with .Values.loki.ingester }}\ningester:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- if .Values.loki.commonConfig}}\ncommon:\n{{- toYaml .Values.loki.commonConfig | nindent 2}}\n storage:\n {{- include \"loki.commonStorageConfig\" . | nindent 4}}\n{{- end}}\n\n{{- with .Values.loki.limits_config }}\nlimits_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\nruntime_config:\n file: /etc/loki/runtime-config/runtime-config.yaml\n\n{{- with .Values.chunksCache }}\n{{- if .enabled }}\nchunk_store_config:\n chunk_cache_config:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached:\n batch_size: {{ .batchSize }}\n parallelism: {{ .parallelism }}\n memcached_client:\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-chunks-cache.{{ $.Release.Namespace }}.svc\n consistent_hash: true\n timeout: {{ .timeout }}\n max_idle_conns: 72\n{{- end }}\n{{- end }}\n\n{{- if .Values.loki.schemaConfig }}\nschema_config:\n{{- toYaml .Values.loki.schemaConfig | nindent 2}}\n{{- end }}\n\n{{- if .Values.loki.useTestSchema }}\nschema_config:\n{{- toYaml .Values.loki.testSchemaConfig | nindent 2}}\n{{- end }}\n\n{{ include \"loki.rulerConfig\" . }}\n\n{{- if or .Values.tableManager.retention_deletes_enabled .Values.tableManager.retention_period }}\ntable_manager:\n retention_deletes_enabled: {{ .Values.tableManager.retention_deletes_enabled }}\n retention_period: {{ .Values.tableManager.retention_period }}\n{{- end }}\n\nquery_range:\n align_queries_with_step: true\n {{- with .Values.loki.query_range }}\n {{- tpl (. | toYaml) $ | nindent 4 }}\n {{- end }}\n {{- if .Values.resultsCache.enabled }}\n {{- with .Values.resultsCache }}\n cache_results: true\n results_cache:\n cache:\n default_validity: {{ .defaultValidity }}\n background:\n writeback_goroutines: {{ .writebackParallelism }}\n writeback_buffer: {{ .writebackBuffer }}\n writeback_size_limit: {{ .writebackSizeLimit }}\n memcached_client:\n consistent_hash: true\n addresses: dnssrvnoa+_memcached-client._tcp.{{ template \"loki.fullname\" $ }}-results-cache.{{ $.Release.Namespace }}.svc\n timeout: {{ .timeout }}\n update_interval: 1m\n {{- end }}\n {{- end }}\n\n{{- with .Values.loki.storage_config }}\nstorage_config:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.query_scheduler }}\nquery_scheduler:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.compactor }}\ncompactor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.analytics }}\nanalytics:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.querier }}\nquerier:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.index_gateway }}\nindex_gateway:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend }}\nfrontend:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.frontend_worker }}\nfrontend_worker:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\n{{- with .Values.loki.distributor }}\ndistributor:\n {{- tpl (. | toYaml) $ | nindent 4 }}\n{{- end }}\n\ntracing:\n enabled: {{ .Values.loki.tracing.enabled }}\n", "configObjectName": "{{ include \"loki.name\" . }}", "configStorageType": "ConfigMap", "containerSecurityContext": { @@ -5269,6 +5269,9 @@ null "timeout": "500ms" } }, + "pattern_ingester": { + "enabled": false + }, "podAnnotations": {}, "podLabels": {}, "podSecurityContext": { @@ -5627,6 +5630,17 @@ null } } + + + + loki.pattern_ingester + object + Optional pattern ingester configuration + 
+{
+  "enabled": false
+}
+
diff --git a/pkg/loki/loki.go b/pkg/loki/loki.go index 606d27899448..cc79ef85d767 100644 --- a/pkg/loki/loki.go +++ b/pkg/loki/loki.go @@ -695,7 +695,7 @@ func (t *Loki) setupModuleManager() error { MemberlistKV: {Server}, Read: {QueryFrontend, Querier}, - Write: {Ingester, Distributor}, + Write: {Ingester, Distributor, PatternIngester}, Backend: {QueryScheduler, Ruler, Compactor, IndexGateway, BloomGateway, BloomCompactor}, All: {QueryScheduler, QueryFrontend, Querier, Ingester, PatternIngester, Distributor, Ruler, Compactor}, diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md index 5b7cb1905bdb..0634fb590461 100644 --- a/production/helm/loki/CHANGELOG.md +++ b/production/helm/loki/CHANGELOG.md @@ -13,6 +13,10 @@ Entries should include a reference to the pull request that introduced the chang [//]: # ( : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.) +## 6.2.2 + +- [FEATURE] Add support for enabling pattern ingester config via `loki.pattern_ingester.enabled`. + ## 6.2.1 - [BUGFIX] Removed duplicate bucketNames from documentation and fixed key name `deploymentMode` @@ -44,7 +48,6 @@ Entries should include a reference to the pull request that introduced the chang - [CHANGE] Changed version of Loki to 2.9.6 - ## 5.46.0 - [CHANGE] Changed version of Loki to 2.9.5 @@ -202,7 +205,6 @@ Entries should include a reference to the pull request that introduced the chang - [CHANGE] Changed version of Grafana Enterprise Logs to v1.8.4 - ## 5.32.0 - [CHANGE] Grafana dashboards are no longer created solely in scalable mode and with external cloud storage enabled. @@ -454,12 +456,10 @@ Entries should include a reference to the pull request that introduced the chang - [CHANGE] Changed version of Grafana Enterprise Logs to v1.7.2 - ## 5.4.0 - [CHANGE] Changed version of Loki to 2.8.2 - - [CHANGE] Change default GEL and Loki versions to 1.7.1 and 2.8.1 respectively - [BUGFIX] Fix dns port in network-policy @@ -469,12 +469,10 @@ Entries should include a reference to the pull request that introduced the chang - [BUGFIX] Add projected volume type to psp - ## 4.9.0 - [CHANGE] Changed version of Loki to 2.7.5 - - [BUGFIX] Fix role/PSP mapping - [BUGFIX] Fix service/ingress mapping diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index cd1abdde0abd..e7ea0e102ea4 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -3,7 +3,7 @@ name: loki description: Helm chart for Grafana Loki in simple, scalable mode type: application appVersion: 3.0.0 -version: 6.2.1 +version: 6.2.2 home: https://grafana.github.io/helm-charts sources: - https://github.com/grafana/loki diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md index 7bde283b2dff..d994600001c3 100644 --- a/production/helm/loki/README.md +++ b/production/helm/loki/README.md @@ -1,6 +1,6 @@ # loki -![Version: 6.2.1](https://img.shields.io/badge/Version-6.2.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square) +![Version: 6.2.2](https://img.shields.io/badge/Version-6.2.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square) Helm chart for Grafana Loki in simple, scalable mode diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 37a941565f42..8c665d13e1cb 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -122,7 +122,6 @@ loki: # The value will be passed through tpl. generatedConfigObjectName: '{{ include "loki.name" . }}' - # -- Config file contents for Loki # @default -- See values.yaml config: | @@ -137,6 +136,9 @@ loki: {{- toYaml . | nindent 2}} {{- end}} + pattern_ingester: + enabled: {{ .Values.loki.pattern_ingester.enabled }} + memberlist: {{- if .Values.loki.memberlistConfig }} {{- toYaml .Values.loki.memberlistConfig | nindent 2 }} @@ -415,6 +417,9 @@ loki: up_to: 3 # -- Optional compactor configuration compactor: {} + # -- Optional pattern ingester configuration + pattern_ingester: + enabled: false # -- Optional analytics configuration analytics: {} # -- Optional querier configuration @@ -851,7 +856,6 @@ adminApi: # -- Grace period to allow the admin-api to shutdown before it is killed terminationGracePeriodSeconds: 60 - ###################################################################################################################### # # Gateway and Ingress @@ -1175,7 +1179,6 @@ ingress: # - loki.example.com # secretName: loki-distributed-tls - ###################################################################################################################### # # Migration @@ -1192,7 +1195,6 @@ migrate: # memberlist service DNS so the new deployment can join its ring. memberlistService: "" - ###################################################################################################################### # # Single Binary Deployment