Allow to define custom parsers to use with fluentbit.io/parser annota…

…tion (grafana#1430) Signed-off-by: Olivier Boudet <oboudet@gmail.com>
mraboosk · Jan 10, 2020 · e0943d9 · e0943d9
1 parent 88d0519
commit e0943d9
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 1 deletion.
diff --git a/production/helm/fluent-bit/Chart.yaml b/production/helm/fluent-bit/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: "v1"
 name: fluent-bit
-version: 0.0.2
+version: 0.0.3
 appVersion: v0.0.1
 kubeVersion: "^1.10.0-0"
 description: "Uses fluent-bit Loki go plugin for gathering logs and sending them to Loki"

diff --git a/production/helm/fluent-bit/README.md b/production/helm/fluent-bit/README.md
@@ -79,9 +79,11 @@ For more details, read the [Fluent Bit documentation](../../../cmd/fluent-bit/RE
 | `config.port`            | the Fluent Bit port to listen. (This is mainly used to serve metrics)                              | `2020`                           |
 | `config.loglevel`        | the Fluent Bit log level (debug,info,warn,error).                                                  | `warn`                           |
 | `config.lineFormat`      | The line format to use to send a record (json/key_value)                                           | `json`                           |
+| `config.k8sLoggingParser`| Allow Kubernetes Pods to suggest a pre-defined Parser. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/kubernetes#kubernetes-annotations).                                                                                      | `Off`                           |
 | `config.removeKeys`      | The list of key to remove from each record                                                         | `[removeKeys,stream]`            |
 | `config.labels`          | A set of labels to send for every log                                                              | `'{job="fluent-bit"}'`           |
 | `config.labelMap`        | Mapping of labels from a record. See [Fluent Bit documentation](../../../cmd/fluent-bit/README.md) |                                  |
+| `config.parsers`         | Definition of extras fluent bit parsers. See [Official Fluent Bit documentation](https://docs.fluentbit.io/manual/filter/parser). The format is a sequence of mappings where each key is the same as the one in the [PARSER] section of parsers.conf file       | `[]`                            |
 | `affinity`               | [affinity][affinity] settings for pod assignment                                                   | `{}`                             |
 | `annotations`            | Annotations to add to Kubernetes resources.                                                        | `{}`                             |
 | `deploymentStrategy`     | The deployment strategy to use with the daemonset                                                  | `RollingUpdate`                  |

diff --git a/production/helm/fluent-bit/templates/configmap.yaml b/production/helm/fluent-bit/templates/configmap.yaml
@@ -27,6 +27,7 @@ data:
         Match          kube.*
         Kube_URL       https://kubernetes.default.svc:443
         Merge_Log On
+        K8S-Logging.Parser {{ .Values.config.k8sLoggingParser }}
     [Output]
         Name loki
         Match *
@@ -47,6 +48,13 @@ data:
         Format      json
         Time_Key    time
         Time_Format %Y-%m-%dT%H:%M:%S.%L
+    {{- range $parser:= .Values.config.parsers }}
+    [PARSER]
+    {{- range $key,$value := $parser }}
+        {{ $key }} = {{ $value }}
+    {{- end }}
+    {{- end }}
 
   labelmap.json: |-
     {{- .Values.config.labelMap | toPrettyJson | nindent 4}}
+
diff --git a/production/helm/fluent-bit/values.yaml b/production/helm/fluent-bit/values.yaml
@@ -10,6 +10,7 @@ config:
   port: 2020
   loglevel: warn
   lineFormat: json
+  k8sLoggingParser: "Off"
   removeKeys:
     - kubernetes
     - stream
@@ -24,6 +25,12 @@ config:
       container_name: container
       pod_name: instance
     stream: stream
+  # parsers: # Allow to define custom parsers. The key here is the same as the one in the [PARSER] section of parsers.conf file.
+  #  - Name: json
+  #    Format: json
+  #    Time_Key: time
+  #    Time_Format: "%d/%b/%Y:%H:%M:%S %z"
+
 
 affinity: {}