From 2f7f746894f2300de6407c88adebb7d09d21727b Mon Sep 17 00:00:00 2001 From: sbaier Date: Tue, 26 Jan 2021 20:32:00 +0100 Subject: [PATCH 1/2] fix fluent-bit output plugin generating invalid JSON --- cmd/fluent-bit/loki.go | 2 +- cmd/fluent-bit/loki_test.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/fluent-bit/loki.go b/cmd/fluent-bit/loki.go index 4b0d62102325..ad59b3167c58 100644 --- a/cmd/fluent-bit/loki.go +++ b/cmd/fluent-bit/loki.go @@ -224,7 +224,7 @@ func createLine(records map[string]interface{}, f format) (string, error) { if err != nil { return "", err } - return lineReplacer.Replace(string(js)), nil + return string(js), nil case kvPairFormat: buf := &bytes.Buffer{} enc := logfmt.NewEncoder(buf) diff --git a/cmd/fluent-bit/loki_test.go b/cmd/fluent-bit/loki_test.go index a2754669e849..e5c1e0e83a50 100644 --- a/cmd/fluent-bit/loki_test.go +++ b/cmd/fluent-bit/loki_test.go @@ -52,6 +52,15 @@ func Test_loki_sendRecord(t *testing.T) { }, }, } + var nestedJsonFixture = map[interface{}]interface{}{ + "kubernetes": map[interface{}]interface{}{ + "annotations": map[interface{}]interface{}{ + "kubernetes.io/psp": "test", + "prometheus.io/port": "8085", + }, + }, + "log": "\tstatus code: 403, request id: b41c1ffa-c586-4359-a7da-457dd8da4bad\n", + } tests := []struct { name string @@ -71,6 +80,7 @@ func Test_loki_sendRecord(t *testing.T) { {"labelmap", &config{labelMap: map[string]interface{}{"bar": "other"}, lineFormat: jsonFormat, removeKeys: []string{"bar", "error"}}, simpleRecordFixture, []api.Entry{{Labels: model.LabelSet{"other": "500"}, Entry: logproto.Entry{Line: `{"foo":"bar"}`, Timestamp: now}}}, false}, {"byte array", &config{labelKeys: []string{"label"}, lineFormat: jsonFormat}, byteArrayRecordFixture, []api.Entry{{Labels: model.LabelSet{"label": "label"}, Entry: logproto.Entry{Line: `{"map":{"inner":"bar"},"outer":"foo"}`, Timestamp: now}}}, false}, {"mixed types", &config{labelKeys: []string{"label"}, lineFormat: jsonFormat}, mixedTypesRecordFixture, []api.Entry{{Labels: model.LabelSet{"label": "label"}, Entry: logproto.Entry{Line: `{"array":[42,42.42,"foo"],"float":42.42,"int":42,"map":{"nested":{"foo":"bar","invalid":"a\ufffdz"}}}`, Timestamp: now}}}, false}, + {"JSON inner string escaping", &config{removeKeys: []string{"kubernetes"}, labelMap: map[string]interface{}{"kubernetes": map[string]interface{}{"annotations": map[string]interface{}{"kubernetes.io/psp": "label"}}}, lineFormat: jsonFormat}, nestedJsonFixture, []api.Entry{{Labels: model.LabelSet{"label": "test"}, Entry: logproto.Entry{Line: `{"log":"\tstatus code: 403, request id: b41c1ffa-c586-4359-a7da-457dd8da4bad\n"}`, Timestamp: now}}}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 52473b8083008b9b12a8f9a1ed73c0b5b92ade99 Mon Sep 17 00:00:00 2001 From: sbaier Date: Tue, 26 Jan 2021 20:51:43 +0100 Subject: [PATCH 2/2] fix golint error --- cmd/fluent-bit/loki_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/fluent-bit/loki_test.go b/cmd/fluent-bit/loki_test.go index e5c1e0e83a50..cf26ebb2897d 100644 --- a/cmd/fluent-bit/loki_test.go +++ b/cmd/fluent-bit/loki_test.go @@ -52,7 +52,7 @@ func Test_loki_sendRecord(t *testing.T) { }, }, } - var nestedJsonFixture = map[interface{}]interface{}{ + var nestedJSONFixture = map[interface{}]interface{}{ "kubernetes": map[interface{}]interface{}{ "annotations": map[interface{}]interface{}{ "kubernetes.io/psp": "test", @@ -80,7 +80,7 @@ func Test_loki_sendRecord(t *testing.T) { {"labelmap", &config{labelMap: map[string]interface{}{"bar": "other"}, lineFormat: jsonFormat, removeKeys: []string{"bar", "error"}}, simpleRecordFixture, []api.Entry{{Labels: model.LabelSet{"other": "500"}, Entry: logproto.Entry{Line: `{"foo":"bar"}`, Timestamp: now}}}, false}, {"byte array", &config{labelKeys: []string{"label"}, lineFormat: jsonFormat}, byteArrayRecordFixture, []api.Entry{{Labels: model.LabelSet{"label": "label"}, Entry: logproto.Entry{Line: `{"map":{"inner":"bar"},"outer":"foo"}`, Timestamp: now}}}, false}, {"mixed types", &config{labelKeys: []string{"label"}, lineFormat: jsonFormat}, mixedTypesRecordFixture, []api.Entry{{Labels: model.LabelSet{"label": "label"}, Entry: logproto.Entry{Line: `{"array":[42,42.42,"foo"],"float":42.42,"int":42,"map":{"nested":{"foo":"bar","invalid":"a\ufffdz"}}}`, Timestamp: now}}}, false}, - {"JSON inner string escaping", &config{removeKeys: []string{"kubernetes"}, labelMap: map[string]interface{}{"kubernetes": map[string]interface{}{"annotations": map[string]interface{}{"kubernetes.io/psp": "label"}}}, lineFormat: jsonFormat}, nestedJsonFixture, []api.Entry{{Labels: model.LabelSet{"label": "test"}, Entry: logproto.Entry{Line: `{"log":"\tstatus code: 403, request id: b41c1ffa-c586-4359-a7da-457dd8da4bad\n"}`, Timestamp: now}}}, false}, + {"JSON inner string escaping", &config{removeKeys: []string{"kubernetes"}, labelMap: map[string]interface{}{"kubernetes": map[string]interface{}{"annotations": map[string]interface{}{"kubernetes.io/psp": "label"}}}, lineFormat: jsonFormat}, nestedJSONFixture, []api.Entry{{Labels: model.LabelSet{"label": "test"}, Entry: logproto.Entry{Line: `{"log":"\tstatus code: 403, request id: b41c1ffa-c586-4359-a7da-457dd8da4bad\n"}`, Timestamp: now}}}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {