From 171a09b82236333ef0bbd4db546d16bee01d58d2 Mon Sep 17 00:00:00 2001 From: carlospeon Date: Fri, 12 Aug 2022 17:44:54 +0200 Subject: [PATCH] Add support for journal matches (only conjuntions) (#6656) --- CHANGELOG.md | 1 + .../pkg/promtail/scrapeconfig/scrapeconfig.go | 4 +++ .../promtail/targets/journal/journaltarget.go | 20 +++++++++-- .../targets/journal/journaltarget_test.go | 35 +++++++++++++++++++ docs/sources/clients/promtail/scraping.md | 5 ++- 5 files changed, 62 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 162058d6c37b..41c445661fe8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ ##### Enhancements * [6395](https://github.com/grafana/loki/pull/6395) **DylanGuedes**: Add encoding support * [6828](https://github.com/grafana/loki/pull/6828) **alexandre1984rj** Add the BotScore and BotScoreSrc fields once the Cloudflare API returns those two fields on the list of all available log fields. +* [6656](https://github.com/grafana/loki/pull/6656) **carlospeon**: Allow promtail to add matches to the journal reader ##### Fixes * [6766](https://github.com/grafana/loki/pull/6766) **kavirajk**: fix(logql): Make `LabelSampleExtractor` ignore processing the line if it doesn't contain that specific label. Fixes unwrap behavior explained in the issue https://github.com/grafana/loki/issues/6713 diff --git a/clients/pkg/promtail/scrapeconfig/scrapeconfig.go b/clients/pkg/promtail/scrapeconfig/scrapeconfig.go index aa1b6cb004ce..0f7d96a45b9e 100644 --- a/clients/pkg/promtail/scrapeconfig/scrapeconfig.go +++ b/clients/pkg/promtail/scrapeconfig/scrapeconfig.go @@ -160,6 +160,10 @@ type JournalTargetConfig struct { // Path to a directory to read journal entries from. Defaults to system path // if empty. Path string `yaml:"path"` + + // Journal matches to filter. Character (+) is not supported, only logical AND + // matches will be added. + Matches string `yaml:"matches"` } // SyslogTargetConfig describes a scrape config that listens for log lines over syslog. diff --git a/clients/pkg/promtail/targets/journal/journaltarget.go b/clients/pkg/promtail/targets/journal/journaltarget.go index 32ca82c5e27b..33ccee896fca 100644 --- a/clients/pkg/promtail/targets/journal/journaltarget.go +++ b/clients/pkg/promtail/targets/journal/journaltarget.go @@ -174,12 +174,26 @@ func journalTargetWithReader( return nil, errors.Wrap(err, "parsing journal reader 'max_age' config value") } - cfg := t.generateJournalConfig(journalConfigBuilder{ + cb := journalConfigBuilder{ JournalPath: targetConfig.Path, Position: position, MaxAge: maxAge, EntryFunc: entryFunc, - }) + } + + matches := strings.Fields(targetConfig.Matches) + for _, m := range matches { + fv := strings.Split(m, "=") + if len(fv) != 2 { + return nil, errors.New("Error parsing journal reader 'matches' config value") + } + cb.Matches = append(cb.Matches, sdjournal.Match{ + Field: fv[0], + Value: fv[1], + }) + } + + cfg := t.generateJournalConfig(cb) t.r, err = readerFunc(cfg) if err != nil { return nil, errors.Wrap(err, "creating journal reader") @@ -208,6 +222,7 @@ func journalTargetWithReader( type journalConfigBuilder struct { JournalPath string Position string + Matches []sdjournal.Match MaxAge time.Duration EntryFunc journalEntryFunc } @@ -221,6 +236,7 @@ func (t *JournalTarget) generateJournalConfig( cfg := sdjournal.JournalReaderConfig{ Path: cb.JournalPath, + Matches: cb.Matches, Formatter: t.formatter, } diff --git a/clients/pkg/promtail/targets/journal/journaltarget_test.go b/clients/pkg/promtail/targets/journal/journaltarget_test.go index 68ecab6e3cd3..14d7c28fc512 100644 --- a/clients/pkg/promtail/targets/journal/journaltarget_test.go +++ b/clients/pkg/promtail/targets/journal/journaltarget_test.go @@ -385,3 +385,38 @@ func Test_MakeJournalFields(t *testing.T) { } assert.Equal(t, expectedFields, receivedFields) } + +func TestJournalTarget_Matches(t *testing.T) { + w := log.NewSyncWriter(os.Stderr) + logger := log.NewLogfmtLogger(w) + + testutils.InitRandom() + dirName := "/tmp/" + testutils.RandName() + positionsFileName := dirName + "/positions.yml" + + // Set the sync period to a really long value, to guarantee the sync timer + // never runs, this way we know everything saved was done through channel + // notifications when target.stop() was called. + ps, err := positions.New(logger, positions.Config{ + SyncPeriod: 10 * time.Second, + PositionsFile: positionsFileName, + }) + if err != nil { + t.Fatal(err) + } + + client := fake.New(func() {}) + + cfg := scrapeconfig.JournalTargetConfig{ + Matches: "UNIT=foo.service PRIORITY=1", + } + + jt, err := journalTargetWithReader(NewMetrics(prometheus.NewRegistry()), logger, client, ps, "test", nil, + &cfg, newMockJournalReader, newMockJournalEntry(nil)) + require.NoError(t, err) + + r := jt.r.(*mockJournalReader) + matches := []sdjournal.Match{{Field: "UNIT", Value: "foo.service"}, {Field: "PRIORITY", Value: "1"}} + require.Equal(t, r.config.Matches, matches) + client.Stop() +} diff --git a/docs/sources/clients/promtail/scraping.md b/docs/sources/clients/promtail/scraping.md index e938c4d9f1a9..58508dc8d5d0 100644 --- a/docs/sources/clients/promtail/scraping.md +++ b/docs/sources/clients/promtail/scraping.md @@ -97,6 +97,7 @@ scrape_configs: json: false max_age: 12h path: /var/log/journal + matches: _TRANSPORT=kernel labels: job: systemd-journal relabel_configs: @@ -109,7 +110,9 @@ here for reference. The `max_age` field ensures that no older entry than the time specified will be sent to Loki; this circumvents "entry too old" errors. The `path` field tells Promtail where to read journal entries from. The labels map defines a constant list of labels to add to every journal entry that Promtail -reads. +reads. The `matches` field adds journal filters. If multiple filters are specified +matching different fields, the log entries are filtered by both, if two filters +apply to the same field, then they are automatically matched as alternatives. When the `json` field is set to `true`, messages from the journal will be passed through the pipeline as JSON, keeping all of the original fields from the