From 0c847cdbc94c112d19ad17de1f0f5b34293c0c85 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Mon, 6 Jan 2020 09:58:01 -0500 Subject: [PATCH 1/3] positions.ignore-corruptions --- CHANGELOG.md | 4 +++ docs/clients/promtail/configuration.md | 3 ++ pkg/promtail/positions/positions.go | 24 ++++++++++----- pkg/promtail/positions/positions_test.go | 39 ++++++++++++++++++++++-- 4 files changed, 60 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0232fb1539c1..a2ac301396d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## master / unreleased + +* [FEATURE] promtail positions file corruptions can be ignored with the `positions.ignore-corruptions` flag. In the case the positions yaml is corrupted an empty positions config will be used and should later overwrite the malformed yaml. + # 1.2.0 (2019-12-09) One week has passed since the last Loki release, and it's time for a new one! diff --git a/docs/clients/promtail/configuration.md b/docs/clients/promtail/configuration.md index 1d042e824e4b..78a963b3d4a0 100644 --- a/docs/clients/promtail/configuration.md +++ b/docs/clients/promtail/configuration.md @@ -226,6 +226,9 @@ is restarted to allow it to continue from where it left off. # How often to update the positions file [sync_period: | default = 10s] + +# Whether to ignore & later overwrite positions files that are corrupted +[ignore_corruptions: | default = false] ``` ## scrape_config diff --git a/pkg/promtail/positions/positions.go b/pkg/promtail/positions/positions.go index 1b3201af9d58..ae67c6df8ec9 100644 --- a/pkg/promtail/positions/positions.go +++ b/pkg/promtail/positions/positions.go @@ -20,14 +20,16 @@ const positionFileMode = 0600 // Config describes where to get postition information from. type Config struct { - SyncPeriod time.Duration `yaml:"sync_period"` - PositionsFile string `yaml:"filename"` + SyncPeriod time.Duration `yaml:"sync_period"` + PositionsFile string `yaml:"filename"` + IgnoreCorruptions bool `yaml:"ignore_corruptions"` } // RegisterFlags register flags. func (cfg *Config) RegisterFlags(flags *flag.FlagSet) { flags.DurationVar(&cfg.SyncPeriod, "positions.sync-period", 10*time.Second, "Period with this to sync the position file.") flag.StringVar(&cfg.PositionsFile, "positions.file", "/var/log/positions.yaml", "Location to read/write positions from.") + flag.BoolVar(&cfg.IgnoreCorruptions, "positions.ignore-corruptions", false, "whether to ignore & later overwrite positions files that are corrupted") } // Positions tracks how far through each file we've read. @@ -47,7 +49,7 @@ type File struct { // New makes a new Positions. func New(logger log.Logger, cfg Config) (*Positions, error) { - positions, err := readPositionsFile(cfg.PositionsFile) + positions, err := readPositionsFile(cfg, logger) if err != nil { return nil, err } @@ -181,8 +183,9 @@ func (p *Positions) cleanup() { } } -func readPositionsFile(filename string) (map[string]string, error) { - cleanfn := filepath.Clean(filename) +func readPositionsFile(cfg Config, logger log.Logger) (map[string]string, error) { + + cleanfn := filepath.Clean(cfg.PositionsFile) buf, err := ioutil.ReadFile(cleanfn) if err != nil { if os.IsNotExist(err) { @@ -192,8 +195,15 @@ func readPositionsFile(filename string) (map[string]string, error) { } var p File - if err := yaml.UnmarshalStrict(buf, &p); err != nil { - return nil, fmt.Errorf("%s: %v", cleanfn, err) + err = yaml.UnmarshalStrict(buf, &p) + if err != nil { + // return empty if cfg option enabled + if cfg.IgnoreCorruptions { + level.Debug(logger).Log("msg", "ignoring corrupted positions file", "file", cleanfn, "error", err) + return map[string]string{}, nil + } + + return nil, fmt.Errorf("invalid yaml positions file [%s]: %v", cleanfn, err) } return p.Positions, nil diff --git a/pkg/promtail/positions/positions_test.go b/pkg/promtail/positions/positions_test.go index 866d624a6f85..5d17df3c0ef8 100644 --- a/pkg/promtail/positions/positions_test.go +++ b/pkg/promtail/positions/positions_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/go-kit/kit/log" "github.com/stretchr/testify/require" ) @@ -44,7 +45,10 @@ func TestReadPositionsOK(t *testing.T) { t.Fatal(err) } - pos, err := readPositionsFile(temp) + pos, err := readPositionsFile(Config{ + PositionsFile: temp, + }, log.NewNopLogger()) + require.NoError(t, err) require.Equal(t, "17623", pos["/tmp/random.log"]) } @@ -60,7 +64,10 @@ func TestReadPositionsFromDir(t *testing.T) { _ = os.Remove(temp) }() - _, err = readPositionsFile(temp) + _, err = readPositionsFile(Config{ + PositionsFile: temp, + }, log.NewNopLogger()) + require.Error(t, err) require.True(t, strings.Contains(err.Error(), temp)) // error must contain filename } @@ -79,7 +86,33 @@ func TestReadPositionsFromBadYaml(t *testing.T) { t.Fatal(err) } - _, err = readPositionsFile(temp) + _, err = readPositionsFile(Config{ + PositionsFile: temp, + }, log.NewNopLogger()) + require.Error(t, err) require.True(t, strings.Contains(err.Error(), temp)) // error must contain filename } + +func TestReadPositionsFromBadYamlIgnoreCorruption(t *testing.T) { + temp := tempFilename(t) + defer func() { + _ = os.Remove(temp) + }() + + badYaml := []byte(`positions: + /tmp/random.log: "176 +`) + err := ioutil.WriteFile(temp, badYaml, 0644) + if err != nil { + t.Fatal(err) + } + + out, err := readPositionsFile(Config{ + PositionsFile: temp, + IgnoreCorruptions: true, + }, log.NewNopLogger()) + + require.NoError(t, err) + require.Equal(t, map[string]string{}, out) +} From 20a756bdac676afbb9cad611abe024aba92f86cd Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Mon, 6 Jan 2020 16:07:07 -0500 Subject: [PATCH 2/3] semantic change: s/ignore-corruptions/ignore-invalid-yaml/ --- CHANGELOG.md | 2 +- docs/clients/promtail/configuration.md | 2 +- pkg/promtail/positions/positions.go | 6 +++--- pkg/promtail/positions/positions_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2ac301396d5..8b6eb147280b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## master / unreleased -* [FEATURE] promtail positions file corruptions can be ignored with the `positions.ignore-corruptions` flag. In the case the positions yaml is corrupted an empty positions config will be used and should later overwrite the malformed yaml. +* [FEATURE] promtail positions file corruptions can be ignored with the `positions.ignore-invalid-yaml` flag. In the case the positions yaml is corrupted an empty positions config will be used and should later overwrite the malformed yaml. # 1.2.0 (2019-12-09) diff --git a/docs/clients/promtail/configuration.md b/docs/clients/promtail/configuration.md index 78a963b3d4a0..3ffc9cbadbe3 100644 --- a/docs/clients/promtail/configuration.md +++ b/docs/clients/promtail/configuration.md @@ -228,7 +228,7 @@ is restarted to allow it to continue from where it left off. [sync_period: | default = 10s] # Whether to ignore & later overwrite positions files that are corrupted -[ignore_corruptions: | default = false] +[ignore_invalid_yaml: | default = false] ``` ## scrape_config diff --git a/pkg/promtail/positions/positions.go b/pkg/promtail/positions/positions.go index ae67c6df8ec9..9e03d9b54839 100644 --- a/pkg/promtail/positions/positions.go +++ b/pkg/promtail/positions/positions.go @@ -22,14 +22,14 @@ const positionFileMode = 0600 type Config struct { SyncPeriod time.Duration `yaml:"sync_period"` PositionsFile string `yaml:"filename"` - IgnoreCorruptions bool `yaml:"ignore_corruptions"` + IgnoreInvalidYaml bool `yaml:"ignore_invalid_yaml"` } // RegisterFlags register flags. func (cfg *Config) RegisterFlags(flags *flag.FlagSet) { flags.DurationVar(&cfg.SyncPeriod, "positions.sync-period", 10*time.Second, "Period with this to sync the position file.") flag.StringVar(&cfg.PositionsFile, "positions.file", "/var/log/positions.yaml", "Location to read/write positions from.") - flag.BoolVar(&cfg.IgnoreCorruptions, "positions.ignore-corruptions", false, "whether to ignore & later overwrite positions files that are corrupted") + flag.BoolVar(&cfg.IgnoreInvalidYaml, "positions.ignore-invalid-yaml", false, "whether to ignore & later overwrite positions files that are corrupted") } // Positions tracks how far through each file we've read. @@ -198,7 +198,7 @@ func readPositionsFile(cfg Config, logger log.Logger) (map[string]string, error) err = yaml.UnmarshalStrict(buf, &p) if err != nil { // return empty if cfg option enabled - if cfg.IgnoreCorruptions { + if cfg.IgnoreInvalidYaml { level.Debug(logger).Log("msg", "ignoring corrupted positions file", "file", cleanfn, "error", err) return map[string]string{}, nil } diff --git a/pkg/promtail/positions/positions_test.go b/pkg/promtail/positions/positions_test.go index 5d17df3c0ef8..39f3621bdf5e 100644 --- a/pkg/promtail/positions/positions_test.go +++ b/pkg/promtail/positions/positions_test.go @@ -110,7 +110,7 @@ func TestReadPositionsFromBadYamlIgnoreCorruption(t *testing.T) { out, err := readPositionsFile(Config{ PositionsFile: temp, - IgnoreCorruptions: true, + IgnoreInvalidYaml: true, }, log.NewNopLogger()) require.NoError(t, err) From 36b467f67d6079f7ab44605c1c93d5c6be9971d3 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Mon, 6 Jan 2020 16:31:00 -0500 Subject: [PATCH 3/3] Update pkg/promtail/positions/positions.go Co-Authored-By: Robert Fratto --- pkg/promtail/positions/positions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/promtail/positions/positions.go b/pkg/promtail/positions/positions.go index 9e03d9b54839..1dac0b7f1de0 100644 --- a/pkg/promtail/positions/positions.go +++ b/pkg/promtail/positions/positions.go @@ -199,7 +199,7 @@ func readPositionsFile(cfg Config, logger log.Logger) (map[string]string, error) if err != nil { // return empty if cfg option enabled if cfg.IgnoreInvalidYaml { - level.Debug(logger).Log("msg", "ignoring corrupted positions file", "file", cleanfn, "error", err) + level.Debug(logger).Log("msg", "ignoring invalid positions file", "file", cleanfn, "error", err) return map[string]string{}, nil }