Loki: add -print-config-stderr flag to dump loki's runtime config to …

…stderr (#2336)

* Loki: add -print-config-stderr flag to dump loki's runtime config to stderr

* add ability to print inline config for better readability in grafana

* add an option to log the config in reverse order to make viewing in Grafana easier.

Also added this capability to Promtail

cmd/loki/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"reflect"
+	"strings"
 
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
@@ -15,6 +16,7 @@ import (
 	_ "github.com/grafana/loki/pkg/build"
 	"github.com/grafana/loki/pkg/cfg"
 	"github.com/grafana/loki/pkg/loki"
+	logutil "github.com/grafana/loki/pkg/util"
 
 	"github.com/cortexproject/cortex/pkg/util"
 
@@ -25,8 +27,13 @@ func init() {
 	prometheus.MustRegister(version.NewCollector("loki"))
 }
 
+var lineReplacer = strings.NewReplacer("\n", "\\n  ")
+
 func main() {
 	printVersion := flag.Bool("version", false, "Print this builds version information")
+	printConfig := flag.Bool("print-config-stderr", false, "Dump the entire Loki config object to stderr")
+	logConfig := flag.Bool("log-config-reverse-order", false, "Dump the entire Loki config object at Info log "+
+		"level with the order reversed, reversing the order makes viewing the entries easier in Grafana.")
 
 	var config loki.Config
 	if err := cfg.Parse(&config); err != nil {
@@ -58,6 +65,20 @@ func main() {
 		os.Exit(1)
 	}
 
+	if *printConfig {
+		err := logutil.PrintConfig(os.Stderr, &config)
+		if err != nil {
+			level.Error(util.Logger).Log("msg", "failed to print config to stderr", "err", err.Error())
+		}
+	}
+
+	if *logConfig {
+		err := logutil.LogConfig(&config)
+		if err != nil {
+			level.Error(util.Logger).Log("msg", "failed to log config object", "err", err.Error())
+		}
+	}
+
 	if config.Tracing.Enabled {
 		// Setting the environment variable JAEGER_AGENT_HOST enables tracing
 		trace, err := tracing.NewFromEnv(fmt.Sprintf("loki-%s", config.Target))

cmd/promtail/main.go

@@ -19,6 +19,7 @@ import (
 	"github.com/grafana/loki/pkg/logentry/stages"
 	"github.com/grafana/loki/pkg/promtail"
 	"github.com/grafana/loki/pkg/promtail/config"
+	logutil "github.com/grafana/loki/pkg/util"
 )
 
 func init() {
@@ -28,6 +29,9 @@ func init() {
 func main() {
 	printVersion := flag.Bool("version", false, "Print this builds version information")
 	dryRun := flag.Bool("dry-run", false, "Start Promtail but print entries instead of sending them to Loki.")
+	printConfig := flag.Bool("print-config-stderr", false, "Dump the entire Loki config object to stderr")
+	logConfig := flag.Bool("log-config-reverse-order", false, "Dump the entire Loki config object at Info log "+
+		"level with the order reversed, reversing the order makes viewing the entries easier in Grafana.")
 
 	// Load config, merging config file and CLI flags
 	var config config.Config
@@ -58,6 +62,20 @@ func main() {
 		stages.Debug = true
 	}
 
+	if *printConfig {
+		err := logutil.PrintConfig(os.Stderr, &config)
+		if err != nil {
+			level.Error(util.Logger).Log("msg", "failed to print config to stderr", "err", err.Error())
+		}
+	}
+
+	if *logConfig {
+		err := logutil.LogConfig(&config)
+		if err != nil {
+			level.Error(util.Logger).Log("msg", "failed to log config object", "err", err.Error())
+		}
+	}
+
 	p, err := promtail.New(config, *dryRun)
 	if err != nil {
 		level.Error(util.Logger).Log("msg", "error creating promtail", "error", err)

docs/best-practices/current-best-practices.md

@@ -86,3 +86,11 @@ Depending on the compression used (we have been using snappy which has less comp
 Lots of small, unfilled chunks are currently kryptonite for Loki. We are always working to improve this and may consider a compactor to improve this in some situations. But, in general, the guidance should stay about the same: Try your best to fill chunks!
 
 If you have an application that can log fast enough to fill these chunks quickly (much less than `max_chunk_age`), then it becomes more reasonable to use dynamic labels to break that up into separate streams.
+
+## 8. Use `-print-config-stderr` or `-log-config-reverse-order`
+
+Starting in version 1.6.0 Loki and Promtail have flags which will dump the entire config object to stderr, or the log file, when they start.
+
+`-print-config-stderr` is nice when running loki directly e.g. `./loki ` as you can get a quick output of the entire Loki config. 
+
+`-log-config-reverse-order` is the flag we run Loki with in all our environments, the config entries are reversed so that the order of configs reads correctly top to bottom when viewed in Grafana's Explore.

docs/clients/promtail/configuration.md

@@ -4,6 +4,7 @@ Promtail is configured in a YAML file (usually referred to as `config.yaml`)
 which contains information on the Promtail server, where positions are stored,
 and how to scrape logs from files.
 
+* [Printing Promtail Config At Runtime](#printing-promtail-config-at-runtime)
 * [Configuration File Reference](#configuration-file-reference)
 * [server_config](#server_config)
 * [client_config](#client_config)
@@ -37,6 +38,25 @@ and how to scrape logs from files.
 * [Example Journal Config](#example-journal-config)
 * [Example Syslog Config](#example-syslog-config)
 
+## Printing Promtail Config At Runtime
+
+If you pass Promtail the flag `-print-config-stderr` or `-log-config-reverse-order`, (or `-print-config-stderr=true`)
+Promtail will dump the entire config object it has created from the built in defaults combined first with
+overrides from config file, and second by overrides from flags.
+
+The result is the value for every config object in the Promtail config struct.
+
+Some values may not be relevant to your install, this is expected as every option has a default value if it is being used or not.
+
+This config is what Promtail will use to run, it can be invaluable for debugging issues related to configuration and
+is especially useful in making sure your config files and flags are being read and loaded properly.
+
+`-print-config-stderr` is nice when running Promtail directly e.g. `./promtail ` as you can get a quick output of the entire Promtail config. 
+
+`-log-config-reverse-order` is the flag we run Promtail with in all our environments, the config entries are reversed so 
+that the order of configs reads correctly top to bottom when viewed in Grafana's Explore.
+
+
 ## Configuration File Reference
 
 To specify which configuration file to load, pass the `-config.file` flag at the

docs/configuration/README.md

@@ -6,6 +6,7 @@ depending on which mode Loki is launched in.
 
 Configuration examples can be found in the [Configuration Examples](examples.md) document.
 
+* [Printing Loki Config At Runtime](#printing-loki-config-at-runtime)
 * [Configuration File Reference](#configuration-file-reference)
 * [server_config](#server_config)
 * [distributor_config](#distributor_config)
@@ -29,6 +30,24 @@ Configuration examples can be found in the [Configuration Examples](examples.md)
 * [tracing_config](#tracing_config)
 * [Runtime Configuration file](#runtime-configuration-file)
 
+## Printing Loki Config At Runtime
+
+If you pass Loki the flag `-print-config-stderr` or `-log-config-reverse-order`, (or `-print-config-stderr=true`)
+Loki will dump the entire config object it has created from the built in defaults combined first with
+overrides from config file, and second by overrides from flags.
+
+The result is the value for every config object in the Loki config struct, which is very large...
+
+Many values will not be relevant to your install such as storage configs which you are not using and which you did not define, 
+this is expected as every option has a default value if it is being used or not.
+
+This config is what Loki will use to run, it can be invaluable for debugging issues related to configuration and
+is especially useful in making sure your config files and flags are being read and loaded properly.
+
+`-print-config-stderr` is nice when running Loki directly e.g. `./loki ` as you can get a quick output of the entire Loki config. 
+
+`-log-config-reverse-order` is the flag we run Loki with in all our environments, the config entries are reversed so 
+that the order of configs reads correctly top to bottom when viewed in Grafana's Explore.
 
 ## Configuration File Reference
 

pkg/util/config.go

@@ -0,0 +1,39 @@
+package util
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/go-kit/kit/log/level"
+	"github.com/prometheus/common/version"
+	"gopkg.in/yaml.v2"
+)
+
+// LogConfig takes a pointer to a config object, marshalls it to YAML and prints each line in REVERSE order
+// The reverse order makes display in Grafana in easier which typically sorts newest entries at the top.
+func LogConfig(cfg interface{}) error {
+	lc, err := yaml.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	cfgStr := string(lc)
+	cfgStrs := strings.Split(cfgStr, "\n")
+	for i := len(cfgStrs) - 1; i >= 0; i-- {
+		level.Info(util.Logger).Log("type", "config", "msg", cfgStrs[i])
+	}
+	return nil
+}
+
+// PrintConfig will takes a pointer to a config object, marshalls it to YAML and prints the result to the provided writer
+// unlike LogConfig, PrintConfig prints the object in naturally ocurring order.
+func PrintConfig(w io.Writer, config interface{}) error {
+	lc, err := yaml.Marshal(config)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(w, "---\n# Loki Config\n# %s\n%s\n\n", version.Info(), string(lc))
+	return nil
+}