From 5031e7a49cd2be831365436ba3b2f9da7ccaaead Mon Sep 17 00:00:00 2001
From: Robert Fratto <robert.fratto@grafana.com>
Date: Tue, 18 Feb 2020 09:18:21 -0500
Subject: [PATCH 1/6] clarify logcli commands and output

This commit attempts to do two things:

1. Differentiate between log entries and raw log lines, where the latter
   is what is produced by `-oraw`.
2. Explain the difference between `logcli query` and
   `logcli instant-query`.

Fixes #1676.
---
 cmd/logcli/main.go             |  24 +++++--
 docs/getting-started/logcli.md | 114 +++++++++++++++++++++++++++------
 2 files changed, 114 insertions(+), 24 deletions(-)

diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go
index 7dab2540f101..afb61775f0e9 100644
--- a/cmd/logcli/main.go
+++ b/cmd/logcli/main.go
@@ -20,20 +20,36 @@ import (
 
 var (
 	app        = kingpin.New("logcli", "A command-line for loki.").Version(version.Print("logcli"))
-	quiet      = app.Flag("quiet", "suppress everything but log lines").Default("false").Short('q').Bool()
+	quiet      = app.Flag("quiet", "suppress everything but log entries").Default("false").Short('q').Bool()
 	statistics = app.Flag("stats", "show query statistics").Default("false").Bool()
 	outputMode = app.Flag("output", "specify output mode [default, raw, jsonl]").Default("default").Short('o').Enum("default", "raw", "jsonl")
 	timezone   = app.Flag("timezone", "Specify the timezone to use when formatting output timestamps [Local, UTC]").Default("Local").Short('z').Enum("Local", "UTC")
 
 	queryClient = newQueryClient(app)
 
-	queryCmd   = app.Command("query", "Run a LogQL query.")
+	queryCmd = app.Command("query", `Run a LogQL query.
+
+The default output of this command are log entries (combination of
+timestamp, labels, and log line) along with metainformation about the query
+made to Loki.  The metainformation can be filtered out using the --quiet
+flag. Raw log lines (i.e., no labels or timestamp) can be retrieved using
+-oraw.
+
+When running a metrics query, this command outputs multiple data points
+between the start and the end query time. This produces values that are
+used to build graphs. If you just want a single data point (i.e., the
+Grafana explore "table"), then you should use instant-query instead.`)
 	rangeQuery = newQuery(false, queryCmd)
 	tail       = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool()
 	delayFor   = queryCmd.Flag("delay-for", "Delay in tailing by number of seconds to accumulate logs for re-ordering").Default("0").Int()
 
-	instantQueryCmd = app.Command("instant-query", "Run an instant LogQL query")
-	instantQuery    = newQuery(true, instantQueryCmd)
+	instantQueryCmd = app.Command("instant-query", `Run an instant LogQL query.
+
+This query type can only be used for metrics queries, where the query is
+evaluated for a single point in time. This is equivalent to the Grafana
+explore "table" view; if you want data that is used to build the Grafana
+graph, you should use query instead.`)
+	instantQuery = newQuery(true, instantQueryCmd)
 
 	labelsCmd = app.Command("labels", "Find values for a given label.")
 	labelName = labelsCmd.Arg("label", "The name of the label.").HintAction(hintActionLabelNames).String()
diff --git a/docs/getting-started/logcli.md b/docs/getting-started/logcli.md
index a5153831c2e7..4828bc7feb92 100644
--- a/docs/getting-started/logcli.md
+++ b/docs/getting-started/logcli.md
@@ -72,53 +72,128 @@ A command-line for loki.
 
 Flags:
       --help             Show context-sensitive help (also try --help-long and --help-man).
-  -q, --quiet            suppress everything but log lines
+      --version          Show application version.
+  -q, --quiet            suppress everything but log entries
+      --stats            show query statistics
   -o, --output=default   specify output mode [default, raw, jsonl]
-      --addr="https://logs-us-west1.grafana.net"
-                         Server address.
-      --username=""      Username for HTTP basic auth.
-      --password=""      Password for HTTP basic auth.
-      --ca-cert=""       Path to the server Certificate Authority.
+  -z, --timezone=Local   Specify the timezone to use when formatting output timestamps [Local, UTC]
+      --addr="http://localhost:3100"
+                         Server address. Can also be set using LOKI_ADDR env var.
+      --username=""      Username for HTTP basic auth. Can also be set using LOKI_USERNAME env var.
+      --password=""      Password for HTTP basic auth. Can also be set using LOKI_PASSWORD env var.
+      --ca-cert=""       Path to the server Certificate Authority. Can also be set using LOKI_CA_CERT_PATH env var.
       --tls-skip-verify  Server certificate TLS skip verify.
-      --cert=""          Path to the client certificate.
-      --key=""           Path to the client certificate key.
+      --cert=""          Path to the client certificate. Can also be set using LOKI_CLIENT_CERT_PATH env var.
+      --key=""           Path to the client certificate key. Can also be set using LOKI_CLIENT_KEY_PATH env var.
       --org-id=ORG-ID    org ID header to be substituted for auth
 
 Commands:
   help [<command>...]
     Show help.
 
-  query [<flags>] <query> [<regex>]
+  query [<flags>] <query>
     Run a LogQL query.
 
+    The default output of this command are log entries (combination of
+    timestamp, labels, and log line) along with metainformation about the query
+    made to Loki. The metainformation can be filtered out using the --quiet
+    flag. Raw log lines (i.e., no labels or timestamp) can be retrieved using
+    -oraw.
+
+    When running a metrics query, this command outputs multiple data points
+    between the start and the end query time.  This produces values that are
+    used to build graphs. If you just want a single data point (i.e., the
+    Grafana explore "table"), then you should use instant-query instead.
+
+  instant-query [<flags>] <query>
+    Run an instant LogQL query.
+
+    This query type can only be used for metrics queries, where the query is
+    evaluated for a single point in time. This is equivalent to the Grafana
+    explore "table" view; if you want data that is used to build the Grafana
+    graph, you should use query instead.
+
   labels [<label>]
     Find values for a given label.
 
 $ logcli help query
-usage: logcli query [<flags>] <query> [<regex>]
+usage: logcli query [<flags>] <query>
 
 Run a LogQL query.
 
+The default output of this command are log entries (combination of timestamp,
+labels, and log line) along with metainformation about the query made to Loki.
+The metainformation can be filtered out using the --quiet flag. Raw log lines
+(i.e., no labels or timestamp) can be retrieved using -oraw.
+
+When running a metrics query, this command outputs multiple data points between
+the start and the end query time. This produces values that are used to build
+graphs. If you just want a single data point (i.e., the Grafana explore
+"table"), then you should use instant-query instead.
+
 Flags:
       --help             Show context-sensitive help (also try --help-long and --help-man).
-  -q, --quiet            suppress everything but log lines
+      --version          Show application version.
+  -q, --quiet            suppress everything but log entries
+      --stats            show query statistics
   -o, --output=default   specify output mode [default, raw, jsonl]
-      --addr="https://logs-us-west1.grafana.net"
-                         Server address.
-      --username=""      Username for HTTP basic auth.
-      --password=""      Password for HTTP basic auth.
-      --ca-cert=""       Path to the server Certificate Authority.
+  -z, --timezone=Local   Specify the timezone to use when formatting output timestamps [Local, UTC]
+      --addr="http://localhost:3100"
+                         Server address. Can also be set using LOKI_ADDR env var.
+      --username=""      Username for HTTP basic auth. Can also be set using LOKI_USERNAME env var.
+      --password=""      Password for HTTP basic auth. Can also be set using LOKI_PASSWORD env var.
+      --ca-cert=""       Path to the server Certificate Authority. Can also be set using LOKI_CA_CERT_PATH env var.
       --tls-skip-verify  Server certificate TLS skip verify.
-      --cert=""          Path to the client certificate.
-      --key=""           Path to the client certificate key.
+      --cert=""          Path to the client certificate. Can also be set using LOKI_CLIENT_CERT_PATH env var.
+      --key=""           Path to the client certificate key. Can also be set using LOKI_CLIENT_KEY_PATH env var.
+      --org-id=ORG-ID    org ID header to be substituted for auth
       --limit=30         Limit on number of entries to print.
       --since=1h         Lookback window.
       --from=FROM        Start looking for logs at this absolute time (inclusive)
       --to=TO            Stop looking for logs at this absolute time (exclusive)
+      --step=STEP        Query resolution step width
       --forward          Scan forwards through logs.
       --local-config=""  Execute the current query using a configured storage from a given Loki configuration file.
+      --no-labels        Do not print any labels
+      --exclude-label=EXCLUDE-LABEL ...
+                         Exclude labels given the provided key during output.
+      --include-label=INCLUDE-LABEL ...
+                         Include labels given the provided key during output.
+      --labels-length=0  Set a fixed padding to labels
   -t, --tail             Tail the logs
       --delay-for=0      Delay in tailing by number of seconds to accumulate logs for re-ordering
+
+Args:
+  <query>  eg '{foo="bar",baz=~".*blip"} |~ ".*error.*"'
+
+$ logcli help instant-query
+usage: logcli instant-query [<flags>] <query>
+
+Run an instant LogQL query.
+
+This query type can only be used for metrics queries, where the query is evaluated for a single point in time. This is
+equivalent to the Grafana explore "table" view; if you want data that is used to build the Grafana graph, you should
+use query instead.
+
+Flags:
+      --help             Show context-sensitive help (also try --help-long and --help-man).
+      --version          Show application version.
+  -q, --quiet            suppress everything but log entries
+      --stats            show query statistics
+  -o, --output=default   specify output mode [default, raw, jsonl]
+  -z, --timezone=Local   Specify the timezone to use when formatting output timestamps [Local, UTC]
+      --addr="http://localhost:3100"
+                         Server address. Can also be set using LOKI_ADDR env var.
+      --username=""      Username for HTTP basic auth. Can also be set using LOKI_USERNAME env var.
+      --password=""      Password for HTTP basic auth. Can also be set using LOKI_PASSWORD env var.
+      --ca-cert=""       Path to the server Certificate Authority. Can also be set using LOKI_CA_CERT_PATH env var.
+      --tls-skip-verify  Server certificate TLS skip verify.
+      --cert=""          Path to the client certificate. Can also be set using LOKI_CLIENT_CERT_PATH env var.
+      --key=""           Path to the client certificate key. Can also be set using LOKI_CLIENT_KEY_PATH env var.
+      --org-id=ORG-ID    org ID header to be substituted for auth
+      --limit=30         Limit on number of entries to print.
+      --now=NOW          Time at which to execute the instant query.
+      --forward          Scan forwards through logs.
       --no-labels        Do not print any labels
       --exclude-label=EXCLUDE-LABEL ...
                          Exclude labels given the provided key during output.
@@ -127,6 +202,5 @@ Flags:
       --labels-length=0  Set a fixed padding to labels
 
 Args:
-  <query>    eg '{foo="bar",baz="blip"}'
-  [<regex>]
+  <query>  eg '{foo="bar",baz=~".*blip"} |~ ".*error.*"'
 ```

From 16d69ddf233828b597e4dbb6b07a5d5927d15002 Mon Sep 17 00:00:00 2001
From: Robert Fratto <robert.fratto@grafana.com>
Date: Tue, 18 Feb 2020 09:32:24 -0500
Subject: [PATCH 2/6] address feedback

---
 cmd/logcli/main.go | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go
index afb61775f0e9..c29c7ccd87a3 100644
--- a/cmd/logcli/main.go
+++ b/cmd/logcli/main.go
@@ -20,9 +20,9 @@ import (
 
 var (
 	app        = kingpin.New("logcli", "A command-line for loki.").Version(version.Print("logcli"))
-	quiet      = app.Flag("quiet", "suppress everything but log entries").Default("false").Short('q').Bool()
+	quiet      = app.Flag("quiet", "suppress query metadata").Default("false").Short('q').Bool()
 	statistics = app.Flag("stats", "show query statistics").Default("false").Bool()
-	outputMode = app.Flag("output", "specify output mode [default, raw, jsonl]").Default("default").Short('o').Enum("default", "raw", "jsonl")
+	outputMode = app.Flag("output", "specify output mode [default, raw, jsonl]. raw suppresses log labels and timestamp.").Default("default").Short('o').Enum("default", "raw", "jsonl")
 	timezone   = app.Flag("timezone", "Specify the timezone to use when formatting output timestamps [Local, UTC]").Default("Local").Short('z').Enum("Local", "UTC")
 
 	queryClient = newQueryClient(app)
@@ -30,10 +30,11 @@ var (
 	queryCmd = app.Command("query", `Run a LogQL query.
 
 The default output of this command are log entries (combination of
-timestamp, labels, and log line) along with metainformation about the query
-made to Loki.  The metainformation can be filtered out using the --quiet
-flag. Raw log lines (i.e., no labels or timestamp) can be retrieved using
--oraw.
+timestamp, labels, and log line) along with various extra information
+about the query and its results. Raw log lines (i.e., no labels or timestamp)
+can be retrieved using -oraw, while the extra information (URL queried,
+set of common labels, excluded labels) can be suppressed with the
+--quiet flag.
 
 When running a metrics query, this command outputs multiple data points
 between the start and the end query time. This produces values that are

From 13ddf0bcf05540a6a0f6c0071008649b60b1caf5 Mon Sep 17 00:00:00 2001
From: Robert Fratto <robert.fratto@grafana.com>
Date: Tue, 18 Feb 2020 09:45:01 -0500
Subject: [PATCH 3/6] address more feedback

---
 cmd/logcli/main.go             |  4 ++--
 docs/getting-started/logcli.md | 38 +++++++++++++++++++---------------
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go
index c29c7ccd87a3..7843ba512905 100644
--- a/cmd/logcli/main.go
+++ b/cmd/logcli/main.go
@@ -32,8 +32,8 @@ var (
 The default output of this command are log entries (combination of
 timestamp, labels, and log line) along with various extra information
 about the query and its results. Raw log lines (i.e., no labels or timestamp)
-can be retrieved using -oraw, while the extra information (URL queried,
-set of common labels, excluded labels) can be suppressed with the
+can be retrieved by passing the flag "-o raw", while the extra information (URL
+queried, set of common labels, excluded labels) can be suppressed with the
 --quiet flag.
 
 When running a metrics query, this command outputs multiple data points
diff --git a/docs/getting-started/logcli.md b/docs/getting-started/logcli.md
index 4828bc7feb92..b8fa32882cf0 100644
--- a/docs/getting-started/logcli.md
+++ b/docs/getting-started/logcli.md
@@ -64,7 +64,7 @@ Configuration values are considered in the following order (lowest to highest):
 
 ### Details
 
-```bash
+```nohighlight
 $ logcli help
 usage: logcli [<flags>] <command> [<args> ...]
 
@@ -73,9 +73,9 @@ A command-line for loki.
 Flags:
       --help             Show context-sensitive help (also try --help-long and --help-man).
       --version          Show application version.
-  -q, --quiet            suppress everything but log entries
+  -q, --quiet            suppress query metadata
       --stats            show query statistics
-  -o, --output=default   specify output mode [default, raw, jsonl]
+  -o, --output=default   specify output mode [default, raw, jsonl]. raw suppresses log labels and timestamp.
   -z, --timezone=Local   Specify the timezone to use when formatting output timestamps [Local, UTC]
       --addr="http://localhost:3100"
                          Server address. Can also be set using LOKI_ADDR env var.
@@ -95,10 +95,11 @@ Commands:
     Run a LogQL query.
 
     The default output of this command are log entries (combination of
-    timestamp, labels, and log line) along with metainformation about the query
-    made to Loki. The metainformation can be filtered out using the --quiet
-    flag. Raw log lines (i.e., no labels or timestamp) can be retrieved using
-    -oraw.
+    timestamp, labels, and log line) along with various extra information about
+    the query and its results. Raw log lines (i.e., no labels or timestamp) can
+    be retrieved by passing the flag "-o raw", while the extra information (URL
+    queried, set of common labels, excluded labels) can be suppressed with the
+    --quiet flag.
 
     When running a metrics query, this command outputs multiple data points
     between the start and the end query time.  This produces values that are
@@ -122,9 +123,10 @@ usage: logcli query [<flags>] <query>
 Run a LogQL query.
 
 The default output of this command are log entries (combination of timestamp,
-labels, and log line) along with metainformation about the query made to Loki.
-The metainformation can be filtered out using the --quiet flag. Raw log lines
-(i.e., no labels or timestamp) can be retrieved using -oraw.
+labels, and log line) along with various extra information about the query and
+its results. Raw log lines (i.e., no labels or timestamp) can be retrieved by
+passing the flag "-o raw", while the extra information (URL queried, set of
+common labels, excluded labels) can be suppressed with the --quiet flag.
 
 When running a metrics query, this command outputs multiple data points between
 the start and the end query time. This produces values that are used to build
@@ -134,9 +136,9 @@ graphs. If you just want a single data point (i.e., the Grafana explore
 Flags:
       --help             Show context-sensitive help (also try --help-long and --help-man).
       --version          Show application version.
-  -q, --quiet            suppress everything but log entries
+  -q, --quiet            suppress query metadata
       --stats            show query statistics
-  -o, --output=default   specify output mode [default, raw, jsonl]
+  -o, --output=default   specify output mode [default, raw, jsonl]. raw suppresses log labels and timestamp.
   -z, --timezone=Local   Specify the timezone to use when formatting output timestamps [Local, UTC]
       --addr="http://localhost:3100"
                          Server address. Can also be set using LOKI_ADDR env var.
@@ -166,21 +168,23 @@ Flags:
 Args:
   <query>  eg '{foo="bar",baz=~".*blip"} |~ ".*error.*"'
 
+
 $ logcli help instant-query
 usage: logcli instant-query [<flags>] <query>
 
 Run an instant LogQL query.
 
-This query type can only be used for metrics queries, where the query is evaluated for a single point in time. This is
-equivalent to the Grafana explore "table" view; if you want data that is used to build the Grafana graph, you should
-use query instead.
+This query type can only be used for metrics queries, where the query is
+evaluated for a single point in time. This is equivalent to the Grafana explore
+"table" view; if you want data that is used to build the Grafana graph, you
+should use query instead.
 
 Flags:
       --help             Show context-sensitive help (also try --help-long and --help-man).
       --version          Show application version.
-  -q, --quiet            suppress everything but log entries
+  -q, --quiet            suppress query metadata
       --stats            show query statistics
-  -o, --output=default   specify output mode [default, raw, jsonl]
+  -o, --output=default   specify output mode [default, raw, jsonl]. raw suppresses log labels and timestamp.
   -z, --timezone=Local   Specify the timezone to use when formatting output timestamps [Local, UTC]
       --addr="http://localhost:3100"
                          Server address. Can also be set using LOKI_ADDR env var.

From fe898ad566c1c8ed5a0f39a0608c85a3fface9a8 Mon Sep 17 00:00:00 2001
From: Robert Fratto <robert.fratto@grafana.com>
Date: Tue, 18 Feb 2020 11:07:33 -0500
Subject: [PATCH 4/6] explain that instant-query is not useful for log lines

---
 cmd/logcli/main.go             |  39 ++++----
 docs/getting-started/logcli.md | 158 +++++++++++++++++++++------------
 2 files changed, 122 insertions(+), 75 deletions(-)

diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go
index 7843ba512905..f0d43d1c15c9 100644
--- a/cmd/logcli/main.go
+++ b/cmd/logcli/main.go
@@ -29,27 +29,33 @@ var (
 
 	queryCmd = app.Command("query", `Run a LogQL query.
 
-The default output of this command are log entries (combination of
-timestamp, labels, and log line) along with various extra information
-about the query and its results. Raw log lines (i.e., no labels or timestamp)
-can be retrieved by passing the flag "-o raw", while the extra information (URL
-queried, set of common labels, excluded labels) can be suppressed with the
---quiet flag.
-
-When running a metrics query, this command outputs multiple data points
-between the start and the end query time. This produces values that are
-used to build graphs. If you just want a single data point (i.e., the
-Grafana explore "table"), then you should use instant-query instead.`)
+The "query" command is useful for querying for log lines. The default
+output of this command are log entries (a combination of timestamp,
+labels, and a log line) along with various extra information about
+the performed query and its results. Raw log lines (i.e., without a
+label and timestamp) can be retrieved by passing the "-o raw" flag.
+The extra information about the query (API URL, set of common labels,
+excluded labels) can be suppressed with the --query flag.
+
+While "query" does support metrics queries, its output contains multiple
+data points between the start and end query time. This output is used to
+build graphs, like what is seen in the Grafana Explore graph view. If
+you are querying metrics and just want the most recent data point
+(like what is seen in the Grafana Explore table view), then you should use
+the "instant-query" command instead.`)
 	rangeQuery = newQuery(false, queryCmd)
 	tail       = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool()
 	delayFor   = queryCmd.Flag("delay-for", "Delay in tailing by number of seconds to accumulate logs for re-ordering").Default("0").Int()
 
 	instantQueryCmd = app.Command("instant-query", `Run an instant LogQL query.
 
-This query type can only be used for metrics queries, where the query is
-evaluated for a single point in time. This is equivalent to the Grafana
-explore "table" view; if you want data that is used to build the Grafana
-graph, you should use query instead.`)
+The "instant-query" command is useful for evaluating a metric query for
+a single point in time. This is equivalent to the Grafana Explore table
+view; if you want a metrics query that is used to build a Grafana graph,
+you should use the "query" command instead.
+
+This command does not produce useful output when querying for log lines;
+you should always use the "query" command when you are running log queries.`)
 	instantQuery = newQuery(true, instantQueryCmd)
 
 	labelsCmd = app.Command("labels", "Find values for a given label.")
@@ -170,11 +176,12 @@ func newQuery(instant bool, cmd *kingpin.CmdClause) *query.Query {
 		return nil
 	})
 
-	cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'").Required().StringVar(&query.QueryString)
 	cmd.Flag("limit", "Limit on number of entries to print.").Default("30").IntVar(&query.Limit)
 	if instant {
+		cmd.Arg("query", "eg 'rate({foo=\"bar\"} |~ \".*error.*\" [5m])'").Required().StringVar(&query.QueryString)
 		cmd.Flag("now", "Time at which to execute the instant query.").StringVar(&now)
 	} else {
+		cmd.Arg("query", "eg '{foo=\"bar\",baz=~\".*blip\"} |~ \".*error.*\"'").Required().StringVar(&query.QueryString)
 		cmd.Flag("since", "Lookback window.").Default("1h").DurationVar(&since)
 		cmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").StringVar(&from)
 		cmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").StringVar(&to)
diff --git a/docs/getting-started/logcli.md b/docs/getting-started/logcli.md
index b8fa32882cf0..b6530ba78d16 100644
--- a/docs/getting-started/logcli.md
+++ b/docs/getting-started/logcli.md
@@ -71,20 +71,29 @@ usage: logcli [<flags>] <command> [<args> ...]
 A command-line for loki.
 
 Flags:
-      --help             Show context-sensitive help (also try --help-long and --help-man).
+      --help             Show context-sensitive help (also try --help-long and
+                         --help-man).
       --version          Show application version.
   -q, --quiet            suppress query metadata
       --stats            show query statistics
-  -o, --output=default   specify output mode [default, raw, jsonl]. raw suppresses log labels and timestamp.
-  -z, --timezone=Local   Specify the timezone to use when formatting output timestamps [Local, UTC]
+  -o, --output=default   specify output mode [default, raw, jsonl]. raw
+                         suppresses log labels and timestamp.
+  -z, --timezone=Local   Specify the timezone to use when formatting output
+                         timestamps [Local, UTC]
       --addr="http://localhost:3100"
-                         Server address. Can also be set using LOKI_ADDR env var.
-      --username=""      Username for HTTP basic auth. Can also be set using LOKI_USERNAME env var.
-      --password=""      Password for HTTP basic auth. Can also be set using LOKI_PASSWORD env var.
-      --ca-cert=""       Path to the server Certificate Authority. Can also be set using LOKI_CA_CERT_PATH env var.
+                         Server address. Can also be set using LOKI_ADDR env
+                         var.
+      --username=""      Username for HTTP basic auth. Can also be set using
+                         LOKI_USERNAME env var.
+      --password=""      Password for HTTP basic auth. Can also be set using
+                         LOKI_PASSWORD env var.
+      --ca-cert=""       Path to the server Certificate Authority. Can also be
+                         set using LOKI_CA_CERT_PATH env var.
       --tls-skip-verify  Server certificate TLS skip verify.
-      --cert=""          Path to the client certificate. Can also be set using LOKI_CLIENT_CERT_PATH env var.
-      --key=""           Path to the client certificate key. Can also be set using LOKI_CLIENT_KEY_PATH env var.
+      --cert=""          Path to the client certificate. Can also be set using
+                         LOKI_CLIENT_CERT_PATH env var.
+      --key=""           Path to the client certificate key. Can also be set
+                         using LOKI_CLIENT_KEY_PATH env var.
       --org-id=ORG-ID    org ID header to be substituted for auth
 
 Commands:
@@ -94,25 +103,31 @@ Commands:
   query [<flags>] <query>
     Run a LogQL query.
 
-    The default output of this command are log entries (combination of
-    timestamp, labels, and log line) along with various extra information about
-    the query and its results. Raw log lines (i.e., no labels or timestamp) can
-    be retrieved by passing the flag "-o raw", while the extra information (URL
-    queried, set of common labels, excluded labels) can be suppressed with the
-    --quiet flag.
-
-    When running a metrics query, this command outputs multiple data points
-    between the start and the end query time.  This produces values that are
-    used to build graphs. If you just want a single data point (i.e., the
-    Grafana explore "table"), then you should use instant-query instead.
+    The "query" command is useful for querying for log lines. The default output
+    of this command are log entries (a combination of timestamp, labels, and a
+    log line) along with various extra information about the performed query and
+    its results. Raw log lines (i.e., without a label and timestamp) can be
+    retrieved by passing the "-o raw" flag. The extra information about the
+    query (API URL, set of common labels, excluded labels) can be suppressed
+    with the --query flag.
+
+    While "query" does support metrics queries, its output contains multiple
+    data points between the start and end query time. This output is used to
+    build graphs, like what is seen in the Grafana Explore graph view. If you
+    are querying metrics and just want the most recent data point (like what is
+    seen in the Grafana Explore table view), then you should use the
+    "instant-query" command instead.
 
   instant-query [<flags>] <query>
     Run an instant LogQL query.
 
-    This query type can only be used for metrics queries, where the query is
-    evaluated for a single point in time. This is equivalent to the Grafana
-    explore "table" view; if you want data that is used to build the Grafana
-    graph, you should use query instead.
+    The "instant-query" command is useful for evaluating a metric query for a
+    single point in time. This is equivalent to the Grafana Explore table view;
+    if you want a metrics query that is used to build a Grafana graph, you
+    should use the "query" command instead.
+
+    This command does not produce useful output when querying for log lines; you
+    should always use the "query" command when you are running log queries.
 
   labels [<label>]
     Find values for a given label.
@@ -122,36 +137,49 @@ usage: logcli query [<flags>] <query>
 
 Run a LogQL query.
 
-The default output of this command are log entries (combination of timestamp,
-labels, and log line) along with various extra information about the query and
-its results. Raw log lines (i.e., no labels or timestamp) can be retrieved by
-passing the flag "-o raw", while the extra information (URL queried, set of
-common labels, excluded labels) can be suppressed with the --quiet flag.
+The "query" command is useful for querying for log lines. The default output of
+this command are log entries (a combination of timestamp, labels, and a log
+line) along with various extra information about the performed query and its
+results. Raw log lines (i.e., without a label and timestamp) can be retrieved by
+passing the "-o raw" flag. The extra information about the query (API URL, set
+of common labels, excluded labels) can be suppressed with the --query flag.
 
-When running a metrics query, this command outputs multiple data points between
-the start and the end query time. This produces values that are used to build
-graphs. If you just want a single data point (i.e., the Grafana explore
-"table"), then you should use instant-query instead.
+While "query" does support metrics queries, its output contains multiple data
+points between the start and end query time. This output is used to build
+graphs, like what is seen in the Grafana Explore graph view. If you are querying
+metrics and just want the most recent data point (like what is seen in the
+Grafana Explore table view), then you should use the "instant-query" command
+instead.
 
 Flags:
-      --help             Show context-sensitive help (also try --help-long and --help-man).
+      --help             Show context-sensitive help (also try --help-long and
+                         --help-man).
       --version          Show application version.
   -q, --quiet            suppress query metadata
       --stats            show query statistics
-  -o, --output=default   specify output mode [default, raw, jsonl]. raw suppresses log labels and timestamp.
-  -z, --timezone=Local   Specify the timezone to use when formatting output timestamps [Local, UTC]
+  -o, --output=default   specify output mode [default, raw, jsonl]. raw
+                         suppresses log labels and timestamp.
+  -z, --timezone=Local   Specify the timezone to use when formatting output
+                         timestamps [Local, UTC]
       --addr="http://localhost:3100"
-                         Server address. Can also be set using LOKI_ADDR env var.
-      --username=""      Username for HTTP basic auth. Can also be set using LOKI_USERNAME env var.
-      --password=""      Password for HTTP basic auth. Can also be set using LOKI_PASSWORD env var.
-      --ca-cert=""       Path to the server Certificate Authority. Can also be set using LOKI_CA_CERT_PATH env var.
+                         Server address. Can also be set using LOKI_ADDR env
+                         var.
+      --username=""      Username for HTTP basic auth. Can also be set using
+                         LOKI_USERNAME env var.
+      --password=""      Password for HTTP basic auth. Can also be set using
+                         LOKI_PASSWORD env var.
+      --ca-cert=""       Path to the server Certificate Authority. Can also be
+                         set using LOKI_CA_CERT_PATH env var.
       --tls-skip-verify  Server certificate TLS skip verify.
-      --cert=""          Path to the client certificate. Can also be set using LOKI_CLIENT_CERT_PATH env var.
-      --key=""           Path to the client certificate key. Can also be set using LOKI_CLIENT_KEY_PATH env var.
+      --cert=""          Path to the client certificate. Can also be set using
+                         LOKI_CLIENT_CERT_PATH env var.
+      --key=""           Path to the client certificate key. Can also be set
+                         using LOKI_CLIENT_KEY_PATH env var.
       --org-id=ORG-ID    org ID header to be substituted for auth
       --limit=30         Limit on number of entries to print.
       --since=1h         Lookback window.
-      --from=FROM        Start looking for logs at this absolute time (inclusive)
+      --from=FROM        Start looking for logs at this absolute time
+                         (inclusive)
       --to=TO            Stop looking for logs at this absolute time (exclusive)
       --step=STEP        Query resolution step width
       --forward          Scan forwards through logs.
@@ -163,37 +191,49 @@ Flags:
                          Include labels given the provided key during output.
       --labels-length=0  Set a fixed padding to labels
   -t, --tail             Tail the logs
-      --delay-for=0      Delay in tailing by number of seconds to accumulate logs for re-ordering
+      --delay-for=0      Delay in tailing by number of seconds to accumulate
+                         logs for re-ordering
 
 Args:
   <query>  eg '{foo="bar",baz=~".*blip"} |~ ".*error.*"'
 
-
 $ logcli help instant-query
 usage: logcli instant-query [<flags>] <query>
 
 Run an instant LogQL query.
 
-This query type can only be used for metrics queries, where the query is
-evaluated for a single point in time. This is equivalent to the Grafana explore
-"table" view; if you want data that is used to build the Grafana graph, you
-should use query instead.
+The "instant-query" command is useful for evaluating a metric query for a single
+point in time. This is equivalent to the Grafana Explore table view; if you want
+a metrics query that is used to build a Grafana graph, you should use the
+"query" command instead.
+
+This command does not produce useful output when querying for log lines; you
+should always use the "query" command when you are running log queries.
 
 Flags:
-      --help             Show context-sensitive help (also try --help-long and --help-man).
+      --help             Show context-sensitive help (also try --help-long and
+                         --help-man).
       --version          Show application version.
   -q, --quiet            suppress query metadata
       --stats            show query statistics
-  -o, --output=default   specify output mode [default, raw, jsonl]. raw suppresses log labels and timestamp.
-  -z, --timezone=Local   Specify the timezone to use when formatting output timestamps [Local, UTC]
+  -o, --output=default   specify output mode [default, raw, jsonl]. raw
+                         suppresses log labels and timestamp.
+  -z, --timezone=Local   Specify the timezone to use when formatting output
+                         timestamps [Local, UTC]
       --addr="http://localhost:3100"
-                         Server address. Can also be set using LOKI_ADDR env var.
-      --username=""      Username for HTTP basic auth. Can also be set using LOKI_USERNAME env var.
-      --password=""      Password for HTTP basic auth. Can also be set using LOKI_PASSWORD env var.
-      --ca-cert=""       Path to the server Certificate Authority. Can also be set using LOKI_CA_CERT_PATH env var.
+                         Server address. Can also be set using LOKI_ADDR env
+                         var.
+      --username=""      Username for HTTP basic auth. Can also be set using
+                         LOKI_USERNAME env var.
+      --password=""      Password for HTTP basic auth. Can also be set using
+                         LOKI_PASSWORD env var.
+      --ca-cert=""       Path to the server Certificate Authority. Can also be
+                         set using LOKI_CA_CERT_PATH env var.
       --tls-skip-verify  Server certificate TLS skip verify.
-      --cert=""          Path to the client certificate. Can also be set using LOKI_CLIENT_CERT_PATH env var.
-      --key=""           Path to the client certificate key. Can also be set using LOKI_CLIENT_KEY_PATH env var.
+      --cert=""          Path to the client certificate. Can also be set using
+                         LOKI_CLIENT_CERT_PATH env var.
+      --key=""           Path to the client certificate key. Can also be set
+                         using LOKI_CLIENT_KEY_PATH env var.
       --org-id=ORG-ID    org ID header to be substituted for auth
       --limit=30         Limit on number of entries to print.
       --now=NOW          Time at which to execute the instant query.
@@ -206,5 +246,5 @@ Flags:
       --labels-length=0  Set a fixed padding to labels
 
 Args:
-  <query>  eg '{foo="bar",baz=~".*blip"} |~ ".*error.*"'
+  <query>  eg 'rate({foo="bar"} |~ ".*error.*" [5m])'
 ```

From b16ddee3a1f8733f7308d11e1135312bb2458b10 Mon Sep 17 00:00:00 2001
From: Robert Fratto <robert.fratto@grafana.com>
Date: Thu, 5 Mar 2020 10:55:33 -0500
Subject: [PATCH 5/6] address review feedback

This commit leaves logcli.md unmodified; that will be done in a later
commit if the changes are approved.
---
 cmd/logcli/main.go | 30 +++++++++++++++++++++---------
 docs/logql.md      | 44 +++++++++++++++++++++++++++++---------------
 2 files changed, 50 insertions(+), 24 deletions(-)

diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go
index f0d43d1c15c9..869449a36318 100644
--- a/cmd/logcli/main.go
+++ b/cmd/logcli/main.go
@@ -29,13 +29,20 @@ var (
 
 	queryCmd = app.Command("query", `Run a LogQL query.
 
-The "query" command is useful for querying for log lines. The default
-output of this command are log entries (a combination of timestamp,
-labels, and a log line) along with various extra information about
-the performed query and its results. Raw log lines (i.e., without a
-label and timestamp) can be retrieved by passing the "-o raw" flag.
-The extra information about the query (API URL, set of common labels,
-excluded labels) can be suppressed with the --query flag.
+The "query" command is useful for querying for logs. Logs can be
+returned in a few output modes:
+
+	raw: log line
+	default: log timestamp + log labels + log line
+	jsonl: JSON response from Loki API of log line
+
+The output of the log can be specified with the "-o" flag, for
+example, "-o raw" for the raw output format.
+
+The "query" command will output extra information about the query
+and its results, such as the API URL, set of common labels, and set
+of excluded labels. This extra information can be suppressed with the
+--quiet flag.
 
 While "query" does support metrics queries, its output contains multiple
 data points between the start and end query time. This output is used to
@@ -55,7 +62,12 @@ view; if you want a metrics query that is used to build a Grafana graph,
 you should use the "query" command instead.
 
 This command does not produce useful output when querying for log lines;
-you should always use the "query" command when you are running log queries.`)
+you should always use the "query" command when you are running log queries.
+
+For more information about log queries and metric queries, refer to the
+LogQL documentation:
+
+https://github.com/grafana/loki/blob/master/docs/logql.md`)
 	instantQuery = newQuery(true, instantQueryCmd)
 
 	labelsCmd = app.Command("labels", "Find values for a given label.")
@@ -141,7 +153,7 @@ func newQueryClient(app *kingpin.Application) *client.Client {
 	app.Flag("tls-skip-verify", "Server certificate TLS skip verify.").Default("false").BoolVar(&client.TLSConfig.InsecureSkipVerify)
 	app.Flag("cert", "Path to the client certificate. Can also be set using LOKI_CLIENT_CERT_PATH env var.").Default("").Envar("LOKI_CLIENT_CERT_PATH").StringVar(&client.TLSConfig.CertFile)
 	app.Flag("key", "Path to the client certificate key. Can also be set using LOKI_CLIENT_KEY_PATH env var.").Default("").Envar("LOKI_CLIENT_KEY_PATH").StringVar(&client.TLSConfig.KeyFile)
-	app.Flag("org-id", "org ID header to be substituted for auth").StringVar(&client.OrgID)
+	app.Flag("org-id", "adds X-Scope-OrgID to API requests for representing tenant ID. Useful for requesting tenant data when bypassing an auth gateway.").StringVar(&client.OrgID)
 
 	return client
 }
diff --git a/docs/logql.md b/docs/logql.md
index 935a02830c1f..8c0c28db9c69 100644
--- a/docs/logql.md
+++ b/docs/logql.md
@@ -1,25 +1,32 @@
 # LogQL: Log Query Language
 
-Loki comes with its very own language for querying logs called *LogQL*. LogQL
-can be considered a distributed `grep` with labels for filtering.
+Loki comes with its own PromQL-inspired language for queries called *LogQL*.
+LogQL can be considered a distributed `grep` that aggregates log sources and
+using labels and operators for filtering.
 
-A basic LogQL query consists of two parts: the **log stream selector** and a
+There are two types of LogQL queries: *log queries* which return the contents of
+log lines, and *metric queries* which extend log queries and calculates values
+based on the counts of logs from a log query.
+
+A basic log query consists of two parts: the **log stream selector** and a
 **filter expression**. Due to Loki's design, all LogQL queries are required to
 contain a log stream selector.
 
-The log stream selector will reduce the number of log streams to a manageable
-volume. Depending how many labels you use to filter down the log streams will
-affect the relative performance of the query's execution. The filter expression
-is then used to do a distributed `grep` over the retrieved log streams.
+The log stream selector determines how many log streams (unique sources of log
+content, such as files) will be searched. A more granular log stream selector
+reduces the number of searched streams to a manageable volume. This means that
+the labels passed to the log stream selector will affect the relative
+performance of the query's execution. The filter expression is then used to do a
+distributed `grep` over the aggregated logs from the matching log streams.
 
 ### Log Stream Selector
 
 The log stream selector determines which log streams should be included in your
-query. The stream selector is comprised of one or more key-value pairs, where
-each key is a **log label** and the value is that label's value.
+query results. The stream selector is comprised of one or more key-value pairs,
+where each key is a **log label** and the value is that label's value.
 
-The log stream selector is written by wrapping the key-value pairs in a
-pair of curly braces:
+The log stream selector is written by wrapping the key-value pairs in a pair of
+curly braces:
 
 ```
 {app="mysql",name="mysql-backup"}
@@ -27,7 +34,10 @@ pair of curly braces:
 
 In this example, log streams that have a label of `app` whose value is `mysql`
 _and_ a label of `name` whose value is `mysql-backup` will be included in the
-query results.
+query results. Note that this will match any log stream whose labels _at least_
+contain `mysql-backup` for their name label; if there are multiple streams that
+contain that label, logs from all of the matching streams will be shown in the
+results.
 
 The `=` operator after the label name is a **label matching operator**. The
 following label matching operators are supported:
@@ -74,10 +84,14 @@ When using `|~` and `!~`,
 matching is case-sensitive by default and can be switched to case-insensitive
 prefixing the regex with `(?i)`.
 
-## Counting logs
+## Metric Queries
+
+LogQL also supports wrapping a log query with functions that allows for counting
+entries per stream.
 
-LogQL also supports functions that wrap a query and allow for counting entries
-per stream.
+Metric queries can be used to calculate things such as the rate of error
+messages, or the top N log sources with the most amount of logs over the last 3
+hours.
 
 ### Range Vector aggregation
 

From bda05103bc4e5325d4a9254ef016d8f218a4fafd Mon Sep 17 00:00:00 2001
From: Robert Fratto <robert.fratto@grafana.com>
Date: Wed, 18 Mar 2020 13:18:56 -0400
Subject: [PATCH 6/6] sync logcli.md with command output

---
 docs/getting-started/logcli.md | 103 ++++++++++++---------------------
 1 file changed, 37 insertions(+), 66 deletions(-)

diff --git a/docs/getting-started/logcli.md b/docs/getting-started/logcli.md
index b6530ba78d16..545a4d38a03d 100644
--- a/docs/getting-started/logcli.md
+++ b/docs/getting-started/logcli.md
@@ -94,7 +94,9 @@ Flags:
                          LOKI_CLIENT_CERT_PATH env var.
       --key=""           Path to the client certificate key. Can also be set
                          using LOKI_CLIENT_KEY_PATH env var.
-      --org-id=ORG-ID    org ID header to be substituted for auth
+      --org-id=ORG-ID    adds X-Scope-OrgID to API requests for representing
+                         tenant ID. Useful for requesting tenant data when
+                         bypassing an auth gateway.
 
 Commands:
   help [<command>...]
@@ -103,13 +105,19 @@ Commands:
   query [<flags>] <query>
     Run a LogQL query.
 
-    The "query" command is useful for querying for log lines. The default output
-    of this command are log entries (a combination of timestamp, labels, and a
-    log line) along with various extra information about the performed query and
-    its results. Raw log lines (i.e., without a label and timestamp) can be
-    retrieved by passing the "-o raw" flag. The extra information about the
-    query (API URL, set of common labels, excluded labels) can be suppressed
-    with the --query flag.
+    The "query" command is useful for querying for logs. Logs can be returned in
+    a few output modes:
+
+      raw: log line
+      default: log timestamp + log labels + log line
+      jsonl: JSON response from Loki API of log line
+
+    The output of the log can be specified with the "-o" flag, for example, "-o
+    raw" for the raw output format.
+
+    The "query" command will output extra information about the query and its
+    results, such as the API URL, set of common labels, and set of excluded
+    labels. This extra information can be suppressed with the --quiet flag.
 
     While "query" does support metrics queries, its output contains multiple
     data points between the start and end query time. This output is used to
@@ -129,6 +137,11 @@ Commands:
     This command does not produce useful output when querying for log lines; you
     should always use the "query" command when you are running log queries.
 
+    For more information about log queries and metric queries, refer to the
+    LogQL documentation:
+
+    https://github.com/grafana/loki/blob/master/docs/logql.md
+
   labels [<label>]
     Find values for a given label.
 
@@ -137,12 +150,19 @@ usage: logcli query [<flags>] <query>
 
 Run a LogQL query.
 
-The "query" command is useful for querying for log lines. The default output of
-this command are log entries (a combination of timestamp, labels, and a log
-line) along with various extra information about the performed query and its
-results. Raw log lines (i.e., without a label and timestamp) can be retrieved by
-passing the "-o raw" flag. The extra information about the query (API URL, set
-of common labels, excluded labels) can be suppressed with the --query flag.
+The "query" command is useful for querying for logs. Logs can be returned in a
+few output modes:
+
+  raw: log line
+  default: log timestamp + log labels + log line
+  jsonl: JSON response from Loki API of log line
+
+The output of the log can be specified with the "-o" flag, for example, "-o raw"
+for the raw output format.
+
+The "query" command will output extra information about the query and its
+results, such as the API URL, set of common labels, and set of excluded labels.
+This extra information can be suppressed with the --quiet flag.
 
 While "query" does support metrics queries, its output contains multiple data
 points between the start and end query time. This output is used to build
@@ -175,7 +195,9 @@ Flags:
                          LOKI_CLIENT_CERT_PATH env var.
       --key=""           Path to the client certificate key. Can also be set
                          using LOKI_CLIENT_KEY_PATH env var.
-      --org-id=ORG-ID    org ID header to be substituted for auth
+      --org-id=ORG-ID    adds X-Scope-OrgID to API requests for representing
+                         tenant ID. Useful for requesting tenant data when
+                         bypassing an auth gateway.
       --limit=30         Limit on number of entries to print.
       --since=1h         Lookback window.
       --from=FROM        Start looking for logs at this absolute time
@@ -196,55 +218,4 @@ Flags:
 
 Args:
   <query>  eg '{foo="bar",baz=~".*blip"} |~ ".*error.*"'
-
-$ logcli help instant-query
-usage: logcli instant-query [<flags>] <query>
-
-Run an instant LogQL query.
-
-The "instant-query" command is useful for evaluating a metric query for a single
-point in time. This is equivalent to the Grafana Explore table view; if you want
-a metrics query that is used to build a Grafana graph, you should use the
-"query" command instead.
-
-This command does not produce useful output when querying for log lines; you
-should always use the "query" command when you are running log queries.
-
-Flags:
-      --help             Show context-sensitive help (also try --help-long and
-                         --help-man).
-      --version          Show application version.
-  -q, --quiet            suppress query metadata
-      --stats            show query statistics
-  -o, --output=default   specify output mode [default, raw, jsonl]. raw
-                         suppresses log labels and timestamp.
-  -z, --timezone=Local   Specify the timezone to use when formatting output
-                         timestamps [Local, UTC]
-      --addr="http://localhost:3100"
-                         Server address. Can also be set using LOKI_ADDR env
-                         var.
-      --username=""      Username for HTTP basic auth. Can also be set using
-                         LOKI_USERNAME env var.
-      --password=""      Password for HTTP basic auth. Can also be set using
-                         LOKI_PASSWORD env var.
-      --ca-cert=""       Path to the server Certificate Authority. Can also be
-                         set using LOKI_CA_CERT_PATH env var.
-      --tls-skip-verify  Server certificate TLS skip verify.
-      --cert=""          Path to the client certificate. Can also be set using
-                         LOKI_CLIENT_CERT_PATH env var.
-      --key=""           Path to the client certificate key. Can also be set
-                         using LOKI_CLIENT_KEY_PATH env var.
-      --org-id=ORG-ID    org ID header to be substituted for auth
-      --limit=30         Limit on number of entries to print.
-      --now=NOW          Time at which to execute the instant query.
-      --forward          Scan forwards through logs.
-      --no-labels        Do not print any labels
-      --exclude-label=EXCLUDE-LABEL ...
-                         Exclude labels given the provided key during output.
-      --include-label=INCLUDE-LABEL ...
-                         Include labels given the provided key during output.
-      --labels-length=0  Set a fixed padding to labels
-
-Args:
-  <query>  eg 'rate({foo="bar"} |~ ".*error.*" [5m])'
 ```