impr/logcli: Added label output filters + tests (grafana#563)

* impr/logcli: Added label filters + tests

* Address review

* udpate readme

cmd/logcli/client.go

@@ -68,7 +68,7 @@ func doRequest(path string, out interface{}) error {
 	}
 	defer func() {
 		if err := resp.Body.Close(); err != nil {
-			fmt.Println("error closing body", err)
+			log.Println("error closing body", err)
 		}
 	}()
 
@@ -92,7 +92,7 @@ func wsConnect(path string) (*websocket.Conn, error) {
 	} else if strings.HasPrefix(url, "http") {
 		url = strings.Replace(url, "http", "ws", 1)
 	}
-	fmt.Println(url)
+	log.Println(url)
 
 	h := http.Header{"Authorization": {"Basic " + base64.StdEncoding.EncodeToString([]byte(*username+":"+*password))}}
 	c, resp, err := websocket.DefaultDialer.Dial(url, h)

cmd/logcli/main.go

@@ -14,14 +14,17 @@ var (
 	username = app.Flag("username", "Username for HTTP basic auth.").Default("").Envar("GRAFANA_USERNAME").String()
 	password = app.Flag("password", "Password for HTTP basic auth.").Default("").Envar("GRAFANA_PASSWORD").String()
 
-	queryCmd  = app.Command("query", "Run a LogQL query.")
-	queryStr  = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String()
-	regexpStr = queryCmd.Arg("regex", "").String()
-	limit     = queryCmd.Flag("limit", "Limit on number of entries to print.").Default("30").Int()
-	since     = queryCmd.Flag("since", "Lookback window.").Default("1h").Duration()
-	forward   = queryCmd.Flag("forward", "Scan forwards through logs.").Default("false").Bool()
-	tail      = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool()
-	noLabels  = queryCmd.Flag("no-labels", "Do not print labels").Default("false").Bool()
+	queryCmd        = app.Command("query", "Run a LogQL query.")
+	queryStr        = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String()
+	regexpStr       = queryCmd.Arg("regex", "").String()
+	limit           = queryCmd.Flag("limit", "Limit on number of entries to print.").Default("30").Int()
+	since           = queryCmd.Flag("since", "Lookback window.").Default("1h").Duration()
+	forward         = queryCmd.Flag("forward", "Scan forwards through logs.").Default("false").Bool()
+	tail            = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool()
+	noLabels        = queryCmd.Flag("no-labels", "Do not print any labels").Default("false").Bool()
+	ignoreLabelsKey = queryCmd.Flag("exclude-label", "Exclude labels given the provided key during output.").Strings()
+	showLabelsKey   = queryCmd.Flag("include-label", "Include labels given the provided key during output.").Strings()
+	fixedLabelsLen  = queryCmd.Flag("labels-length", "Set a fixed padding to labels").Default("0").Int()
 
 	labelsCmd = app.Command("labels", "Find values for a given label.")
 	labelName = labelsCmd.Arg("label", "The name of the label.").HintAction(listLabels).String()

cmd/logcli/query.go

@@ -1,14 +1,12 @@
 package main
 
 import (
-	"fmt"
 	"log"
 	"strings"
 	"time"
 
 	"github.com/fatih/color"
 	"github.com/prometheus/prometheus/pkg/labels"
-	"github.com/prometheus/prometheus/promql"
 
 	"github.com/grafana/loki/pkg/iter"
 	"github.com/grafana/loki/pkg/logproto"
@@ -21,9 +19,8 @@ func doQuery() {
 	}
 
 	var (
-		i            iter.EntryIterator
-		common       labels.Labels
-		maxLabelsLen = 100
+		i      iter.EntryIterator
+		common labels.Labels
 	)
 
 	end := time.Now()
@@ -43,24 +40,43 @@ func doQuery() {
 	labelsCache := func(labels string) labels.Labels {
 		return cache[labels]
 	}
+
 	common = commonLabels(lss)
-	i = iter.NewQueryResponseIterator(resp, d)
+
+	// Remove the labels we want to show from common
+	if len(*showLabelsKey) > 0 {
+		common = common.MatchLabels(false, *showLabelsKey...)
+	}
 
 	if len(common) > 0 {
-		fmt.Println("Common labels:", color.RedString(common.String()))
+		log.Println("Common labels:", color.RedString(common.String()))
+	}
+
+	if len(*ignoreLabelsKey) > 0 {
+		log.Println("Ignoring labels key:", color.RedString(strings.Join(*ignoreLabelsKey, ",")))
 	}
 
+	// Get the max size of labels
+	maxLabelsLen := *fixedLabelsLen
 	for _, ls := range cache {
 		ls = subtract(common, ls)
+		if len(*ignoreLabelsKey) > 0 {
+			ls = ls.MatchLabels(false, *ignoreLabelsKey...)
+		}
 		len := len(ls.String())
 		if maxLabelsLen < len {
 			maxLabelsLen = len
 		}
 	}
 
+	i = iter.NewQueryResponseIterator(resp, d)
+
 	for i.Next() {
 		ls := labelsCache(i.Labels())
 		ls = subtract(ls, common)
+		if len(*ignoreLabelsKey) > 0 {
+			ls = ls.MatchLabels(false, *ignoreLabelsKey...)
+		}
 
 		labels := ""
 		if !*noLabels {
@@ -74,95 +90,3 @@ func doQuery() {
 		log.Fatalf("Error from iterator: %v", err)
 	}
 }
-
-func printLogEntry(ts time.Time, lbls string, line string) {
-	fmt.Println(
-		color.BlueString(ts.Format(time.RFC3339)),
-		color.RedString(lbls),
-		strings.TrimSpace(line),
-	)
-}
-
-func padLabel(ls labels.Labels, maxLabelsLen int) string {
-	labels := ls.String()
-	if len(labels) < maxLabelsLen {
-		labels += strings.Repeat(" ", maxLabelsLen-len(labels))
-	}
-	return labels
-}
-
-func mustParseLabels(labels string) labels.Labels {
-	ls, err := promql.ParseMetric(labels)
-	if err != nil {
-		log.Fatalf("Failed to parse labels: %+v", err)
-	}
-	return ls
-}
-
-func parseLabels(resp *logproto.QueryResponse) (map[string]labels.Labels, []labels.Labels) {
-	cache := make(map[string]labels.Labels, len(resp.Streams))
-	lss := make([]labels.Labels, 0, len(resp.Streams))
-	for _, stream := range resp.Streams {
-		ls := mustParseLabels(stream.Labels)
-		cache[stream.Labels] = ls
-		lss = append(lss, ls)
-	}
-	return cache, lss
-}
-
-func commonLabels(lss []labels.Labels) labels.Labels {
-	if len(lss) == 0 {
-		return nil
-	}
-
-	result := lss[0]
-	for i := 1; i < len(lss); i++ {
-		result = intersect(result, lss[i])
-	}
-	return result
-}
-
-func intersect(a, b labels.Labels) labels.Labels {
-	var result labels.Labels
-	for i, j := 0, 0; i < len(a) && j < len(b); {
-		k := strings.Compare(a[i].Name, b[j].Name)
-		switch {
-		case k == 0:
-			if a[i].Value == b[j].Value {
-				result = append(result, a[i])
-			}
-			i++
-			j++
-		case k < 0:
-			i++
-		case k > 0:
-			j++
-		}
-	}
-	return result
-}
-
-// subtract b from a
-func subtract(a, b labels.Labels) labels.Labels {
-	var result labels.Labels
-	i, j := 0, 0
-	for i < len(a) && j < len(b) {
-		k := strings.Compare(a[i].Name, b[j].Name)
-		if k != 0 || a[i].Value != b[j].Value {
-			result = append(result, a[i])
-		}
-		switch {
-		case k == 0:
-			i++
-			j++
-		case k < 0:
-			i++
-		case k > 0:
-			j++
-		}
-	}
-	for ; i < len(a); i++ {
-		result = append(result, a[i])
-	}
-	return result
-}

cmd/logcli/query_test.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/prometheus/prometheus/pkg/labels"
+)
+
+func Test_commonLabels(t *testing.T) {
+	type args struct {
+		lss []labels.Labels
+	}
+	tests := []struct {
+		name string
+		args args
+		want labels.Labels
+	}{
+		{
+			"Extract common labels source > target",
+			args{
+				[]labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo", foo="foo", baz="baz"}`)},
+			},
+			mustParseLabels(`{bar="foo"}`),
+		},
+		{
+			"Extract common labels source > target",
+			args{
+				[]labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo", foo="bar", baz="baz"}`)},
+			},
+			mustParseLabels(`{foo="bar", bar="foo"}`),
+		},
+		{
+			"Extract common labels source < target",
+			args{
+				[]labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo"}`)},
+			},
+			mustParseLabels(`{bar="foo"}`),
+		},
+		{
+			"Extract common labels source < target no common",
+			args{
+				[]labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{fo="bar"}`)},
+			},
+			labels.Labels{},
+		},
+		{
+			"Extract common labels source = target no common",
+			args{
+				[]labels.Labels{mustParseLabels(`{foo="bar"}`), mustParseLabels(`{fooo="bar"}`)},
+			},
+			labels.Labels{},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := commonLabels(tt.args.lss); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("commonLabels() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_subtract(t *testing.T) {
+	type args struct {
+		a labels.Labels
+		b labels.Labels
+	}
+	tests := []struct {
+		name string
+		args args
+		want labels.Labels
+	}{
+		{
+			"Subtract labels source > target",
+			args{
+				mustParseLabels(`{foo="bar", bar="foo"}`),
+				mustParseLabels(`{bar="foo", foo="foo", baz="baz"}`),
+			},
+			mustParseLabels(`{foo="bar"}`),
+		},
+		{
+			"Subtract labels source < target",
+			args{
+				mustParseLabels(`{foo="bar", bar="foo"}`),
+				mustParseLabels(`{bar="foo"}`),
+			},
+			mustParseLabels(`{foo="bar"}`),
+		},
+		{
+			"Subtract labels source < target no sub",
+			args{
+				mustParseLabels(`{foo="bar", bar="foo"}`),
+				mustParseLabels(`{fo="bar"}`),
+			},
+			mustParseLabels(`{bar="foo", foo="bar"}`),
+		},
+		{
+			"Subtract labels source = target no sub",
+			args{
+				mustParseLabels(`{foo="bar"}`),
+				mustParseLabels(`{fiz="buz"}`),
+			},
+			mustParseLabels(`{foo="bar"}`),
+		},
+		{
+			"Subtract labels source > target no sub",
+			args{
+				mustParseLabels(`{foo="bar"}`),
+				mustParseLabels(`{fiz="buz", foo="baz"}`),
+			},
+			mustParseLabels(`{foo="bar"}`),
+		},
+		{
+			"Subtract labels source > target no sub",
+			args{
+				mustParseLabels(`{a="b", foo="bar", baz="baz", fizz="fizz"}`),
+				mustParseLabels(`{foo="bar", baz="baz", buzz="buzz", fizz="fizz"}`),
+			},
+			mustParseLabels(`{a="b"}`),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := subtract(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("subtract() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/logcli/tail.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"log"
+	"strings"
+
+	"github.com/fatih/color"
 
 	"github.com/grafana/loki/pkg/logproto"
 )
@@ -14,6 +17,14 @@ func tailQuery() {
 
 	stream := new(logproto.Stream)
 
+	if len(*ignoreLabelsKey) > 0 {
+		log.Println("Ingoring labels key:", color.RedString(strings.Join(*ignoreLabelsKey, ",")))
+	}
+
+	if len(*showLabelsKey) > 0 {
+		log.Println("Print only labels key:", color.RedString(strings.Join(*showLabelsKey, ",")))
+	}
+
 	for {
 		err := conn.ReadJSON(stream)
 		if err != nil {
@@ -23,7 +34,25 @@ func tailQuery() {
 
 		labels := ""
 		if !*noLabels {
-			labels = stream.Labels
+
+			if len(*ignoreLabelsKey) > 0 || len(*showLabelsKey) > 0 {
+
+				ls := mustParseLabels(stream.GetLabels())
+
+				if len(*showLabelsKey) > 0 {
+					ls = ls.MatchLabels(true, *showLabelsKey...)
+				}
+
+				if len(*ignoreLabelsKey) > 0 {
+					ls = ls.MatchLabels(false, *ignoreLabelsKey...)
+				}
+
+				labels = ls.String()
+
+			} else {
+
+				labels = stream.Labels
+			}
 		}
 		for _, entry := range stream.Entries {
 			printLogEntry(entry.Timestamp, labels, entry.Line)