From 6bbb61eb2f52288038ae0112a3b577a1b7861d4d Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Tue, 11 Feb 2020 11:36:36 -0500 Subject: [PATCH] Binary operators in LogQL (#1662) * binops in ast * bin op associativity & precedence * binOpEvaluator work * defers close only if constructed without error * tests binary ops * more binops * updates docs * changelog * better logql parsing errors for binops Signed-off-by: Owen Diehl * adds ^ operator --- CHANGELOG.md | 1 + docs/logql.md | 47 ++++++ pkg/logql/ast.go | 46 ++++++ pkg/logql/ast_test.go | 8 + pkg/logql/engine.go | 3 +- pkg/logql/engine_test.go | 296 ++++++++++++++++++++++++++++++++- pkg/logql/evaluator.go | 193 ++++++++++++++++++++++ pkg/logql/evaluator_test.go | 31 ++++ pkg/logql/expr.y | 36 +++- pkg/logql/expr.y.go | 319 ++++++++++++++++++++++++------------ pkg/logql/lex.go | 11 ++ pkg/logql/parser_test.go | 151 ++++++++++++++++- 12 files changed, 1027 insertions(+), 115 deletions(-) create mode 100644 pkg/logql/evaluator_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c910b53664e..52833472abde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### Features +* [1662](https://github.com/grafana/loki/pull/1662) **owen-d**: Introduces binary operators in LogQL * [1572](https://github.com/grafana/loki/pull/1572) **owen-d**: Introduces the `querier.query-ingesters-within` flag and associated yaml config. When enabled, queries for a time range that do not overlap this lookback interval will not be sent to the ingesters. * [1558](https://github.com/grafana/loki/pull/1558) **owen-d**: Introduces `ingester.max-chunk-age` which specifies the maximum chunk age before it's cut. * [1565](https://github.com/grafana/loki/pull/1565) **owen-d**: The query frontend's `split_queries_by_interval` can now be specified as an override diff --git a/docs/logql.md b/docs/logql.md index 9fb70a0c1a26..284e825af054 100644 --- a/docs/logql.md +++ b/docs/logql.md @@ -151,3 +151,50 @@ by level: Get the rate of HTTP GET requests from NGINX logs: > `avg(rate(({job="nginx"} |= "GET")[10s])) by (region)` + +### Binary Operators + +#### Arithmetic Binary Operators + +Arithmetic binary operators +The following binary arithmetic operators exist in Loki: + +- `+` (addition) +- `-` (subtraction) +- `*` (multiplication) +- `/` (division) +- `%` (modulo) +- `^` (power/exponentiation) + +Binary arithmetic operators are defined only between two vectors. + +Between two instant vectors, a binary arithmetic operator is applied to each entry in the left-hand side vector and its matching element in the right-hand vector. The result is propagated into the result vector with the grouping labels becoming the output label set. Entries for which no matching entry in the right-hand vector can be found are not part of the result. + +##### Examples + +Get proportion of warning logs to error logs for the `foo` app + +> `sum(rate({app="foo", level="warn"}[1m])) / sum(rate({app="foo", level="error"}[1m]))` + +Operators on the same precedence level are left-associative (queries substituted with numbers here for simplicity). For example, 2 * 3 % 2 is equivalent to (2 * 3) % 2. However, some operators have different priorities: 1 + 2 / 3 will still be 1 + ( 2 / 3 ). These function identically to mathematical conventions. + + +#### Logical/set binary operators + +These logical/set binary operators are only defined between two vectors: + +- `and` (intersection) +- `or` (union) +- `unless` (complement) + +`vector1 and vector2` results in a vector consisting of the elements of vector1 for which there are elements in vector2 with exactly matching label sets. Other elements are dropped. + +`vector1 or vector2` results in a vector that contains all original elements (label sets + values) of vector1 and additionally all elements of vector2 which do not have matching label sets in vector1. + +`vector1 unless vector2` results in a vector consisting of the elements of vector1 for which there are no elements in vector2 with exactly matching label sets. All matching elements in both vectors are dropped. + +##### Examples + +This contrived query will return the intersection of these queries, effectively `rate({app="bar"})` + +> `rate({app=~"foo|bar"}[1m]) and rate({app="bar"}[1m])` diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 84cb803410eb..e00659f9f590 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -227,6 +227,17 @@ const ( OpTypeTopK = "topk" OpTypeCountOverTime = "count_over_time" OpTypeRate = "rate" + + // binops + OpTypeOr = "or" + OpTypeAnd = "and" + OpTypeUnless = "unless" + OpTypeAdd = "+" + OpTypeSub = "-" + OpTypeMul = "*" + OpTypeDiv = "/" + OpTypeMod = "%" + OpTypePow = "^" ) // SampleExpr is a LogQL expression filtering logs and returning metric samples. @@ -370,6 +381,41 @@ func (e *vectorAggregationExpr) String() string { return formatOperation(e.operation, e.grouping, params...) } +type binOpExpr struct { + SampleExpr + RHS SampleExpr + op string +} + +func (e *binOpExpr) String() string { + return fmt.Sprintf("%s %s %s", e.SampleExpr.String(), e.op, e.RHS.String()) +} + +func mustNewBinOpExpr(op string, lhs, rhs Expr) SampleExpr { + left, ok := lhs.(SampleExpr) + if !ok { + panic(newParseError(fmt.Sprintf( + "unexpected type for left leg of binary operation (%s): %T", + op, + lhs, + ), 0, 0)) + } + + right, ok := rhs.(SampleExpr) + if !ok { + panic(newParseError(fmt.Sprintf( + "unexpected type for right leg of binary operation (%s): %T", + op, + rhs, + ), 0, 0)) + } + return &binOpExpr{ + SampleExpr: left, + RHS: right, + op: op, + } +} + // helper used to impl Stringer for vector and range aggregations // nolint:interfacer func formatOperation(op string, grouping *grouping, params ...string) string { diff --git a/pkg/logql/ast_test.go b/pkg/logql/ast_test.go index 303cf52986ea..063fecf1561b 100644 --- a/pkg/logql/ast_test.go +++ b/pkg/logql/ast_test.go @@ -43,6 +43,14 @@ func Test_SampleExpr_String(t *testing.T) { `sum(count_over_time({job="mysql"}[5m]))`, `topk(10,sum(rate({region="us-east1"}[5m])) by (name))`, `avg( rate( ( {job="nginx"} |= "GET" ) [10s] ) ) by (region)`, + `sum by (cluster) (count_over_time({job="mysql"}[5m]))`, + `sum by (cluster) (count_over_time({job="mysql"}[5m])) / sum by (cluster) (count_over_time({job="postgres"}[5m])) `, + ` + sum by (cluster) (count_over_time({job="postgres"}[5m])) / + sum by (cluster) (count_over_time({job="postgres"}[5m])) / + sum by (cluster) (count_over_time({job="postgres"}[5m])) + `, + `sum by (cluster) (count_over_time({job="mysql"}[5m])) / min(count_over_time({job="mysql"}[5m])) `, } { t.Run(tc, func(t *testing.T) { expr, err := ParseExpr(tc) diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go index 5e367852b871..12d6572fa6e7 100644 --- a/pkg/logql/engine.go +++ b/pkg/logql/engine.go @@ -213,10 +213,11 @@ func (ng *engine) exec(ctx context.Context, q *query) (promql.Value, error) { func (ng *engine) evalSample(ctx context.Context, expr SampleExpr, q *query) (promql.Value, error) { stepEvaluator, err := ng.evaluator.Evaluator(ctx, expr, q) - defer helpers.LogError("closing SampleExpr", stepEvaluator.Close) if err != nil { return nil, err } + defer helpers.LogError("closing SampleExpr", stepEvaluator.Close) + seriesIndex := map[uint64]*promql.Series{} next, ts, vec := stepEvaluator.Next() diff --git a/pkg/logql/engine_test.go b/pkg/logql/engine_test.go index d8717ad485e1..347ce06687de 100644 --- a/pkg/logql/engine_test.go +++ b/pkg/logql/engine_test.go @@ -3,6 +3,7 @@ package logql import ( "context" "fmt" + "math" "testing" "time" @@ -653,13 +654,17 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, { - `bottomk(3,rate(({app=~"foo|bar"} |~".+bar")[1m])) without (app)`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + `bottomk(3,rate(({app=~"foo|bar|fuzz|buzz"} |~".+bar")[1m])) without (app)`, time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, [][]*logproto.Stream{ - {newStream(testSize, factor(10, identity), `{app="foo"}`), newStream(testSize, factor(20, identity), `{app="bar"}`), - newStream(testSize, factor(5, identity), `{app="fuzz"}`), newStream(testSize, identity, `{app="buzz"}`)}, + { + newStream(testSize, factor(10, identity), `{app="foo"}`), + newStream(testSize, factor(20, identity), `{app="bar"}`), + newStream(testSize, factor(5, identity), `{app="fuzz"}`), + newStream(testSize, identity, `{app="buzz"}`), + }, }, []SelectParams{ - {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar|fuzz|buzz"}|~".+bar"`}}, }, promql.Matrix{ promql.Series{ @@ -676,6 +681,289 @@ func TestEngine_NewRangeQuery(t *testing.T) { }, }, }, + // binops + { + `rate({app="foo"}[1m]) or rate({app="bar"}[1m])`, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="foo"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, + }, + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, + }, + }, + }, + { + ` + rate({app=~"foo|bar"}[1m]) and + rate({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, + }, + }, + }, + { + ` + rate({app=~"foo|bar"}[1m]) unless + rate({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.2}, {T: 90 * 1000, V: 0.2}, {T: 120 * 1000, V: 0.2}, {T: 150 * 1000, V: 0.2}, {T: 180 * 1000, V: 0.2}}, + }, + }, + }, + { + ` + rate({app=~"foo|bar"}[1m]) + + rate({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0.4}, {T: 90 * 1000, V: 0.4}, {T: 120 * 1000, V: 0.4}, {T: 150 * 1000, V: 0.4}, {T: 180 * 1000, V: 0.4}}, + }, + }, + }, + { + ` + rate({app=~"foo|bar"}[1m]) - + rate({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0}, {T: 90 * 1000, V: 0}, {T: 120 * 1000, V: 0}, {T: 150 * 1000, V: 0}, {T: 180 * 1000, V: 0}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) * + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 144}, {T: 90 * 1000, V: 144}, {T: 120 * 1000, V: 144}, {T: 150 * 1000, V: 144}, {T: 180 * 1000, V: 144}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) * + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 144}, {T: 90 * 1000, V: 144}, {T: 120 * 1000, V: 144}, {T: 150 * 1000, V: 144}, {T: 180 * 1000, V: 144}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) / + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1}, {T: 90 * 1000, V: 1}, {T: 120 * 1000, V: 1}, {T: 150 * 1000, V: 1}, {T: 180 * 1000, V: 1}}, + }, + }, + }, + { + ` + count_over_time({app=~"foo|bar"}[1m]) % + count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}`}}, + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 0}, {T: 90 * 1000, V: 0}, {T: 120 * 1000, V: 0}, {T: 150 * 1000, V: 0}, {T: 180 * 1000, V: 0}}, + }, + }, + }, + // tests precedence: should be x + (x/x) + { + ` + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) / + sum by (app) (rate({app=~"foo|bar"} |~".+bar" [1m])) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="foo"}`), + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app=~"foo|bar"}|~".+bar"`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1.2}, {T: 90 * 1000, V: 1.2}, {T: 120 * 1000, V: 1.2}, {T: 150 * 1000, V: 1.2}, {T: 180 * 1000, V: 1.2}}, + }, + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "foo"}}, + Points: []promql.Point{{T: 60 * 1000, V: 1.2}, {T: 90 * 1000, V: 1.2}, {T: 120 * 1000, V: 1.2}, {T: 150 * 1000, V: 1.2}, {T: 180 * 1000, V: 1.2}}, + }, + }, + }, + { + ` + count_over_time({app="bar"}[1m]) ^ count_over_time({app="bar"}[1m]) + `, + time.Unix(60, 0), time.Unix(180, 0), 30 * time.Second, logproto.FORWARD, 100, + [][]*logproto.Stream{ + { + newStream(testSize, factor(5, identity), `{app="bar"}`), + }, + }, + []SelectParams{ + {&logproto.QueryRequest{Direction: logproto.FORWARD, Start: time.Unix(0, 0), End: time.Unix(180, 0), Limit: 0, Selector: `{app="bar"}`}}, + }, + promql.Matrix{ + promql.Series{ + Metric: labels.Labels{{Name: "app", Value: "bar"}}, + Points: []promql.Point{{T: 60 * 1000, V: math.Pow(12, 12)}, {T: 90 * 1000, V: math.Pow(12, 12)}, {T: 120 * 1000, V: math.Pow(12, 12)}, {T: 150 * 1000, V: math.Pow(12, 12)}, {T: 180 * 1000, V: math.Pow(12, 12)}}, + }, + }, + }, } { test := test t.Run(fmt.Sprintf("%s %s", test.qs, test.direction), func(t *testing.T) { diff --git a/pkg/logql/evaluator.go b/pkg/logql/evaluator.go index 2fdf8f0d99ab..b7539121228c 100644 --- a/pkg/logql/evaluator.go +++ b/pkg/logql/evaluator.go @@ -104,6 +104,8 @@ func (ev *defaultEvaluator) Evaluator(ctx context.Context, expr SampleExpr, q Pa return ev.vectorAggEvaluator(ctx, e, q) case *rangeAggregationExpr: return ev.rangeAggEvaluator(ctx, e, q) + case *binOpExpr: + return ev.binOpEvaluator(ctx, e, q) default: return nil, errors.Errorf("unexpected type (%T): %v", e, e) @@ -335,3 +337,194 @@ func (ev *defaultEvaluator) rangeAggEvaluator(ctx context.Context, expr *rangeAg }, vecIter.Close) } + +func (ev *defaultEvaluator) binOpEvaluator( + ctx context.Context, + expr *binOpExpr, + q Params, +) (StepEvaluator, error) { + lhs, err := ev.Evaluator(ctx, expr.SampleExpr, q) + if err != nil { + return nil, err + } + rhs, err := ev.Evaluator(ctx, expr.RHS, q) + if err != nil { + return nil, err + } + + return newStepEvaluator(func() (bool, int64, promql.Vector) { + pairs := map[uint64][2]*promql.Sample{} + var ts int64 + + // populate pairs + for i, eval := range []StepEvaluator{lhs, rhs} { + next, timestamp, vec := eval.Next() + ts = timestamp + + // These should _always_ happen at the same step on each evaluator. + if !next { + return next, ts, nil + } + + for _, sample := range vec { + // TODO(owen-d): this seems wildly inefficient: we're calculating + // the hash on each sample & step per evaluator. + // We seem limited to this approach due to using the StepEvaluator ifc. + hash := sample.Metric.Hash() + pair := pairs[hash] + pair[i] = &promql.Sample{ + Metric: sample.Metric, + Point: sample.Point, + } + pairs[hash] = pair + } + } + + results := make(promql.Vector, 0, len(pairs)) + for _, pair := range pairs { + + // merge + if merged := ev.mergeBinOp(expr.op, pair[0], pair[1]); merged != nil { + results = append(results, *merged) + } + } + + return true, ts, results + }, func() (lastError error) { + for _, ev := range []StepEvaluator{lhs, rhs} { + if err := ev.Close(); err != nil { + lastError = err + } + } + return lastError + }) +} + +func (ev *defaultEvaluator) mergeBinOp(op string, left, right *promql.Sample) *promql.Sample { + var merger func(left, right *promql.Sample) *promql.Sample + + switch op { + case OpTypeOr: + merger = func(left, right *promql.Sample) *promql.Sample { + // return the left entry found (prefers left hand side) + if left != nil { + return left + } + return right + } + + case OpTypeAnd: + merger = func(left, right *promql.Sample) *promql.Sample { + // return left sample if there's a second sample for that label set + if left != nil && right != nil { + return left + } + return nil + } + + case OpTypeUnless: + merger = func(left, right *promql.Sample) *promql.Sample { + // return left sample if there's not a second sample for that label set + if right == nil { + return left + } + return nil + } + + case OpTypeAdd: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, + } + res.Point.V += right.Point.V + return &res + } + + case OpTypeSub: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, + } + res.Point.V -= right.Point.V + return &res + } + + case OpTypeMul: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, + } + res.Point.V *= right.Point.V + return &res + } + + case OpTypeDiv: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + res := promql.Sample{ + Metric: left.Metric.Copy(), + Point: left.Point, + } + + // guard against divide by zero + if right.Point.V == 0 { + res.Point.V = math.NaN() + } else { + res.Point.V /= right.Point.V + } + return &res + } + + case OpTypeMod: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, + } + // guard against divide by zero + if right.Point.V == 0 { + res.Point.V = math.NaN() + } else { + res.Point.V = math.Mod(res.Point.V, right.Point.V) + } + return &res + } + + case OpTypePow: + merger = func(left, right *promql.Sample) *promql.Sample { + if left == nil || right == nil { + return nil + } + + res := promql.Sample{ + Metric: left.Metric, + Point: left.Point, + } + res.Point.V = math.Pow(left.Point.V, right.Point.V) + return &res + } + + default: + panic(errors.Errorf("should never happen: unexpected operation: (%s)", op)) + } + + return merger(left, right) + +} diff --git a/pkg/logql/evaluator_test.go b/pkg/logql/evaluator_test.go new file mode 100644 index 000000000000..23ef3d2f71c9 --- /dev/null +++ b/pkg/logql/evaluator_test.go @@ -0,0 +1,31 @@ +package logql + +import ( + "math" + "testing" + + "github.com/prometheus/prometheus/promql" + "github.com/stretchr/testify/require" +) + +func TestDefaultEvaluator_DivideByZero(t *testing.T) { + ev := &defaultEvaluator{} + + require.Equal(t, true, math.IsNaN(ev.mergeBinOp(OpTypeDiv, + &promql.Sample{ + Point: promql.Point{T: 1, V: 1}, + }, + &promql.Sample{ + Point: promql.Point{T: 1, V: 0}, + }, + ).Point.V)) + + require.Equal(t, true, math.IsNaN(ev.mergeBinOp(OpTypeMod, + &promql.Sample{ + Point: promql.Point{T: 1, V: 1}, + }, + &promql.Sample{ + Point: promql.Point{T: 1, V: 0}, + }, + ).Point.V)) +} diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index a053b3224d36..9a811985191f 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -21,6 +21,8 @@ import ( Selector []*labels.Matcher VectorAggregationExpr SampleExpr VectorOp string + BinOpExpr SampleExpr + binOp string str string duration time.Duration int int64 @@ -41,26 +43,36 @@ import ( %type selector %type vectorAggregationExpr %type vectorOp +%type binOpExpr %token IDENTIFIER STRING %token DURATION %token MATCHERS LABELS EQ NEQ RE NRE OPEN_BRACE CLOSE_BRACE OPEN_BRACKET CLOSE_BRACKET COMMA DOT PIPE_MATCH PIPE_EXACT OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK +// Operators are listed with increasing precedence. +%left OR +%left AND UNLESS +%left ADD SUB +%left MUL DIV MOD +%right POW + %% root: expr { exprlex.(*lexer).expr = $1 }; expr: - logExpr { $$ = $1 } - | rangeAggregationExpr { $$ = $1 } - | vectorAggregationExpr { $$ = $1 } + logExpr { $$ = $1 } + | rangeAggregationExpr { $$ = $1 } + | vectorAggregationExpr { $$ = $1 } + | binOpExpr { $$ = $1 } + | OPEN_PARENTHESIS expr CLOSE_PARENTHESIS { $$ = $2 } ; logExpr: selector { $$ = newMatcherExpr($1)} | logExpr filter STRING { $$ = NewFilterExpr( $1, $2, $3 ) } - | OPEN_PARENTHESIS logExpr CLOSE_PARENTHESIS { $$ = $2} + | OPEN_PARENTHESIS logExpr CLOSE_PARENTHESIS { $$ = $2 } | logExpr filter error | logExpr error ; @@ -115,6 +127,22 @@ matcher: | IDENTIFIER NRE STRING { $$ = mustNewMatcher(labels.MatchNotRegexp, $1, $3) } ; +// TODO(owen-d): add (on,ignoring) clauses to binOpExpr +// Comparison operators are currently avoided due to symbol collisions in our grammar: "!=" means not equal in prometheus, +// but is part of our filter grammar. +// reference: https://prometheus.io/docs/prometheus/latest/querying/operators/ +// Operator precedence only works if each of these is listed separately. +binOpExpr: + expr OR expr { $$ = mustNewBinOpExpr("or", $1, $3) } + | expr AND expr { $$ = mustNewBinOpExpr("and", $1, $3) } + | expr UNLESS expr { $$ = mustNewBinOpExpr("unless", $1, $3) } + | expr ADD expr { $$ = mustNewBinOpExpr("+", $1, $3) } + | expr SUB expr { $$ = mustNewBinOpExpr("-", $1, $3) } + | expr MUL expr { $$ = mustNewBinOpExpr("*", $1, $3) } + | expr DIV expr { $$ = mustNewBinOpExpr("/", $1, $3) } + | expr MOD expr { $$ = mustNewBinOpExpr("%", $1, $3) } + | expr POW expr { $$ = mustNewBinOpExpr("^", $1, $3) } + vectorOp: SUM { $$ = OpTypeSum } | AVG { $$ = OpTypeAvg } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 4730258790db..6e2a851a6a0a 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -25,6 +25,8 @@ type exprSymType struct { Selector []*labels.Matcher VectorAggregationExpr SampleExpr VectorOp string + BinOpExpr SampleExpr + binOp string str string duration time.Duration int int64 @@ -62,6 +64,15 @@ const STDDEV = 57374 const STDVAR = 57375 const BOTTOMK = 57376 const TOPK = 57377 +const OR = 57378 +const AND = 57379 +const UNLESS = 57380 +const ADD = 57381 +const SUB = 57382 +const MUL = 57383 +const DIV = 57384 +const MOD = 57385 +const POW = 57386 var exprToknames = [...]string{ "$end", @@ -99,6 +110,15 @@ var exprToknames = [...]string{ "STDVAR", "BOTTOMK", "TOPK", + "OR", + "AND", + "UNLESS", + "ADD", + "SUB", + "MUL", + "DIV", + "MOD", + "POW", } var exprStatenames = [...]string{} @@ -113,92 +133,127 @@ var exprExca = [...]int{ -2, 0, -1, 3, 1, 2, + 22, 2, + 36, 2, + 37, 2, + 38, 2, + 39, 2, + 40, 2, + 41, 2, + 42, 2, + 43, 2, + 44, 2, + -2, 0, + -1, 39, + 36, 2, + 37, 2, + 38, 2, + 39, 2, + 40, 2, + 41, 2, + 42, 2, + 43, 2, + 44, 2, -2, 0, } const exprPrivate = 57344 -const exprLast = 149 +const exprLast = 202 var exprAct = [...]int{ - 31, 5, 4, 22, 36, 69, 10, 41, 30, 49, - 32, 33, 86, 46, 7, 32, 33, 88, 11, 12, - 13, 14, 16, 17, 15, 18, 19, 20, 21, 90, - 89, 48, 45, 44, 11, 12, 13, 14, 16, 17, - 15, 18, 19, 20, 21, 58, 85, 86, 3, 68, - 67, 63, 87, 84, 23, 71, 28, 72, 61, 65, - 64, 47, 27, 29, 26, 80, 81, 58, 82, 83, - 66, 24, 25, 53, 40, 76, 75, 74, 42, 11, - 12, 13, 14, 16, 17, 15, 18, 19, 20, 21, - 92, 93, 62, 59, 10, 78, 10, 59, 77, 73, - 91, 27, 43, 26, 7, 27, 37, 26, 23, 51, - 24, 25, 70, 79, 24, 25, 27, 60, 26, 23, - 9, 50, 23, 61, 52, 24, 25, 27, 40, 26, - 27, 39, 26, 35, 38, 37, 24, 25, 6, 24, - 25, 54, 55, 56, 57, 8, 34, 2, 1, + 42, 5, 4, 32, 47, 90, 60, 3, 62, 26, + 27, 28, 29, 30, 31, 39, 28, 29, 30, 31, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 31, 41, + 70, 43, 44, 66, 65, 43, 44, 111, 63, 24, + 25, 26, 27, 28, 29, 30, 31, 33, 107, 107, + 110, 82, 106, 109, 108, 37, 79, 36, 105, 86, + 89, 88, 83, 84, 34, 35, 92, 61, 93, 85, + 69, 68, 40, 11, 87, 11, 101, 102, 79, 103, + 104, 7, 67, 64, 72, 12, 13, 14, 15, 17, + 18, 16, 19, 20, 21, 22, 71, 74, 97, 73, + 96, 113, 114, 12, 13, 14, 15, 17, 18, 16, + 19, 20, 21, 22, 12, 13, 14, 15, 17, 18, + 16, 19, 20, 21, 22, 2, 80, 95, 99, 59, + 33, 98, 58, 38, 37, 94, 36, 46, 37, 48, + 36, 112, 48, 34, 35, 91, 100, 34, 35, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 80, 6, + 10, 8, 33, 9, 45, 1, 37, 0, 36, 0, + 37, 0, 36, 33, 0, 34, 35, 82, 81, 34, + 35, 37, 61, 36, 75, 76, 77, 78, 0, 0, + 34, 35, } var exprPact = [...]int{ - -7, -1000, -1000, 120, -1000, -1000, -1000, 83, 42, -13, - 131, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 129, -1000, -1000, -1000, -1000, -1000, 106, 81, - 9, 40, 10, -12, 107, 59, -1000, 132, -1000, -1000, - -1000, 95, 117, 81, 38, 37, 53, 54, 108, 108, - -1000, -1000, 102, -1000, 94, 72, 71, 70, 93, -1000, - -1000, -1000, 52, 91, -8, -8, 54, 31, 24, 30, - -1000, -5, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 8, 7, -1000, -1000, 96, -1000, -1000, -8, - -8, -1000, -1000, -1000, + 70, -1000, -7, 138, -1000, -1000, -1000, 70, -1000, 61, + 18, 145, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 70, 70, 70, 70, 70, 70, 70, + 70, 70, 137, -1000, -1000, -1000, -1000, -1000, -16, 170, + 72, 88, 60, 59, 19, 92, 93, -1000, 185, 12, + -30, -30, -25, -25, -6, -6, -6, -6, -1000, -1000, + -1000, -1000, 166, 181, 72, 57, 47, 67, 99, 151, + 151, -1000, -1000, 148, -1000, 140, 132, 105, 103, 136, + -1000, -1000, -1000, 55, 134, 22, 22, 99, 46, 40, + 42, -1000, 41, -1000, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, -1000, 38, 25, -1000, -1000, 147, -1000, -1000, + 22, 22, -1000, -1000, -1000, } var exprPgo = [...]int{ - 0, 148, 147, 3, 0, 5, 48, 7, 4, 146, - 2, 145, 138, 1, 120, + 0, 175, 135, 3, 0, 5, 7, 8, 4, 174, + 2, 173, 171, 1, 170, 169, } var exprR1 = [...]int{ - 0, 1, 2, 2, 2, 6, 6, 6, 6, 6, - 7, 7, 7, 7, 7, 10, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 3, 3, 3, 3, - 12, 12, 12, 9, 9, 8, 8, 8, 8, 14, - 14, 14, 14, 14, 14, 14, 14, 14, 11, 11, - 5, 5, 4, 4, + 0, 1, 2, 2, 2, 2, 2, 6, 6, 6, + 6, 6, 7, 7, 7, 7, 7, 10, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 3, 3, + 3, 3, 12, 12, 12, 9, 9, 8, 8, 8, + 8, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 11, + 11, 5, 5, 4, 4, } var exprR2 = [...]int{ - 0, 1, 1, 1, 1, 1, 3, 3, 3, 2, - 2, 3, 3, 3, 2, 4, 4, 4, 5, 5, - 5, 5, 6, 7, 6, 7, 1, 1, 1, 1, - 3, 3, 3, 1, 3, 3, 3, 3, 3, 1, + 0, 1, 1, 1, 1, 1, 3, 1, 3, 3, + 3, 2, 2, 3, 3, 3, 2, 4, 4, 4, + 5, 5, 5, 5, 6, 7, 6, 7, 1, 1, + 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 3, 4, 4, + 1, 1, 3, 4, 4, } var exprChk = [...]int{ - -1000, -1, -2, -6, -10, -13, -12, 21, -11, -14, - 13, 25, 26, 27, 28, 31, 29, 30, 32, 33, - 34, 35, -3, 2, 19, 20, 12, 10, -6, 21, - 21, -4, 23, 24, -9, 2, -8, 4, 5, 2, - 22, -7, -6, 21, -10, -13, 4, 21, 21, 21, - 14, 2, 17, 14, 9, 10, 11, 12, -3, 2, - 22, 6, -6, -7, 22, 22, 17, -10, -13, -5, - 4, -5, -8, 5, 5, 5, 5, 5, 2, 22, - -4, -4, -13, -10, 22, 22, 17, 22, 22, 22, - 22, 4, -4, -4, + -1000, -1, -2, -6, -10, -13, -15, 21, -12, -11, + -14, 13, 25, 26, 27, 28, 31, 29, 30, 32, + 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, -3, 2, 19, 20, 12, 10, -2, -6, + 21, 21, -4, 23, 24, -9, 2, -8, 4, -2, + -2, -2, -2, -2, -2, -2, -2, -2, 5, 2, + 22, 22, -7, -6, 21, -10, -13, 4, 21, 21, + 21, 14, 2, 17, 14, 9, 10, 11, 12, -3, + 2, 22, 6, -6, -7, 22, 22, 17, -10, -13, + -5, 4, -5, -8, 5, 5, 5, 5, 5, 2, + 22, -4, -4, -13, -10, 22, 22, 17, 22, 22, + 22, 22, 4, -4, -4, } var exprDef = [...]int{ - 0, -2, 1, -2, 3, 4, 5, 0, 0, 0, - 0, 48, 49, 39, 40, 41, 42, 43, 44, 45, - 46, 47, 0, 9, 26, 27, 28, 29, 0, 0, - 0, 0, 0, 0, 0, 0, 33, 0, 6, 8, - 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 30, 31, 0, 32, 0, 0, 0, 0, 0, 14, - 15, 10, 0, 0, 16, 17, 0, 0, 0, 0, - 50, 0, 34, 35, 36, 37, 38, 11, 13, 12, - 20, 21, 0, 0, 18, 19, 0, 52, 53, 22, - 24, 51, 23, 25, + 0, -2, 1, -2, 3, 4, 5, 0, 7, 0, + 0, 0, 59, 60, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 11, 28, 29, 30, 31, 0, -2, + 0, 0, 0, 0, 0, 0, 0, 35, 0, 41, + 42, 43, 44, 45, 46, 47, 48, 49, 8, 10, + 6, 9, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 32, 33, 0, 34, 0, 0, 0, 0, 0, + 16, 17, 12, 0, 0, 18, 19, 0, 0, 0, + 0, 61, 0, 36, 37, 38, 39, 40, 13, 15, + 14, 22, 23, 0, 0, 20, 21, 0, 63, 64, + 24, 26, 62, 25, 27, } var exprTok1 = [...]int{ @@ -209,7 +264,8 @@ var exprTok2 = [...]int{ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, 44, } var exprTok3 = [...]int{ 0, @@ -574,223 +630,278 @@ exprdefault: case 5: exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.LogExpr = newMatcherExpr(exprDollar[1].Selector) + exprVAL.Expr = exprDollar[1].BinOpExpr } case 6: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.LogExpr = NewFilterExpr(exprDollar[1].LogExpr, exprDollar[2].Filter, exprDollar[3].str) + exprVAL.Expr = exprDollar[2].Expr } case 7: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.LogExpr = newMatcherExpr(exprDollar[1].Selector) + } + case 8: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.LogExpr = NewFilterExpr(exprDollar[1].LogExpr, exprDollar[2].Filter, exprDollar[3].str) + } + case 9: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogExpr = exprDollar[2].LogExpr } - case 10: + case 12: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LogRangeExpr = newLogRange(exprDollar[1].LogExpr, exprDollar[2].duration) } - case 11: + case 13: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogRangeExpr = addFilterToLogRangeExpr(exprDollar[1].LogRangeExpr, exprDollar[2].Filter, exprDollar[3].str) } - case 12: + case 14: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LogRangeExpr = exprDollar[2].LogRangeExpr } - case 15: + case 17: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.RangeAggregationExpr = newRangeAggregationExpr(exprDollar[3].LogRangeExpr, exprDollar[1].RangeOp) } - case 16: + case 18: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].RangeAggregationExpr, exprDollar[1].VectorOp, nil, nil) } - case 17: + case 19: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].VectorAggregationExpr, exprDollar[1].VectorOp, nil, nil) } - case 18: + case 20: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } - case 19: + case 21: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[4].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[2].Grouping, nil) } - case 20: + case 22: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } - case 21: + case 23: exprDollar = exprS[exprpt-5 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[3].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[5].Grouping, nil) } - case 22: + case 24: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].VectorAggregationExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } - case 23: + case 25: exprDollar = exprS[exprpt-7 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].VectorAggregationExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } - case 24: + case 26: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].RangeAggregationExpr, exprDollar[1].VectorOp, nil, &exprDollar[3].str) } - case 25: + case 27: exprDollar = exprS[exprpt-7 : exprpt+1] { exprVAL.VectorAggregationExpr = mustNewVectorAggregationExpr(exprDollar[5].RangeAggregationExpr, exprDollar[1].VectorOp, exprDollar[7].Grouping, &exprDollar[3].str) } - case 26: + case 28: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchRegexp } - case 27: + case 29: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchEqual } - case 28: + case 30: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchNotRegexp } - case 29: + case 31: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Filter = labels.MatchNotEqual } - case 30: + case 32: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Selector = exprDollar[2].Matchers } - case 31: + case 33: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Selector = exprDollar[2].Matchers } - case 32: + case 34: exprDollar = exprS[exprpt-3 : exprpt+1] { } - case 33: + case 35: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Matchers = []*labels.Matcher{exprDollar[1].Matcher} } - case 34: + case 36: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matchers = append(exprDollar[1].Matchers, exprDollar[3].Matcher) } - case 35: + case 37: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchEqual, exprDollar[1].str, exprDollar[3].str) } - case 36: + case 38: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchNotEqual, exprDollar[1].str, exprDollar[3].str) } - case 37: + case 39: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchRegexp, exprDollar[1].str, exprDollar[3].str) } - case 38: + case 40: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Matcher = mustNewMatcher(labels.MatchNotRegexp, exprDollar[1].str, exprDollar[3].str) } - case 39: + case 41: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("or", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 42: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("and", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 43: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("unless", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 44: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 45: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("-", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 46: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("*", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 47: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 48: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 49: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.BinOpExpr = mustNewBinOpExpr("^", exprDollar[1].Expr, exprDollar[3].Expr) + } + case 50: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 40: + case 51: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 41: + case 52: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 42: + case 53: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 43: + case 54: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 44: + case 55: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 45: + case 56: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 46: + case 57: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 47: + case 58: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 48: + case 59: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeCountOverTime } - case 49: + case 60: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpTypeRate } - case 50: + case 61: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 51: + case 62: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 52: + case 63: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 53: + case 64: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index d368e7b69e9a..eedc0dbc9b57 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -36,6 +36,17 @@ var tokens = map[string]int{ OpTypeStdvar: STDVAR, OpTypeBottomK: BOTTOMK, OpTypeTopK: TOPK, + + // binops + OpTypeOr: OR, + OpTypeAnd: AND, + OpTypeUnless: UNLESS, + OpTypeAdd: ADD, + OpTypeSub: SUB, + OpTypeMul: MUL, + OpTypeDiv: DIV, + OpTypeMod: MOD, + OpTypePow: POW, } type lexer struct { diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 97b5b3742783..a2c9d0103427 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -534,7 +534,7 @@ func TestParse(t *testing.T) { { in: `{foo="bar"} "foo"`, err: ParseError{ - msg: "syntax error: unexpected STRING, expecting != or !~ or |~ or |=", + msg: "syntax error: unexpected STRING", line: 1, col: 13, }, @@ -542,11 +542,158 @@ func TestParse(t *testing.T) { { in: `{foo="bar"} foo`, err: ParseError{ - msg: "syntax error: unexpected IDENTIFIER, expecting != or !~ or |~ or |=", + msg: "syntax error: unexpected IDENTIFIER", line: 1, col: 13, }, }, + { + in: ` + sum(count_over_time({foo="bar"}[5m])) by (foo) ^ + sum(count_over_time({foo="bar"}[5m])) by (foo) / + sum(count_over_time({foo="bar"}[5m])) by (foo) + `, + exp: mustNewBinOpExpr( + OpTypeDiv, + mustNewBinOpExpr( + OpTypePow, + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + }, + { + // operator precedence before left associativity + in: ` + sum(count_over_time({foo="bar"}[5m])) by (foo) + + sum(count_over_time({foo="bar"}[5m])) by (foo) / + sum(count_over_time({foo="bar"}[5m])) by (foo) + `, + exp: mustNewBinOpExpr( + OpTypeAdd, + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewBinOpExpr( + OpTypeDiv, + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + mustNewVectorAggregationExpr(newRangeAggregationExpr( + &logRange{ + left: &matchersExpr{ + matchers: []*labels.Matcher{ + mustNewMatcher(labels.MatchEqual, "foo", "bar"), + }, + }, + interval: 5 * time.Minute, + }, OpTypeCountOverTime), + "sum", + &grouping{ + without: false, + groups: []string{"foo"}, + }, + nil, + ), + ), + ), + }, + { + in: `{foo="bar"} + {foo="bar"}`, + err: ParseError{ + msg: `unexpected type for left leg of binary operation (+): *logql.matchersExpr`, + line: 0, + col: 0, + }, + }, + { + in: `sum(count_over_time({foo="bar"}[5m])) by (foo) - {foo="bar"}`, + err: ParseError{ + msg: `unexpected type for right leg of binary operation (-): *logql.matchersExpr`, + line: 0, + col: 0, + }, + }, + { + in: `{foo="bar"} / sum(count_over_time({foo="bar"}[5m])) by (foo)`, + err: ParseError{ + msg: `unexpected type for left leg of binary operation (/): *logql.matchersExpr`, + line: 0, + col: 0, + }, + }, } { t.Run(tc.in, func(t *testing.T) { ast, err := ParseExpr(tc.in)