From 8df56cea6e3830a505d7f455894cd72c474f4c50 Mon Sep 17 00:00:00 2001
From: Edward Welch <edward.welch@grafana.com>
Date: Wed, 26 Feb 2020 16:34:27 -0500
Subject: [PATCH] Increasing the threshold for a file lag and reducing the
 severity to warning

Signed-off-by: Edward Welch <edward.welch@grafana.com>
---
 production/promtail-mixin/alerts.libsonnet | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/production/promtail-mixin/alerts.libsonnet b/production/promtail-mixin/alerts.libsonnet
index 82052fa2b1e8..1f3776b863df 100644
--- a/production/promtail-mixin/alerts.libsonnet
+++ b/production/promtail-mixin/alerts.libsonnet
@@ -40,15 +40,15 @@
           {
             alert: 'PromtailFileLagging',
             expr: |||
-              abs(promtail_file_bytes_total - promtail_read_bytes_total) > 100000
+              abs(promtail_file_bytes_total - promtail_read_bytes_total) > 1e6
             |||,
             'for': '15m',
             labels: {
-              severity: 'critical',
+              severity: 'warning',
             },
             annotations: {
               message: |||
-                {{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} has been lagging by more than 100kb for more than 15m.
+                {{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} has been lagging by more than 1MB for more than 15m.
               |||,
             },
           },