Squashed commit of the following:

commit b9e647f
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Mon Aug 12 11:31:31 2024 -0600

    Squashed commit of the following:

    commit c6ab6b3
    Author: Trevor Whitney <trevorjwhitney@gmail.com>
    Date:   Mon Aug 12 11:09:43 2024 -0600

        chore: remove initial metric aggregation experiment (#13729)

    commit 3c0e3e2
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Mon Aug 12 12:00:51 2024 -0400

        fix(deps): update module github.com/baidubce/bce-sdk-go to v0.9.186 (#13864)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit 6f79194
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Mon Aug 12 10:20:34 2024 -0400

        fix(deps): update module github.com/aliyun/aliyun-oss-go-sdk to v2.2.10+incompatible (#13861)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit ad60738
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Mon Aug 12 09:53:11 2024 -0400

        chore(deps): update grafana/loki-build-image docker tag to v0.33.6 (#13859)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit 292f911
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Mon Aug 12 09:46:36 2024 -0400

        chore(deps): update helm/chart-testing-action action to v2.6.1 (#13855)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit e0bde12
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Mon Aug 12 09:45:49 2024 -0400

        chore(deps): update grafana/promtail docker tag to v1.6.1 (#13851)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit a9cd5e5
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Mon Aug 12 09:42:53 2024 -0400

        chore(deps): update grafana/promtail docker tag to v2.9.10 (#13854)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit 0c28cc7
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Mon Aug 12 09:42:21 2024 -0400

        chore(deps): update dependency go to v1.22.6 (#13842)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
        Co-authored-by: Paul Rogers <129207811+paul1r@users.noreply.github.com>

    commit c3a5141
    Author: Paul Rogers <129207811+paul1r@users.noreply.github.com>
    Date:   Mon Aug 12 09:29:19 2024 -0400

        chore: Update loki build image to go 1.22.6 (#13857)

    commit 717623b
    Author: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com>
    Date:   Mon Aug 12 15:47:29 2024 +0300

        chore:  manual changelog backport (#13852)

        Signed-off-by: Vladyslav Diachenko <vlad.diachenko@grafana.com>

    commit f933a3b
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Mon Aug 12 08:40:39 2024 -0400

        chore(deps): update grafana/loki docker tag to v2.9.10 (#13848)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
        Co-authored-by: Paul Rogers <129207811+paul1r@users.noreply.github.com>

    commit ea6395a
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Mon Aug 12 08:33:10 2024 -0400

        chore(deps): update golang docker tag to v1.22.6 (#13847)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit 53c0c48
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Mon Aug 12 08:32:08 2024 -0400

        chore(deps): update grafana/loki-build-image docker tag to v0.33.5 (#13849)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit 9315b3d
    Author: Grot (@grafanabot) <43478413+grafanabot@users.noreply.github.com>
    Date:   Fri Aug 9 12:38:47 2024 +0200

        chore(ci): Update yaml file `./production/helm/loki/values.yaml` (+1 other) (#13832)

        Signed-off-by: Vladyslav Diachenko <vlad.diachenko@grafana.com>
        Co-authored-by: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com>

    commit 66d7138
    Author: Grot (@grafanabot) <43478413+grafanabot@users.noreply.github.com>
    Date:   Thu Aug 8 17:31:41 2024 +0200

        chore: [main] chore(release-3.1.x): release 3.1.1 (#13817)

        Co-authored-by: loki-gh-app[bot] <160051081+loki-gh-app[bot]@users.noreply.github.com>

    commit d5718eb
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Thu Aug 8 10:49:41 2024 -0400

        fix(deps): update github.com/grafana/jsonparser digest to ea80629 (#13814)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit f253db5
    Author: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com>
    Date:   Thu Aug 8 17:12:11 2024 +0300

        fix(ci): fixed release-please manifest (#13810)

        Signed-off-by: Vladyslav Diachenko <vlad.diachenko@grafana.com>

    commit a93f38c
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Thu Aug 8 09:48:10 2024 -0400

        fix(deps): update github.com/c2h5oh/datasize digest to aa82cc1 (#13807)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
        Co-authored-by: Paul Rogers <129207811+paul1r@users.noreply.github.com>

    commit e5a3994
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Thu Aug 8 09:39:56 2024 -0400

        fix(deps): update github.com/docker/go-plugins-helpers digest to 45e2431 (#13808)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit 67295e0
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Thu Aug 8 09:33:48 2024 -0400

        fix(deps): update github.com/axiomhq/hyperloglog digest to af9851f (#13806)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit c9b0343
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Thu Aug 8 09:32:02 2024 -0400

        chore(deps): update github.com/grafana/regexp digest to a468a5b (#13805)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit 5bb928e
    Author: Paul Rogers <129207811+paul1r@users.noreply.github.com>
    Date:   Thu Aug 8 08:52:50 2024 -0400

        chore: Turn off renovate on non-main branches (#13803)

    commit 217f928
    Author: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com>
    Date:   Thu Aug 8 15:31:43 2024 +0300

        fix(ci): add cleanup step into job `dist` (#13801)

        Signed-off-by: Vladyslav Diachenko <vlad.diachenko@grafana.com>

    commit 00e686d
    Author: benclive <ben.clive@grafana.com>
    Date:   Thu Aug 8 10:40:16 2024 +0100

        chore: Add metastore client as dep for rf1 querier & ignore auth for ListBlocks (#13786)

    commit df61482
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Wed Aug 7 18:12:13 2024 -0400

        fix(deps): update module golang.org/x/text to v0.17.0 (main) (#13794)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit f523530
    Author: Periklis Tsirakidis <periklis@redhat.com>
    Date:   Wed Aug 7 20:05:42 2024 +0200

        fix(operator): Don't overwrite annotations for LokiStack ingress resources (#13708)

    commit 5ef83a7
    Author: Ned Andreev <n@andreev.sh>
    Date:   Wed Aug 7 19:44:43 2024 +0300

        fix: panic when parsing and extracting JSON key values (#13790)

    commit bb257f5
    Author: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com>
    Date:   Wed Aug 7 17:39:04 2024 +0300

        feat(loki):  add ability to disable AWS S3 dualstack endpoints usage  (#13785)

        Signed-off-by: Vladyslav Diachenko <vlad.diachenko@grafana.com>

    commit 1bf9791
    Author: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com>
    Date:   Wed Aug 7 16:10:34 2024 +0300

        fix(helm): fixed memcached and provisioner templates (#13788)

        Signed-off-by: Vladyslav Diachenko <vlad.diachenko@grafana.com>

    commit 638f59f
    Author: benclive <ben.clive@grafana.com>
    Date:   Wed Aug 7 14:06:28 2024 +0100

        chore: Remove unused stream index from RF1 ingester (#13758)

    commit 7683a79
    Author: Ned Andreev <n@andreev.sh>
    Date:   Tue Aug 6 20:26:10 2024 +0300

        fix: Include whitespaces in extracted tokens (#13738)

        Co-authored-by: Travis Patterson <travis.patterson@grafana.com>

    commit da63ca7
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Tue Aug 6 12:41:48 2024 -0400

        chore(deps): update module golang.org/x/net to v0.23.0 [security] (main) (#13763)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
        Co-authored-by: Paul Rogers <129207811+paul1r@users.noreply.github.com>

    commit f8bf3bb
    Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
    Date:   Tue Aug 6 12:29:47 2024 -0400

        fix(deps): update module github.com/docker/docker to v27.1.1+incompatible [security] (main) (#13762)

        Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

    commit b88583d
    Author: Marco Pracucci <marco@pracucci.com>
    Date:   Tue Aug 6 16:04:28 2024 +0200

        feat: upgrade prometheus (#13671)

        Signed-off-by: Marco Pracucci <marco@pracucci.com>
        Signed-off-by: Vladyslav Diachenko <vlad.diachenko@grafana.com>
        Co-authored-by: Vladyslav Diachenko <vlad.diachenko@grafana.com>

    commit 3be5a45
    Author: Dylan Guedes <djmgguedes@gmail.com>
    Date:   Mon Aug 5 17:59:21 2024 -0300

        fix(break): helm: Fix how we set imagePullSecrets for enterprise-gateway and admin-api. (#13761)

    commit bdfc86b
    Author: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com>
    Date:   Mon Aug 5 13:46:10 2024 +0300

        chore(helm-chart): added SSE config into AWS storage config (#13746)

        Signed-off-by: Vladyslav Diachenko <vlad.diachenko@grafana.com>

    commit 7e224d5
    Author: Sandeep Sukhani <sandeep.d.sukhani@gmail.com>
    Date:   Sun Aug 4 19:28:29 2024 +0530

        fix: try reading chunks which have incorrect offset for blocks (#13720)

    commit 629671e
    Author: J Stickler <julie.stickler@grafana.com>
    Date:   Fri Aug 2 16:58:41 2024 -0400

        docs: Update the Visualize topic (#13742)

    commit 917053a
    Author: Cyril Tovena <cyril.tovena@gmail.com>
    Date:   Fri Aug 2 18:07:17 2024 +0200

        feat: Introduce wal segment read path. (#13695)

        Co-authored-by: Ben Clive <ben.clive@grafana.com>

    commit 7c50b43
    Author: Paul Rogers <129207811+paul1r@users.noreply.github.com>
    Date:   Fri Aug 2 08:47:33 2024 -0400

        build: Update loki-build-image to Alpine 3.20.2 (#13744)

    commit 6dd6b65
    Author: jackyin <yincong@baidu.com>
    Date:   Fri Aug 2 01:30:11 2024 +0800

        fix: ast left cycular reference result in oom (#13501)

        Co-authored-by: Travis Patterson <travis.patterson@grafana.com>

    commit e81345e
    Author: J Stickler <julie.stickler@grafana.com>
    Date:   Thu Aug 1 11:29:38 2024 -0400

        docs: fix broken links due to Alloy docs reorg (#13715)

    commit 40e8352
    Author: Jack Baldry <jack.baldry@grafana.com>
    Date:   Thu Aug 1 15:41:48 2024 +0100

        docs: rewrite quickstart with Killercoda metadata (#13234)

        Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
        Co-authored-by: J Stickler <julie.stickler@grafana.com>
        Co-authored-by: Jay Clifford <45856600+Jayclifford345@users.noreply.github.com>

commit c65ba1f
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Mon Aug 12 11:08:37 2024 -0600

    chore: rework pattern ingester queue metrics

commit b28572a
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Fri Aug 2 13:16:01 2024 -0600

    fix: change metrics around metric appends

commit e430a96
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Fri Aug 2 08:59:05 2024 -0600

    fix: lint and format

commit 2f5485d
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Fri Aug 2 08:45:57 2024 -0600

    feat: improve async tee request processing

commit 3dcc50d
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Thu Aug 1 15:16:13 2024 -0600

    test: add push and downsample test

commit 019df76
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Thu Aug 1 14:41:41 2024 -0600

    fix: formatting

commit 7d98cb0
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Thu Aug 1 13:35:55 2024 -0600

    chore: move some functions around

commit 06269b0
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Thu Aug 1 13:29:57 2024 -0600

    test: mock out ring in tests

commit 07d57c3
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Thu Aug 1 11:56:34 2024 -0600

    fix: formatting

commit 93e4add
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Thu Aug 1 11:53:42 2024 -0600

    fix: remove now unused test

commit c03db1f
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Thu Aug 1 11:49:59 2024 -0600

    fix: remove duplicate service detection for distributor

commit fe447f5
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Thu Aug 1 11:45:31 2024 -0600

    fix: service name detection on push

commit afb8513
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Wed Jul 31 21:40:39 2024 -0600

    feat: introduce RingClient abstraction

    * lint and format

commit ce87fd8
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Wed Jul 31 20:41:38 2024 -0600

    feat: aggregate byte and count metrics

    * aggregate in the pattern ingester and push back into Loki as a stream

commit b49e52b
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Wed Jul 31 14:36:59 2024 -0600

    chore: fix formatting

commit af22a21
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Wed Jul 31 14:32:05 2024 -0600

    chore: fix linting

commit d51cecc
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Wed Jul 31 14:12:31 2024 -0600

    chore: remove the metric aggregation experiment

    * this will make way for a new, simpler approach

commit dad6fb5
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Wed Jul 31 13:39:07 2024 -0600

    Revert "feat: collect and serve pre-aggregated bytes and counts (#13020)"

    This reverts commit 467eb1b.

commit a6835e1
Author: Trevor Whitney <trevorjwhitney@gmail.com>
Date:   Wed Jul 31 13:18:12 2024 -0600

    Revert "feat: downsample aggregated metrics (#13449)"

    This reverts commit 2c053ee.

cmd/loki/loki-local-config.yaml

@@ -42,7 +42,7 @@ pattern_ingester:
   enabled: true
   metric_aggregation:
     enabled: true
-    log_push_observations: true
+    loki_address: localhost:3100
 
 ruler:
   alertmanager_url: http://localhost:9093

docs/sources/shared/configuration.md

@@ -612,6 +612,189 @@ pattern_ingester:
   # CLI flag: -pattern-ingester.max-eviction-ratio
   [max_eviction_ratio: <float> | default = 0.25]
 
+  # Configures the metric aggregation and storage behavior of the pattern
+  # ingester.
+  metric_aggregation:
+    # Whether the pattern ingester metric aggregation is enabled.
+    # CLI flag: -pattern-ingester.metric-aggregation.enabled
+    [enabled: <boolean> | default = false]
+
+    # How often to downsample metrics from raw push observations.
+    # CLI flag: -pattern-ingester.metric-aggregation.downsample-period
+    [downsample_period: <duration> | default = 10s]
+
+    # The address of the Loki instance to push aggregated metrics to.
+    # CLI flag: -pattern-ingester.metric-aggregation.loki-address
+    [loki_address: <string> | default = ""]
+
+    # The timeout for writing to Loki.
+    # CLI flag: -pattern-ingester.metric-aggregation.timeout
+    [timeout: <duration> | default = 10s]
+
+    # How long to wait in between pushes to Loki.
+    # CLI flag: -pattern-ingester.metric-aggregation.push-period
+    [push_period: <duration> | default = 1m]
+
+    # The HTTP client configuration for pushing metrics to Loki.
+    http_client_config:
+      basic_auth:
+        [username: <string> | default = ""]
+
+        [username_file: <string> | default = ""]
+
+        [username_ref: <string> | default = ""]
+
+        [password: <string> | default = ""]
+
+        [password_file: <string> | default = ""]
+
+        [password_ref: <string> | default = ""]
+
+      authorization:
+        [type: <string> | default = ""]
+
+        [credentials: <string> | default = ""]
+
+        [credentials_file: <string> | default = ""]
+
+        [credentials_ref: <string> | default = ""]
+
+      oauth2:
+        [client_id: <string> | default = ""]
+
+        [client_secret: <string> | default = ""]
+
+        [client_secret_file: <string> | default = ""]
+
+        [client_secret_ref: <string> | default = ""]
+
+        [scopes: <list of strings>]
+
+        [token_url: <string> | default = ""]
+
+        [endpoint_params: <map of string to string>]
+
+        tls_config:
+          [ca: <string> | default = ""]
+
+          [cert: <string> | default = ""]
+
+          [key: <string> | default = ""]
+
+          [ca_file: <string> | default = ""]
+
+          [cert_file: <string> | default = ""]
+
+          [key_file: <string> | default = ""]
+
+          [ca_ref: <string> | default = ""]
+
+          [cert_ref: <string> | default = ""]
+
+          [key_ref: <string> | default = ""]
+
+          [server_name: <string> | default = ""]
+
+          [insecure_skip_verify: <boolean>]
+
+          [min_version: <int>]
+
+          [max_version: <int>]
+
+        proxy_url:
+          [url: <url>]
+
+        [no_proxy: <string> | default = ""]
+
+        [proxy_from_environment: <boolean>]
+
+        [proxy_connect_header: <map of string to list of strings>]
+
+      [bearer_token: <string> | default = ""]
+
+      [bearer_token_file: <string> | default = ""]
+
+      tls_config:
+        [ca: <string> | default = ""]
+
+        [cert: <string> | default = ""]
+
+        [key: <string> | default = ""]
+
+        [ca_file: <string> | default = ""]
+
+        [cert_file: <string> | default = ""]
+
+        [key_file: <string> | default = ""]
+
+        [ca_ref: <string> | default = ""]
+
+        [cert_ref: <string> | default = ""]
+
+        [key_ref: <string> | default = ""]
+
+        [server_name: <string> | default = ""]
+
+        [insecure_skip_verify: <boolean>]
+
+        [min_version: <int>]
+
+        [max_version: <int>]
+
+      [follow_redirects: <boolean>]
+
+      [enable_http2: <boolean>]
+
+      proxy_url:
+        [url: <url>]
+
+      [no_proxy: <string> | default = ""]
+
+      [proxy_from_environment: <boolean>]
+
+      [proxy_connect_header: <map of string to list of strings>]
+
+      http_headers:
+        [: <map of string to Header>]
+
+    # Whether to use TLS for pushing metrics to Loki.
+    # CLI flag: -pattern-ingester.metric-aggregation.tls
+    [use_tls: <boolean> | default = false]
+
+    # The basic auth configuration for pushing metrics to Loki.
+    basic_auth:
+      # Basic auth username for sending aggregations back to Loki.
+      # CLI flag: -pattern-ingester.metric-aggregation.basic-auth.username
+      [username: <string> | default = ""]
+
+      # Basic auth password for sending aggregations back to Loki.
+      # CLI flag: -pattern-ingester.metric-aggregation.basic-auth.password
+      [password: <string> | default = ""]
+
+    # The backoff configuration for pushing metrics to Loki.
+    backoff_config:
+      # Minimum delay when backing off.
+      # CLI flag: -pattern-ingester.metric-aggregation.backoff-min-period
+      [min_period: <duration> | default = 100ms]
+
+      # Maximum delay when backing off.
+      # CLI flag: -pattern-ingester.metric-aggregation.backoff-max-period
+      [max_period: <duration> | default = 10s]
+
+      # Number of times to backoff and retry before failing.
+      # CLI flag: -pattern-ingester.metric-aggregation.backoff-retries
+      [max_retries: <int> | default = 10]
+
+  # The number of parallel goroutines to use for forwarding requests to the
+  # pattern ingester.
+  # CLI flag: -pattern-ingester.tee-parallelism
+  [tee_parallelism: <int> | default = 5]
+
+  # Maxiumum number of pending teed request to pattern ingesters. If the queue
+  # is full the request is dropped.
+  # CLI flag: -pattern-ingester.tee-queue-size
+  [tee_queue_size: <int> | default = 100]
+
 # The index_gateway block configures the Loki index gateway server, responsible
 # for serving index queries without the need to constantly interact with the
 # object store.

pkg/distributor/distributor.go

@@ -61,15 +61,15 @@ const (
 
 	ringAutoForgetUnhealthyPeriods = 2
 
-	levelLabel       = "detected_level"
+	LevelLabel       = "detected_level"
+	LogLevelUnknown  = "unknown"
 	logLevelDebug    = "debug"
 	logLevelInfo     = "info"
 	logLevelWarn     = "warn"
 	logLevelError    = "error"
 	logLevelFatal    = "fatal"
 	logLevelCritical = "critical"
 	logLevelTrace    = "trace"
-	logLevelUnknown  = "unknown"
 )
 
 var (
@@ -304,6 +304,8 @@ func (d *Distributor) running(ctx context.Context) error {
 }
 
 func (d *Distributor) stopping(_ error) error {
+	d.tee.Stop()
+
 	return services.StopManagerAndAwaitStopped(context.Background(), d.subservices)
 }
 
@@ -406,9 +408,9 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log
 					} else {
 						logLevel = detectLogLevelFromLogEntry(entry, structuredMetadata)
 					}
-					if logLevel != logLevelUnknown && logLevel != "" {
+					if logLevel != LogLevelUnknown && logLevel != "" {
 						entry.StructuredMetadata = append(entry.StructuredMetadata, logproto.LabelAdapter{
-							Name:  levelLabel,
+							Name:  LevelLabel,
 							Value: logLevel,
 						})
 					}
@@ -886,7 +888,7 @@ func detectLogLevelFromLogEntry(entry logproto.Entry, structuredMetadata labels.
 			return logLevelInfo
 		}
 		if otlpSeverityNumber == int(plog.SeverityNumberUnspecified) {
-			return logLevelUnknown
+			return LogLevelUnknown
 		} else if otlpSeverityNumber <= int(plog.SeverityNumberTrace4) {
 			return logLevelTrace
 		} else if otlpSeverityNumber <= int(plog.SeverityNumberDebug4) {
@@ -900,7 +902,7 @@ func detectLogLevelFromLogEntry(entry logproto.Entry, structuredMetadata labels.
 		} else if otlpSeverityNumber <= int(plog.SeverityNumberFatal4) {
 			return logLevelFatal
 		}
-		return logLevelUnknown
+		return LogLevelUnknown
 	}
 
 	return extractLogLevelFromLogLine(entry.Line)
@@ -1000,5 +1002,5 @@ func detectLevelFromLogLine(log string) string {
 	if strings.Contains(log, "debug:") || strings.Contains(log, "DEBUG:") {
 		return logLevelDebug
 	}
-	return logLevelUnknown
+	return LogLevelUnknown
 }

pkg/distributor/distributor_test.go

@@ -1380,6 +1380,10 @@ type mockTee struct {
 	tenant     string
 }
 
+func (m *mockTee) Stop() {
+	// no-op
+}
+
 func (mt *mockTee) Duplicate(tenant string, streams []KeyedStream) {
 	mt.mu.Lock()
 	defer mt.mu.Unlock()
@@ -1485,7 +1489,7 @@ func Test_DetectLogLevels(t *testing.T) {
 		require.Equal(t, `{foo="bar"}`, topVal.Streams[0].Labels)
 		require.Equal(t, push.LabelsAdapter{
 			{
-				Name:  levelLabel,
+				Name:  LevelLabel,
 				Value: logLevelWarn,
 			},
 		}, topVal.Streams[0].Entries[0].StructuredMetadata)
@@ -1502,7 +1506,7 @@ func Test_DetectLogLevels(t *testing.T) {
 		require.Equal(t, `{foo="bar", level="debug"}`, topVal.Streams[0].Labels)
 		sm := topVal.Streams[0].Entries[0].StructuredMetadata
 		require.Len(t, sm, 1)
-		require.Equal(t, sm[0].Name, levelLabel)
+		require.Equal(t, sm[0].Name, LevelLabel)
 		require.Equal(t, sm[0].Value, logLevelDebug)
 	})
 
@@ -1527,7 +1531,7 @@ func Test_DetectLogLevels(t *testing.T) {
 				Name:  "severity",
 				Value: logLevelWarn,
 			}, {
-				Name:  levelLabel,
+				Name:  LevelLabel,
 				Value: logLevelWarn,
 			},
 		}, sm)
@@ -1570,7 +1574,7 @@ func Test_detectLogLevelFromLogEntry(t *testing.T) {
 			entry: logproto.Entry{
 				Line: "foo",
 			},
-			expectedLogLevel: logLevelUnknown,
+			expectedLogLevel: LogLevelUnknown,
 		},
 		{
 			name: "non otlp with log level keywords in log line",
@@ -1654,7 +1658,7 @@ func Test_detectLogLevelFromLogEntry(t *testing.T) {
 			entry: logproto.Entry{
 				Line: `foo=bar msg="message with keyword but it should not get picked up" level=NA`,
 			},
-			expectedLogLevel: logLevelUnknown,
+			expectedLogLevel: LogLevelUnknown,
 		},
 		{
 			name: "logfmt log line with label Severity is allowed for level detection",
@@ -1707,7 +1711,7 @@ func Benchmark_extractLogLevelFromLogLine(b *testing.B) {
 
 	for i := 0; i < b.N; i++ {
 		level := extractLogLevelFromLogLine(logLine)
-		require.Equal(b, logLevelUnknown, level)
+		require.Equal(b, LogLevelUnknown, level)
 	}
 }
 

pkg/distributor/tee.go

@@ -3,6 +3,7 @@ package distributor
 // Tee implementations can duplicate the log streams to another endpoint.
 type Tee interface {
 	Duplicate(tenant string, streams []KeyedStream)
+	Stop()
 }
 
 // WrapTee wraps a new Tee around an existing Tee.
@@ -25,3 +26,9 @@ func (m *multiTee) Duplicate(tenant string, streams []KeyedStream) {
 		tee.Duplicate(tenant, streams)
 	}
 }
+
+func (m *multiTee) Stop() {
+	for _, tee := range m.tees {
+		tee.Stop()
+	}
+}

pkg/distributor/tee_test.go

@@ -12,6 +12,10 @@ type mockedTee struct {
 	mock.Mock
 }
 
+func (m *mockedTee) Stop() {
+	// no-op
+}
+
 func (m *mockedTee) Duplicate(tenant string, streams []KeyedStream) {
 	m.Called(tenant, streams)
 }

pkg/distributor/validator.go

@@ -158,6 +158,11 @@ func (v Validator) ValidateLabels(ctx validationContext, ls labels.Labels, strea
 		return fmt.Errorf(validation.MissingLabelsErrorMsg)
 	}
 
+	// Skip validation for aggregated metric streams, as we create those for internal use
+	if ls.Has(push.AggregatedMetricLabel) {
+		return nil
+	}
+
 	numLabelNames := len(ls)
 	// This is a special case that's often added by the Loki infrastructure. It may result in allowing one extra label
 	// if incoming requests already have a service_name

pkg/ingester-rf1/tee.go

@@ -86,3 +86,7 @@ func (t *Tee) sendStream(tenant string, stream distributor.KeyedStream) error {
 	t.ingesterAppends.WithLabelValues(addr, "success").Inc()
 	return nil
 }
+
+func (t *Tee) Stop() {
+	// noop
+}