Increase flush_op_timeout to 10m, 10s is often much too short for a b…

…ig WAL replay which can generate a lot of chunks very quickly. (#5768) Signed-off-by: Ed Welch <edward.welch@grafana.com>
grafana · Apr 5, 2022 · b136d0d · b136d0d
1 parent a9ac01b
commit b136d0d
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/docs/sources/configuration/_index.md b/docs/sources/configuration/_index.md
@@ -1109,7 +1109,7 @@ lifecycler:
 
 # The timeout before a flush is cancelled
 # CLI flag: -ingester.flush-op-timeout
-[flush_op_timeout: <duration> | default = 10s]
+[flush_op_timeout: <duration> | default = 10m]
 
 # How long chunks should be retained in-memory after they've been flushed.
 # CLI flag: -ingester.chunks-retain-period

diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go
@@ -111,7 +111,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&cfg.MaxTransferRetries, "ingester.max-transfer-retries", 0, "Number of times to try and transfer chunks before falling back to flushing. If set to 0 or negative value, transfers are disabled.")
 	f.IntVar(&cfg.ConcurrentFlushes, "ingester.concurrent-flushes", 32, "")
 	f.DurationVar(&cfg.FlushCheckPeriod, "ingester.flush-check-period", 30*time.Second, "")
-	f.DurationVar(&cfg.FlushOpTimeout, "ingester.flush-op-timeout", 10*time.Second, "")
+	f.DurationVar(&cfg.FlushOpTimeout, "ingester.flush-op-timeout", 10*time.Minute, "")
 	f.DurationVar(&cfg.RetainPeriod, "ingester.chunks-retain-period", 0, "")
 	f.DurationVar(&cfg.MaxChunkIdle, "ingester.chunks-idle-period", 30*time.Minute, "")
 	f.IntVar(&cfg.BlockSize, "ingester.chunks-block-size", 256*1024, "")

diff --git a/tools/dev/loki-boltdb-storage-s3/config/loki.yaml b/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
@@ -37,7 +37,7 @@ ingester:
     chunk_idle_period: 15m
     chunk_retain_period: 6m
     chunk_target_size: 1.572864e+06
-    flush_op_timeout: 30s
+    flush_op_timeout: 10m
     lifecycler:
         heartbeat_period: 10s
         interface_names: