Update quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs

quickwit/quickwit-control-plane/src/indexing_scheduler/mod.rs

@@ -127,7 +127,7 @@ impl fmt::Debug for IndexingScheduler {
 /// This function averages their statistics.
 ///
 /// For the moment, this function only takes in account the measured throughput,
-/// and assumes a constant CPU usage of 4 vCPU = 30mb/s.
+/// and assumes a constant CPU usage of 4 vCPU = 20mb/s.
 ///
 /// It does not take in account the variation that could raise from the different
 /// doc mapping / nature of the data, etc.

quickwit/quickwit-proto/src/indexing/mod.rs

@@ -22,6 +22,7 @@ use std::fmt::{Display, Formatter};
 use std::hash::Hash;
 use std::ops::{Add, Mul, Sub};
 
+use bytesize::ByteSize;
 use quickwit_actors::AskError;
 use quickwit_common::pubsub::Event;
 use quickwit_common::tower::{MakeLoadShedError, RpcName};
@@ -176,9 +177,18 @@ impl Display for PipelineMetrics {
 }
 
 /// One full pipeline (including merging) is assumed to consume 4 CPU threads.
-/// The actual number somewhere between 3 and 4.
+/// The actual number somewhere between 3 and 4. Quickwit is not super sensitive to this number.
+///
+/// It simply impacts the point where we prefer to work on balancing the load over the different
+/// indexers and the point where we prefer improving other feature of the system (shard locality,
+/// grouping pipelines associated to a given index on the same node, etc.).
 pub const PIPELINE_FULL_CAPACITY: CpuCapacity = CpuCapacity::from_cpu_millis(4_000u32);
 
+/// One full pipeline (including merging) is supposed to have the capacity to index at least 20mb/s.
+/// This is a defensive value: In reality, this is typically above 30mb/s.
+pub const PIPELINE_THROUGHTPUT: ByteSize = ByteSize::mb(20);
+
+
 /// The CpuCapacity represents an amount of CPU resource available.
 ///
 /// It is usually expressed in CPU millis (For instance, one full CPU thread is