Refactor benchmark module

src/app.rs

@@ -249,6 +249,12 @@ fn build_app() -> App<'static> {
                 .help("Give a meaningful name to a command. This can be specified multiple times \
                        if several commands are benchmarked."),
         )
+        .arg(
+            Arg::new("debug-mode")
+            .long("debug-mode")
+            .hide(true)
+            .help("Enable debug mode which does not actually run commands, but returns fake times when the command is 'sleep <time>'")
+        )
 }
 
 #[test]

src/benchmark/result.rs → src/benchmark/benchmark_result.rs

@@ -9,10 +9,10 @@ use crate::util::units::Second;
 // `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.
 #[derive(Debug, Default, Clone, Serialize, PartialEq)]
 pub struct BenchmarkResult {
-    /// The command that was run
+    /// The full command line of the program that is being benchmarked
     pub command: String,
 
-    /// The mean run time
+    /// The average run time
     pub mean: Second,
 
     /// The standard deviation of all run times. Not available if only one run has been performed
@@ -21,26 +21,26 @@ pub struct BenchmarkResult {
     /// The median run time
     pub median: Second,
 
-    /// Time spend in user space
+    /// Time spent in user mode
     pub user: Second,
 
-    /// Time spent in system space
+    /// Time spent in kernel mode
     pub system: Second,
 
-    /// Min time measured
+    /// Minimum of all measured times
     pub min: Second,
 
-    /// Max time measured
+    /// Maximum of all measured times
     pub max: Second,
 
     /// All run time measurements
     #[serde(skip_serializing_if = "Option::is_none")]
     pub times: Option<Vec<Second>>,
 
-    /// All run exit codes
+    /// Exit codes of all command invocations
     pub exit_codes: Vec<Option<i32>>,
 
-    /// Any parameter values used
+    /// Parameter values for this benchmark
     #[serde(skip_serializing_if = "BTreeMap::is_empty")]
     pub parameters: BTreeMap<String, String>,
 }

src/benchmark/executor.rs

@@ -0,0 +1,230 @@
+use std::process::{ExitStatus, Stdio};
+
+use crate::command::Command;
+use crate::options::{CmdFailureAction, CommandOutputPolicy, Options, OutputStyleOption, Shell};
+use crate::output::progress_bar::get_progress_bar;
+use crate::shell::execute_and_time;
+use crate::timer::wallclocktimer::WallClockTimer;
+use crate::timer::{TimerStart, TimerStop};
+use crate::util::units::Second;
+
+use super::timing_result::TimingResult;
+
+use anyhow::{bail, Result};
+use statistical::mean;
+
+pub trait Executor {
+    fn time_command(
+        &self,
+        command: &Command<'_>,
+        command_failure_action: Option<CmdFailureAction>,
+    ) -> Result<(TimingResult, ExitStatus)>;
+
+    fn calibrate(&mut self) -> Result<()>;
+
+    fn time_overhead(&self) -> Second;
+}
+
+pub struct ShellExecutor<'a> {
+    options: &'a Options,
+    shell: &'a Shell,
+    shell_spawning_time: Option<TimingResult>,
+}
+
+impl<'a> ShellExecutor<'a> {
+    /// Correct for shell spawning time
+    fn subtract_shell_spawning_time(&self, time: Second, shell_spawning_time: Second) -> Second {
+        if time < shell_spawning_time {
+            0.0
+        } else {
+            time - shell_spawning_time
+        }
+    }
+
+    pub fn new(shell: &'a Shell, options: &'a Options) -> Self {
+        ShellExecutor {
+            shell,
+            options,
+            shell_spawning_time: None,
+        }
+    }
+}
+
+impl<'a> Executor for ShellExecutor<'a> {
+    /// Run the given shell command and measure the execution time
+    fn time_command(
+        &self,
+        command: &Command<'_>,
+        command_failure_action: Option<CmdFailureAction>,
+    ) -> Result<(TimingResult, ExitStatus)> {
+        let (stdout, stderr) = match self.options.command_output_policy {
+            CommandOutputPolicy::Discard => (Stdio::null(), Stdio::null()),
+            CommandOutputPolicy::Forward => (Stdio::inherit(), Stdio::inherit()),
+        };
+
+        let wallclock_timer = WallClockTimer::start();
+        let result = execute_and_time(stdout, stderr, &command.get_shell_command(), &self.shell)?;
+        let mut time_real = wallclock_timer.stop();
+
+        let mut time_user = result.user_time;
+        let mut time_system = result.system_time;
+
+        if command_failure_action.unwrap_or(self.options.command_failure_action)
+            == CmdFailureAction::RaiseError
+            && !result.status.success()
+        {
+            bail!(
+                "{}. Use the '-i'/'--ignore-failure' option if you want to ignore this. \
+                Alternatively, use the '--show-output' option to debug what went wrong.",
+                result.status.code().map_or(
+                    "The process has been terminated by a signal".into(),
+                    |c| format!("Command terminated with non-zero exit code: {}", c)
+                )
+            );
+        }
+
+        // Subtract shell spawning time
+        if let Some(spawning_time) = self.shell_spawning_time {
+            time_real = self.subtract_shell_spawning_time(time_real, spawning_time.time_real);
+            time_user = self.subtract_shell_spawning_time(time_user, spawning_time.time_user);
+            time_system = self.subtract_shell_spawning_time(time_system, spawning_time.time_system);
+        }
+
+        Ok((
+            TimingResult {
+                time_real,
+                time_user,
+                time_system,
+            },
+            result.status,
+        ))
+    }
+
+    /// Measure the average shell spawning time
+    fn calibrate(&mut self) -> Result<()> {
+        const COUNT: u64 = 50;
+        let progress_bar = if self.options.output_style != OutputStyleOption::Disabled {
+            Some(get_progress_bar(
+                COUNT,
+                "Measuring shell spawning time",
+                self.options.output_style,
+            ))
+        } else {
+            None
+        };
+
+        let mut times_real: Vec<Second> = vec![];
+        let mut times_user: Vec<Second> = vec![];
+        let mut times_system: Vec<Second> = vec![];
+
+        for _ in 0..COUNT {
+            // Just run the shell without any command
+            let res = self.time_command(&Command::new(None, ""), None);
+
+            match res {
+                Err(_) => {
+                    let shell_cmd = if cfg!(windows) {
+                        format!("{} /C \"\"", self.shell)
+                    } else {
+                        format!("{} -c \"\"", self.shell)
+                    };
+
+                    bail!(
+                        "Could not measure shell execution time. Make sure you can run '{}'.",
+                        shell_cmd
+                    );
+                }
+                Ok((r, _)) => {
+                    times_real.push(r.time_real);
+                    times_user.push(r.time_user);
+                    times_system.push(r.time_system);
+                }
+            }
+
+            if let Some(bar) = progress_bar.as_ref() {
+                bar.inc(1)
+            }
+        }
+
+        if let Some(bar) = progress_bar.as_ref() {
+            bar.finish_and_clear()
+        }
+
+        self.shell_spawning_time = Some(TimingResult {
+            time_real: mean(&times_real),
+            time_user: mean(&times_user),
+            time_system: mean(&times_system),
+        });
+
+        Ok(())
+    }
+
+    fn time_overhead(&self) -> Second {
+        self.shell_spawning_time.unwrap().time_real
+    }
+}
+
+#[derive(Clone)]
+pub struct MockExecutor {
+    shell: Option<String>,
+}
+
+impl MockExecutor {
+    pub fn new(shell: Option<String>) -> Self {
+        MockExecutor { shell }
+    }
+
+    fn extract_time<S: AsRef<str>>(sleep_command: S) -> Second {
+        assert!(sleep_command.as_ref().starts_with("sleep "));
+        sleep_command
+            .as_ref()
+            .trim_start_matches("sleep ")
+            .parse::<Second>()
+            .unwrap()
+    }
+}
+
+impl Executor for MockExecutor {
+    fn time_command(
+        &self,
+        command: &Command<'_>,
+        _command_failure_action: Option<CmdFailureAction>,
+    ) -> Result<(TimingResult, ExitStatus)> {
+        #[cfg(unix)]
+        let status = {
+            use std::os::unix::process::ExitStatusExt;
+            ExitStatus::from_raw(0)
+        };
+
+        #[cfg(windows)]
+        let status = {
+            use std::os::windows::process::ExitStatusExt;
+            ExitStatus::from_raw(0)
+        };
+
+        Ok((
+            TimingResult {
+                time_real: Self::extract_time(command.get_shell_command()),
+                time_user: 0.0,
+                time_system: 0.0,
+            },
+            status,
+        ))
+    }
+
+    fn calibrate(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    fn time_overhead(&self) -> Second {
+        match &self.shell {
+            None => 0.0,
+            Some(shell) => Self::extract_time(shell),
+        }
+    }
+}
+
+#[test]
+fn test_mock_executor_extract_time() {
+    assert_eq!(MockExecutor::extract_time("sleep 0.1"), 0.1);
+}