[fuzz] Configure the differential target

This change is a follow-on from bytecodealliance#4515 to add the ability to configure
the `differential` fuzz target by limiting which engines and modules are
used for fuzzing. This is incredibly useful when troubleshooting, e.g.,
when an engine is more prone to failure, we can target that engine
exclusively. The effect of this configuration is visible in the
statistics now printed out from bytecodealliance#4739.

Engines are configured using the `ALLOWED_ENGINES` environment variable.
We can either subtract from the set of allowed engines (e.g.,
`ALLOWED_ENGINES=-v8`) or build up a set of allowed engines (e.g.,
`ALLOWED_ENGINES=wasmi,spec`), but not both at the same time.
`ALLOWED_ENGINES` only configures the left-hand side engine; the
right-hand side is always Wasmtime. When omitted, `ALLOWED_ENGINES`
defaults to [`wasmtime`, `wasmi`, `spec`, `v8`].

The generated WebAssembly modules are configured using
`ALLOWED_MODULES`. This environment variables works the same as above
but the available options are: [`wasm-smith`, `single-inst`].

crates/fuzzing/src/oracles/engine.rs

@@ -6,36 +6,52 @@ use anyhow::Error;
 use arbitrary::Unstructured;
 use wasmtime::Trap;
 
-/// Pick one of the engines implemented in this module that is compatible with
-/// the Wasm features passed in `features` and, when fuzzing Wasmtime against
-/// itself, an existing `wasmtime_engine`.
+/// Pick one of the engines implemented in this module that is:
+/// - in the list of `allowed` engines
+/// - can evaluate Wasm modules compatible with `existing_config`.
 pub fn choose(
     u: &mut Unstructured<'_>,
     existing_config: &Config,
+    allowed: &[String],
 ) -> arbitrary::Result<Box<dyn DiffEngine>> {
-    // Filter out any engines that cannot match the given configuration.
+    // Filter out any engines that cannot match the `existing_config` or are not
+    // `allowed`.
     let mut engines: Vec<Box<dyn DiffEngine>> = vec![];
-    let mut config2: WasmtimeConfig = u.arbitrary()?; // TODO change to WasmtimeConfig
-    config2.make_compatible_with(&existing_config.wasmtime);
-    let config2 = Config {
-        wasmtime: config2,
-        module_config: existing_config.module_config.clone(),
-    };
-    if let Result::Ok(e) = WasmtimeEngine::new(config2) {
-        engines.push(Box::new(e))
+
+    if allowed.iter().any(|a| a == "wasmtime") {
+        let mut new_wasmtime_config: WasmtimeConfig = u.arbitrary()?;
+        new_wasmtime_config.make_compatible_with(&existing_config.wasmtime);
+        let new_config = Config {
+            wasmtime: new_wasmtime_config,
+            module_config: existing_config.module_config.clone(),
+        };
+        if let Result::Ok(e) = WasmtimeEngine::new(new_config) {
+            engines.push(Box::new(e))
+        }
     }
-    if let Result::Ok(e) = WasmiEngine::new(&existing_config.module_config) {
-        engines.push(Box::new(e))
+
+    if allowed.iter().any(|a| a == "wasmi") {
+        if let Result::Ok(e) = WasmiEngine::new(&existing_config.module_config) {
+            engines.push(Box::new(e))
+        }
     }
+
     #[cfg(feature = "fuzz-spec-interpreter")]
-    if let Result::Ok(e) =
-        crate::oracles::diff_spec::SpecInterpreter::new(&existing_config.module_config)
-    {
-        engines.push(Box::new(e))
+    if allowed.iter().any(|a| a == "spec") {
+        if let Result::Ok(e) =
+            crate::oracles::diff_spec::SpecInterpreter::new(&existing_config.module_config)
+        {
+            engines.push(Box::new(e))
+        }
     }
+
     #[cfg(not(any(windows, target_arch = "s390x")))]
-    if let Result::Ok(e) = crate::oracles::diff_v8::V8Engine::new(&existing_config.module_config) {
-        engines.push(Box::new(e))
+    if allowed.iter().any(|a| a == "v8") {
+        if let Result::Ok(e) =
+            crate::oracles::diff_v8::V8Engine::new(&existing_config.module_config)
+        {
+            engines.push(Box::new(e))
+        }
     }
 
     // Use the input of the fuzzer to pick an engine that we'll be fuzzing
@@ -93,6 +109,83 @@ pub fn setup_engine_runtimes() {
     crate::oracles::diff_spec::setup_ocaml_runtime();
 }
 
+/// Build a list of allowed values from the given `defaults` using the
+/// `env_list`.
+///
+/// ```
+/// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list;
+/// // Passing no `env_list` returns the defaults:
+/// assert_eq!(build_allowed_env_list(None, &["a"]), vec!["a"]);
+/// // We can build up a subset of the defaults:
+/// assert_eq!(build_allowed_env_list(Some(vec!["b".to_string()]), &["a","b"]), vec!["b"]);
+/// // Alternately we can subtract from the defaults:
+/// assert_eq!(build_allowed_env_list(Some(vec!["-a".to_string()]), &["a","b"]), vec!["b"]);
+/// ```
+/// ```should_panic
+/// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list;
+/// // We are not allowed to mix set "addition" and "subtraction"; the following
+/// // will panic:
+/// build_allowed_env_list(Some(vec!["-a".to_string(), "b".to_string()]), &["a", "b"]);
+/// ```
+pub fn build_allowed_env_list(env_list: Option<Vec<String>>, defaults: &[&str]) -> Vec<String> {
+    if let Some(configured) = &env_list {
+        let subtract_from_defaults = configured.iter().all(|c| c.starts_with("-"));
+        let add_from_defaults = configured.iter().all(|c| !c.starts_with("-"));
+        if subtract_from_defaults {
+            // Strip all "-" from subtraction strings; check that the values are
+            // valid.
+            let mut configured_stripped = Vec::with_capacity(configured.len());
+            for c in configured {
+                let c = &c[1..];
+                if !defaults.contains(&c) {
+                    panic!(
+                        "invalid environment configuration `{}`; must be one of: {:?}",
+                        c, defaults
+                    );
+                } else {
+                    configured_stripped.push(c);
+                }
+            }
+            // Allow all defaults that aren't subtracted.
+            let mut allowed = Vec::with_capacity(defaults.len());
+            for &d in defaults {
+                if !configured_stripped.contains(&d) {
+                    allowed.push(d.to_owned());
+                }
+            }
+            allowed
+        } else if add_from_defaults {
+            // Allow all passed values if they are valid defaults.
+            let mut allowed = Vec::with_capacity(configured.len());
+            for c in configured {
+                if defaults.contains(&&c[..]) {
+                    allowed.push(c.to_owned());
+                } else {
+                    panic!(
+                        "invalid environment configuration `{}`; must be one of: {:?}",
+                        c, defaults
+                    );
+                }
+            }
+            allowed
+        } else {
+            panic!(
+                "all values must either subtract or add from defaults; found mixed values: {:?}",
+                &env_list
+            );
+        }
+    } else {
+        defaults.iter().map(|&s| s.to_owned()).collect()
+    }
+}
+
+/// Retrieve a comma-delimited list of values from an environment variable.
+pub fn parse_env_list(env_variable: &str) -> Option<Vec<String>> {
+    std::env::var(env_variable)
+        .ok()
+        .map(|l| l.split(",").map(|s| s.to_owned()).collect())
+}
+
 #[cfg(test)]
 pub fn smoke_test_engine<T>(mk_engine: impl Fn(Config) -> anyhow::Result<T>)
 where

fuzz/fuzz_targets/differential.rs

@@ -8,22 +8,46 @@ use std::sync::Once;
 use wasmtime::Trap;
 use wasmtime_fuzzing::generators::{Config, DiffValue, DiffValueType, SingleInstModule};
 use wasmtime_fuzzing::oracles::diff_wasmtime::WasmtimeInstance;
+use wasmtime_fuzzing::oracles::engine::{build_allowed_env_list, parse_env_list};
 use wasmtime_fuzzing::oracles::{differential, engine, log_wasm};
 
 // Upper limit on the number of invocations for each WebAssembly function
 // executed by this fuzz target.
 const NUM_INVOCATIONS: usize = 5;
 
+// Only run once when the fuzz target loads.
+static SETUP: Once = Once::new();
+
+// Environment-specified configuration for controlling the kinds of engines and
+// modules used by this fuzz target. E.g.:
+// - ALLOWED_ENGINES=wasmi,spec cargo +nightly fuzz run ...
+// - ALLOWED_ENGINES=-v8 cargo +nightly fuzz run ...
+// - ALLOWED_MODULES=single-inst cargo +nightly fuzz run ...
+static mut ALLOWED_ENGINES: Vec<String> = vec![];
+static mut ALLOWED_MODULES: Vec<String> = vec![];
+
 // Statistics about what's actually getting executed during fuzzing
 static STATS: RuntimeStats = RuntimeStats::new();
 
-// The spec interpreter requires special one-time setup.
-static SETUP: Once = Once::new();
-
 fuzz_target!(|data: &[u8]| {
-    // To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on
-    // `setup_ocaml_runtime`.
-    SETUP.call_once(|| engine::setup_engine_runtimes());
+    SETUP.call_once(|| {
+        // To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on
+        // `setup_ocaml_runtime`.
+        engine::setup_engine_runtimes();
+
+        // Retrieve the configuration for this fuzz target from `ALLOWED_*`
+        // environment variables.
+        unsafe {
+            ALLOWED_ENGINES = build_allowed_env_list(
+                parse_env_list("ALLOWED_ENGINES"),
+                &["wasmtime", "wasmi", "spec", "v8"],
+            );
+            ALLOWED_MODULES = build_allowed_env_list(
+                parse_env_list("ALLOWED_MODULES"),
+                &["wasm-smith", "single-inst"],
+            );
+        }
+    });
 
     // Errors in `run` have to do with not enough input in `data`, which we
     // ignore here since it doesn't affect how we'd like to fuzz.
@@ -37,20 +61,35 @@ fn run(data: &[u8]) -> Result<()> {
     let mut config: Config = u.arbitrary()?;
     config.set_differential_config();
 
-    // Generate the Wasm module.
-    let wasm = if u.arbitrary()? {
+    // Generate the Wasm module; this is specified by either the ALLOWED_MODULES
+    // environment variable or a random selection between wasm-smith and
+    // single-inst.
+    let build_wasm_smith_module = |u: &mut Unstructured, config: &mut Config| -> Result<_> {
         STATS.wasm_smith_modules.fetch_add(1, SeqCst);
-        let module = config.generate(&mut u, Some(1000))?;
-        module.to_bytes()
-    } else {
+        let module = config.generate(u, Some(1000))?;
+        Ok(module.to_bytes())
+    };
+    let build_single_inst_module = |u: &mut Unstructured, config: &mut Config| -> Result<_> {
         STATS.single_instruction_modules.fetch_add(1, SeqCst);
-        let module = SingleInstModule::new(&mut u, &mut config.module_config)?;
-        module.to_bytes()
+        let module = SingleInstModule::new(u, &mut config.module_config)?;
+        Ok(module.to_bytes())
+    };
+    let wasm = match unsafe { &ALLOWED_MODULES.as_slice() } {
+        &[] => panic!("unable to generate a module to fuzz against; check `ALLOWED_MODULES`"),
+        &[n] if n == "wasm-smith" => build_wasm_smith_module(&mut u, &mut config)?,
+        &[n] if n == "single-inst" => build_single_inst_module(&mut u, &mut config)?,
+        _ => {
+            if u.arbitrary()? {
+                build_wasm_smith_module(&mut u, &mut config)?
+            } else {
+                build_single_inst_module(&mut u, &mut config)?
+            }
+        }
     };
     log_wasm(&wasm);
 
     // Choose a left-hand side Wasm engine.
-    let mut lhs = engine::choose(&mut u, &config)?;
+    let mut lhs = engine::choose(&mut u, &config, unsafe { &ALLOWED_ENGINES })?;
     let lhs_instance = lhs.instantiate(&wasm);
     STATS.bump_engine(lhs.name());