diff --git a/crates/fuzzing/src/oracles/engine.rs b/crates/fuzzing/src/oracles/engine.rs index 686858ea667d..0f5065af2bdc 100644 --- a/crates/fuzzing/src/oracles/engine.rs +++ b/crates/fuzzing/src/oracles/engine.rs @@ -6,36 +6,52 @@ use anyhow::Error; use arbitrary::Unstructured; use wasmtime::Trap; -/// Pick one of the engines implemented in this module that is compatible with -/// the Wasm features passed in `features` and, when fuzzing Wasmtime against -/// itself, an existing `wasmtime_engine`. +/// Pick one of the engines implemented in this module that is: +/// - in the list of `allowed` engines +/// - can evaluate Wasm modules compatible with `existing_config`. pub fn choose( u: &mut Unstructured<'_>, existing_config: &Config, + allowed: &[String], ) -> arbitrary::Result> { - // Filter out any engines that cannot match the given configuration. + // Filter out any engines that cannot match the `existing_config` or are not + // `allowed`. let mut engines: Vec> = vec![]; - let mut config2: WasmtimeConfig = u.arbitrary()?; // TODO change to WasmtimeConfig - config2.make_compatible_with(&existing_config.wasmtime); - let config2 = Config { - wasmtime: config2, - module_config: existing_config.module_config.clone(), - }; - if let Result::Ok(e) = WasmtimeEngine::new(config2) { - engines.push(Box::new(e)) + + if allowed.iter().any(|a| a == "wasmtime") { + let mut new_wasmtime_config: WasmtimeConfig = u.arbitrary()?; + new_wasmtime_config.make_compatible_with(&existing_config.wasmtime); + let new_config = Config { + wasmtime: new_wasmtime_config, + module_config: existing_config.module_config.clone(), + }; + if let Result::Ok(e) = WasmtimeEngine::new(new_config) { + engines.push(Box::new(e)) + } } - if let Result::Ok(e) = WasmiEngine::new(&existing_config.module_config) { - engines.push(Box::new(e)) + + if allowed.iter().any(|a| a == "wasmi") { + if let Result::Ok(e) = WasmiEngine::new(&existing_config.module_config) { + engines.push(Box::new(e)) + } } + #[cfg(feature = "fuzz-spec-interpreter")] - if let Result::Ok(e) = - crate::oracles::diff_spec::SpecInterpreter::new(&existing_config.module_config) - { - engines.push(Box::new(e)) + if allowed.iter().any(|a| a == "spec") { + if let Result::Ok(e) = + crate::oracles::diff_spec::SpecInterpreter::new(&existing_config.module_config) + { + engines.push(Box::new(e)) + } } + #[cfg(not(any(windows, target_arch = "s390x")))] - if let Result::Ok(e) = crate::oracles::diff_v8::V8Engine::new(&existing_config.module_config) { - engines.push(Box::new(e)) + if allowed.iter().any(|a| a == "v8") { + if let Result::Ok(e) = + crate::oracles::diff_v8::V8Engine::new(&existing_config.module_config) + { + engines.push(Box::new(e)) + } } // Use the input of the fuzzer to pick an engine that we'll be fuzzing @@ -93,6 +109,83 @@ pub fn setup_engine_runtimes() { crate::oracles::diff_spec::setup_ocaml_runtime(); } +/// Build a list of allowed values from the given `defaults` using the +/// `env_list`. +/// +/// ``` +/// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list; +/// // Passing no `env_list` returns the defaults: +/// assert_eq!(build_allowed_env_list(None, &["a"]), vec!["a"]); +/// // We can build up a subset of the defaults: +/// assert_eq!(build_allowed_env_list(Some(vec!["b".to_string()]), &["a","b"]), vec!["b"]); +/// // Alternately we can subtract from the defaults: +/// assert_eq!(build_allowed_env_list(Some(vec!["-a".to_string()]), &["a","b"]), vec!["b"]); +/// ``` +/// ```should_panic +/// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list; +/// // We are not allowed to mix set "addition" and "subtraction"; the following +/// // will panic: +/// build_allowed_env_list(Some(vec!["-a".to_string(), "b".to_string()]), &["a", "b"]); +/// ``` +pub fn build_allowed_env_list(env_list: Option>, defaults: &[&str]) -> Vec { + if let Some(configured) = &env_list { + let subtract_from_defaults = configured.iter().all(|c| c.starts_with("-")); + let add_from_defaults = configured.iter().all(|c| !c.starts_with("-")); + if subtract_from_defaults { + // Strip all "-" from subtraction strings; check that the values are + // valid. + let mut configured_stripped = Vec::with_capacity(configured.len()); + for c in configured { + let c = &c[1..]; + if !defaults.contains(&c) { + panic!( + "invalid environment configuration `{}`; must be one of: {:?}", + c, defaults + ); + } else { + configured_stripped.push(c); + } + } + // Allow all defaults that aren't subtracted. + let mut allowed = Vec::with_capacity(defaults.len()); + for &d in defaults { + if !configured_stripped.contains(&d) { + allowed.push(d.to_owned()); + } + } + allowed + } else if add_from_defaults { + // Allow all passed values if they are valid defaults. + let mut allowed = Vec::with_capacity(configured.len()); + for c in configured { + if defaults.contains(&&c[..]) { + allowed.push(c.to_owned()); + } else { + panic!( + "invalid environment configuration `{}`; must be one of: {:?}", + c, defaults + ); + } + } + allowed + } else { + panic!( + "all values must either subtract or add from defaults; found mixed values: {:?}", + &env_list + ); + } + } else { + defaults.iter().map(|&s| s.to_owned()).collect() + } +} + +/// Retrieve a comma-delimited list of values from an environment variable. +pub fn parse_env_list(env_variable: &str) -> Option> { + std::env::var(env_variable) + .ok() + .map(|l| l.split(",").map(|s| s.to_owned()).collect()) +} + #[cfg(test)] pub fn smoke_test_engine(mk_engine: impl Fn(Config) -> anyhow::Result) where diff --git a/fuzz/fuzz_targets/differential.rs b/fuzz/fuzz_targets/differential.rs index c80883543b68..dbf498fc3193 100644 --- a/fuzz/fuzz_targets/differential.rs +++ b/fuzz/fuzz_targets/differential.rs @@ -8,22 +8,46 @@ use std::sync::Once; use wasmtime::Trap; use wasmtime_fuzzing::generators::{Config, DiffValue, DiffValueType, SingleInstModule}; use wasmtime_fuzzing::oracles::diff_wasmtime::WasmtimeInstance; +use wasmtime_fuzzing::oracles::engine::{build_allowed_env_list, parse_env_list}; use wasmtime_fuzzing::oracles::{differential, engine, log_wasm}; // Upper limit on the number of invocations for each WebAssembly function // executed by this fuzz target. const NUM_INVOCATIONS: usize = 5; +// Only run once when the fuzz target loads. +static SETUP: Once = Once::new(); + +// Environment-specified configuration for controlling the kinds of engines and +// modules used by this fuzz target. E.g.: +// - ALLOWED_ENGINES=wasmi,spec cargo +nightly fuzz run ... +// - ALLOWED_ENGINES=-v8 cargo +nightly fuzz run ... +// - ALLOWED_MODULES=single-inst cargo +nightly fuzz run ... +static mut ALLOWED_ENGINES: Vec = vec![]; +static mut ALLOWED_MODULES: Vec = vec![]; + // Statistics about what's actually getting executed during fuzzing static STATS: RuntimeStats = RuntimeStats::new(); -// The spec interpreter requires special one-time setup. -static SETUP: Once = Once::new(); - fuzz_target!(|data: &[u8]| { - // To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on - // `setup_ocaml_runtime`. - SETUP.call_once(|| engine::setup_engine_runtimes()); + SETUP.call_once(|| { + // To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on + // `setup_ocaml_runtime`. + engine::setup_engine_runtimes(); + + // Retrieve the configuration for this fuzz target from `ALLOWED_*` + // environment variables. + unsafe { + ALLOWED_ENGINES = build_allowed_env_list( + parse_env_list("ALLOWED_ENGINES"), + &["wasmtime", "wasmi", "spec", "v8"], + ); + ALLOWED_MODULES = build_allowed_env_list( + parse_env_list("ALLOWED_MODULES"), + &["wasm-smith", "single-inst"], + ); + } + }); // Errors in `run` have to do with not enough input in `data`, which we // ignore here since it doesn't affect how we'd like to fuzz. @@ -37,20 +61,35 @@ fn run(data: &[u8]) -> Result<()> { let mut config: Config = u.arbitrary()?; config.set_differential_config(); - // Generate the Wasm module. - let wasm = if u.arbitrary()? { + // Generate the Wasm module; this is specified by either the ALLOWED_MODULES + // environment variable or a random selection between wasm-smith and + // single-inst. + let build_wasm_smith_module = |u: &mut Unstructured, config: &mut Config| -> Result<_> { STATS.wasm_smith_modules.fetch_add(1, SeqCst); - let module = config.generate(&mut u, Some(1000))?; - module.to_bytes() - } else { + let module = config.generate(u, Some(1000))?; + Ok(module.to_bytes()) + }; + let build_single_inst_module = |u: &mut Unstructured, config: &mut Config| -> Result<_> { STATS.single_instruction_modules.fetch_add(1, SeqCst); - let module = SingleInstModule::new(&mut u, &mut config.module_config)?; - module.to_bytes() + let module = SingleInstModule::new(u, &mut config.module_config)?; + Ok(module.to_bytes()) + }; + let wasm = match unsafe { &ALLOWED_MODULES.as_slice() } { + &[] => panic!("unable to generate a module to fuzz against; check `ALLOWED_MODULES`"), + &[n] if n == "wasm-smith" => build_wasm_smith_module(&mut u, &mut config)?, + &[n] if n == "single-inst" => build_single_inst_module(&mut u, &mut config)?, + _ => { + if u.arbitrary()? { + build_wasm_smith_module(&mut u, &mut config)? + } else { + build_single_inst_module(&mut u, &mut config)? + } + } }; log_wasm(&wasm); // Choose a left-hand side Wasm engine. - let mut lhs = engine::choose(&mut u, &config)?; + let mut lhs = engine::choose(&mut u, &config, unsafe { &ALLOWED_ENGINES })?; let lhs_instance = lhs.instantiate(&wasm); STATS.bump_engine(lhs.name());