diff --git a/.circleci/config.yml b/.circleci/config.yml index 398b83fed2..5e5517d98d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,5 +10,6 @@ workflows: jobs: - path-filtering/filter: mapping: | + hivesim-rs/.* hivesim-rs-ci true simulators/portal/.* rust-ci true base-revision: origin/master diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index d7c185cb9c..c284de4e20 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -6,6 +6,9 @@ parameters: rust-ci: type: boolean default: false + hivesim-rs-ci: + type: boolean + default: false jobs: # This job builds the hive executable and stores it in the workspace. @@ -73,6 +76,29 @@ jobs: name: "Compile Go simulators" command: ".circleci/compile-simulators.sh" # this makes sure the rust code is good + hivesim-rs: + docker: + - image: cimg/rust:1.75.0 + steps: + - checkout + - run: + name: Install rustfmt + command: rustup component add rustfmt + - run: + name: Install Clippy + command: rustup component add clippy + - run: + name: Install Clang + command: sudo apt update && sudo apt-get install clang -y + - run: + name: "Lint" + command: "cd hivesim-rs && cargo fmt --all -- --check" + - run: + name: "Build" + command: "cd hivesim-rs && cargo clippy --all --all-targets --all-features --no-deps -- --deny warnings" + - run: + name: "Test hivesim-rs" + command: "cd hivesim-rs && cargo test --workspace -- --nocapture" rust-simulators: docker: - image: cimg/rust:1.75.0 @@ -100,7 +126,11 @@ workflows: requires: ["build"] - smoke-tests-remote-docker: requires: ["build"] - rust-jobs: + rust-simulator-jobs: when: << pipeline.parameters.rust-ci >> jobs: - rust-simulators + hivesim-rs-jobs: + when: << pipeline.parameters.hivesim-rs-ci >> + jobs: + - hivesim-rs diff --git a/.gitignore b/.gitignore index eead5cce5d..9335372ced 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ workspace # build output /hive -# build output for rust simulators files +# build output for rust simulators and hivesim-rs files simulators/**/Cargo.lock simulators/**/target +hivesim-rs/Cargo.lock +hivesim-rs/target diff --git a/hivesim-rs/Cargo.toml b/hivesim-rs/Cargo.toml new file mode 100644 index 0000000000..0e1f1e7622 --- /dev/null +++ b/hivesim-rs/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "hivesim" +version = "0.1.0-alpha.1" +authors = ["Kolby ML (Moroz Liebl) ", "Ognyan Genev "] +edition = "2021" + + +[dependencies] +async-trait = "0.1.59" +dyn-clone = "1.0.11" +jsonrpsee = {version="0.20.0", features = ["client"]} +regex = "1.10.5" +reqwest = { version = "0.11.12", default-features = false, features = ["json", "multipart"] } +serde = { version = "1.0.147", features = ["derive"] } +serde_json = "1.0.87" +tokio = { version = "1", features = ["full"] } diff --git a/hivesim-rs/src/lib.rs b/hivesim-rs/src/lib.rs new file mode 100644 index 0000000000..bc842a9fb3 --- /dev/null +++ b/hivesim-rs/src/lib.rs @@ -0,0 +1,12 @@ +#![allow(dead_code)] +#![warn(clippy::unwrap_used)] +mod macros; +mod simulation; +mod testapi; +mod testmatch; +pub mod types; +pub mod utils; + +pub use simulation::Simulation; +pub use testapi::{run_suite, Client, NClientTestSpec, Suite, Test, TestSpec}; +pub use testmatch::TestMatcher; diff --git a/hivesim-rs/src/macros.rs b/hivesim-rs/src/macros.rs new file mode 100644 index 0000000000..397998e968 --- /dev/null +++ b/hivesim-rs/src/macros.rs @@ -0,0 +1,20 @@ +#[macro_export] +macro_rules! dyn_async {( + $( #[$attr:meta] )* // includes doc strings + $pub:vis + async + fn $fname:ident<$lt:lifetime> ( $($args:tt)* ) $(-> $Ret:ty)? + { + $($body:tt)* + } +) => ( + $( #[$attr] )* + #[allow(unused_parens)] + $pub + fn $fname<$lt> ( $($args)* ) -> ::std::pin::Pin<::std::boxed::Box< + dyn $lt + Send + ::std::future::Future + >> + { + Box::pin(async move { $($body)* }) + } +)} diff --git a/hivesim-rs/src/simulation.rs b/hivesim-rs/src/simulation.rs new file mode 100644 index 0000000000..d934880280 --- /dev/null +++ b/hivesim-rs/src/simulation.rs @@ -0,0 +1,178 @@ +use crate::types::{ClientDefinition, StartNodeResponse, SuiteID, TestID, TestRequest, TestResult}; +use crate::TestMatcher; +use std::collections::HashMap; +use std::env; +use std::net::IpAddr; +use std::str::FromStr; + +/// Wraps the simulation HTTP API provided by hive. +#[derive(Clone, Debug)] +pub struct Simulation { + pub url: String, + pub test_matcher: Option, +} + +impl Default for Simulation { + fn default() -> Self { + Self::new() + } +} + +// A struct in the structure of the JSON config shown in simulators.md +// it is used to pass information to the Hive Simulators +#[derive(serde::Serialize, serde::Deserialize)] +struct SimulatorConfig { + client: String, + environment: HashMap, +} + +impl SimulatorConfig { + pub fn new() -> Self { + Self { + client: "".to_string(), + environment: Default::default(), + } + } +} + +impl Simulation { + /// New looks up the hive host URI using the HIVE_SIMULATOR environment variable + /// and connects to it. It will panic if HIVE_SIMULATOR is not set. + pub fn new() -> Self { + let url = env::var("HIVE_SIMULATOR").expect("HIVE_SIMULATOR environment variable not set"); + let test_matcher = match env::var("HIVE_TEST_PATTERN") { + Ok(pattern) => { + if pattern.is_empty() { + None + } else { + Some(TestMatcher::new(&pattern)) + } + } + Err(_) => None, + }; + + if url.is_empty() { + panic!("HIVE_SIMULATOR environment variable is empty") + } + + Self { url, test_matcher } + } + + pub async fn start_suite( + &self, + name: String, + description: String, + _sim_log: String, + ) -> SuiteID { + let url = format!("{}/testsuite", self.url); + let client = reqwest::Client::new(); + let body = TestRequest { name, description }; + + client + .post(url) + .json(&body) + .send() + .await + .expect("Failed to send start suite request") + .json::() + .await + .expect("Failed to convert start suite response to json") + } + + pub async fn end_suite(&self, test_suite: SuiteID) { + let url = format!("{}/testsuite/{}", self.url, test_suite); + let client = reqwest::Client::new(); + client + .delete(url) + .send() + .await + .expect("Failed to send an end suite request"); + } + + /// Starts a new test case, returning the testcase id as a context identifier + pub async fn start_test( + &self, + test_suite: SuiteID, + name: String, + description: String, + ) -> TestID { + let url = format!("{}/testsuite/{}/test", self.url, test_suite); + let client = reqwest::Client::new(); + let body = TestRequest { name, description }; + + client + .post(url) + .json(&body) + .send() + .await + .expect("Failed to send start test request") + .json::() + .await + .expect("Failed to convert start test response to json") + } + + /// Finishes the test case, cleaning up everything, logging results, and returning + /// an error if the process could not be completed. + pub async fn end_test(&self, test_suite: SuiteID, test: TestID, test_result: TestResult) { + let url = format!("{}/testsuite/{}/test/{}", self.url, test_suite, test); + let client = reqwest::Client::new(); + + client + .post(url) + .json(&test_result) + .send() + .await + .expect("Failed to send end test request"); + } + + /// Starts a new node (or other container). + /// Returns container id and ip. + pub async fn start_client( + &self, + test_suite: SuiteID, + test: TestID, + client_type: String, + environment: Option>, + ) -> (String, IpAddr) { + let url = format!("{}/testsuite/{}/test/{}/node", self.url, test_suite, test); + let client = reqwest::Client::new(); + + let mut config = SimulatorConfig::new(); + config.client = client_type; + if let Some(environment) = environment { + config.environment = environment; + } + + let config = serde_json::to_string(&config).expect("Failed to parse config to serde_json"); + let form = reqwest::multipart::Form::new().text("config", config); + + let resp = client + .post(url) + .multipart(form) + .send() + .await + .expect("Failed to send start client request") + .json::() + .await + .expect("Failed to convert start node response to json"); + + let ip = IpAddr::from_str(&resp.ip).expect("Failed to decode IpAddr from string"); + + (resp.id, ip) + } + + /// Returns all client types available to this simulator run. This depends on + /// both the available client set and the command line filters. + pub async fn client_types(&self) -> Vec { + let url = format!("{}/clients", self.url); + let client = reqwest::Client::new(); + client + .get(&url) + .send() + .await + .expect("Failed to send get client types request") + .json::>() + .await + .expect("Failed to convert client types response to json") + } +} diff --git a/hivesim-rs/src/testapi.rs b/hivesim-rs/src/testapi.rs new file mode 100644 index 0000000000..baf0ae1d07 --- /dev/null +++ b/hivesim-rs/src/testapi.rs @@ -0,0 +1,312 @@ +use crate::types::{ClientDefinition, SuiteID, TestData, TestID, TestResult}; +use crate::{simulation, Simulation}; +use ::std::{boxed::Box, future::Future, pin::Pin}; +use async_trait::async_trait; +use core::fmt::Debug; +use dyn_clone::DynClone; +use jsonrpsee::http_client::{HttpClient, HttpClientBuilder}; +use std::collections::HashMap; +use std::net::IpAddr; + +use crate::utils::extract_test_results; + +pub type AsyncTestFunc = fn( + &mut Test, + Option, +) -> Pin< + Box< + dyn Future // future API / pollable + + Send // required by non-single-threaded executors + + '_, + >, +>; + +pub type AsyncNClientsTestFunc = fn( + Vec, + Option, +) -> Pin< + Box< + dyn Future // future API / pollable + + Send // required by non-single-threaded executors + + 'static, + >, +>; + +#[async_trait] +pub trait Testable: DynClone + Send + Sync { + async fn run_test(&self, simulation: Simulation, suite_id: SuiteID, suite: Suite); +} + +impl Debug for dyn Testable { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "Testable") + } +} + +dyn_clone::clone_trait_object!(Testable); +/// Description of a test suite +#[derive(Clone, Debug)] +pub struct Suite { + pub name: String, + pub description: String, + pub tests: Vec>, +} + +impl Suite { + pub fn add(&mut self, test: T) { + self.tests.push(Box::new(test)) + } +} + +/// Represents a running client. +#[derive(Debug, Clone)] +pub struct Client { + pub kind: String, + pub container: String, + pub ip: IpAddr, + pub rpc: HttpClient, + pub test: Test, +} + +#[derive(Clone, Debug)] +pub struct TestRun { + pub suite_id: SuiteID, + pub suite: Suite, + pub name: String, + pub desc: String, + pub always_run: bool, +} + +/// A running test +#[derive(Clone, Debug)] +pub struct Test { + pub sim: Simulation, + pub test_id: TestID, + pub suite: Suite, + pub suite_id: SuiteID, + pub result: TestResult, +} + +impl Test { + pub async fn start_client( + &self, + client_type: String, + environment: Option>, + ) -> Client { + let (container, ip) = self + .sim + .start_client( + self.suite_id, + self.test_id, + client_type.clone(), + environment, + ) + .await; + + let rpc_url = format!("http://{}:8545", ip); + + let rpc_client = HttpClientBuilder::default() + .build(rpc_url) + .expect("Failed to build rpc_client"); + + Client { + kind: client_type, + container, + ip, + rpc: rpc_client, + test: Test { + sim: self.sim.clone(), + test_id: self.test_id, + suite: self.suite.clone(), + suite_id: self.suite_id, + result: self.result.clone(), + }, + } + } + + /// Runs a subtest of this test. + pub async fn run(&self, spec: impl Testable) { + spec.run_test(self.sim.clone(), self.suite_id, self.suite.clone()) + .await + } +} + +#[derive(Clone)] +pub struct TestSpec { + // These fields are displayed in the UI. Be sure to add + // a meaningful description here. + pub name: String, + pub description: String, + // If AlwaysRun is true, the test will run even if Name does not match the test + // pattern. This option is useful for tests that launch a client instance and + // then perform further tests against it. + pub always_run: bool, + // The Run function is invoked when the test executes. + pub run: AsyncTestFunc, + pub client: Option, +} + +#[async_trait] +impl Testable for TestSpec { + async fn run_test(&self, simulation: Simulation, suite_id: SuiteID, suite: Suite) { + if let Some(test_match) = simulation.test_matcher.clone() { + if !self.always_run && !test_match.match_test(&suite.name, &self.name) { + return; + } + } + + let test_run = TestRun { + suite_id, + suite, + name: self.name.to_owned(), + desc: self.description.to_owned(), + always_run: self.always_run, + }; + + run_test(simulation, test_run, self.client.clone(), self.run).await; + } +} + +pub async fn run_test( + host: Simulation, + test: TestRun, + client: Option, + func: AsyncTestFunc, +) { + // Register test on simulation server and initialize the T. + let test_id = host.start_test(test.suite_id, test.name, test.desc).await; + let suite_id = test.suite_id; + + // run test function + let cloned_host = host.clone(); + + let test_result = extract_test_results( + tokio::spawn(async move { + let test = &mut Test { + sim: cloned_host, + test_id, + suite: test.suite, + suite_id, + result: Default::default(), + }; + + test.result.pass = true; + + // run test function + (func)(test, client).await; + }) + .await, + ); + + host.end_test(suite_id, test_id, test_result).await; +} + +#[derive(Clone)] +pub struct NClientTestSpec { + /// These fields are displayed in the UI. Be sure to add + /// a meaningful description here. + pub name: String, + pub description: String, + /// If AlwaysRun is true, the test will run even if Name does not match the test + /// pattern. This option is useful for tests that launch a client instance and + /// then perform further tests against it. + pub always_run: bool, + /// The Run function is invoked when the test executes. + pub run: AsyncNClientsTestFunc, + /// For each client, there is a distinct map of Hive Environment Variable names to values. + /// The environments must be in the same order as the `clients` + pub environments: Option>>>, + /// test data which can be passed to the test + pub test_data: Option, + pub clients: Vec, +} + +#[async_trait] +impl Testable for NClientTestSpec { + async fn run_test(&self, simulation: Simulation, suite_id: SuiteID, suite: Suite) { + if let Some(test_match) = simulation.test_matcher.clone() { + if !self.always_run && !test_match.match_test(&suite.name, &self.name) { + return; + } + } + + let test_run = TestRun { + suite_id, + suite, + name: self.name.to_owned(), + desc: self.description.to_owned(), + always_run: self.always_run, + }; + + run_n_client_test( + simulation, + test_run, + self.environments.to_owned(), + self.test_data.to_owned(), + self.clients.to_owned(), + self.run, + ) + .await; + } +} + +// Write a test that runs against N clients. +async fn run_n_client_test( + host: Simulation, + test: TestRun, + environments: Option>>>, + test_data: Option, + clients: Vec, + func: AsyncNClientsTestFunc, +) { + // Register test on simulation server and initialize the T. + let test_id = host.start_test(test.suite_id, test.name, test.desc).await; + let suite_id = test.suite_id; + + // run test function + let cloned_host = host.clone(); + let test_result = extract_test_results( + tokio::spawn(async move { + let test = &mut Test { + sim: cloned_host, + test_id, + suite: test.suite, + suite_id, + result: Default::default(), + }; + + test.result.pass = true; + + let mut client_vec: Vec = Vec::new(); + let env_iter = environments.unwrap_or(vec![None; clients.len()]); + for (client, environment) in clients.into_iter().zip(env_iter) { + client_vec.push(test.start_client(client.name.to_owned(), environment).await); + } + (func)(client_vec, test_data).await; + }) + .await, + ); + + host.end_test(suite_id, test_id, test_result).await; +} + +pub async fn run_suite(host: Simulation, suites: Vec) { + for suite in suites { + if let Some(test_match) = host.test_matcher.clone() { + if !test_match.match_test(&suite.name, "") { + continue; + } + } + + let name = suite.clone().name; + let description = suite.clone().description; + + let suite_id = host.start_suite(name, description, "".to_string()).await; + + for test in &suite.tests { + test.run_test(host.clone(), suite_id, suite.clone()).await; + } + + host.end_suite(suite_id).await; + } +} diff --git a/hivesim-rs/src/testmatch.rs b/hivesim-rs/src/testmatch.rs new file mode 100644 index 0000000000..c28dff275d --- /dev/null +++ b/hivesim-rs/src/testmatch.rs @@ -0,0 +1,157 @@ +use regex::Regex; + +#[derive(Clone, Debug)] +pub struct TestMatcher { + pub suite: Regex, + pub test: Regex, + pub pattern: String, +} + +impl TestMatcher { + pub fn new(pattern: &str) -> Self { + let parts = Self::split_regexp(pattern); + let suite = Regex::new(&format!("(?i:{})", parts[0])).unwrap(); + let test = if parts.len() > 1 { + Regex::new(&format!("(?i:{})", parts[1..].join("/"))).unwrap() + } else { + Regex::new("").unwrap() + }; + Self { + suite, + test, + pattern: pattern.to_string(), + } + } + + pub fn match_test(&self, suite: &str, test: &str) -> bool { + if !self.suite.is_match(suite) { + return false; + } + + if test != "" && !self.test.is_match(test) { + return false; + } + + true + } + + /// split_regexp splits the pattern into /-separated parts. + /// This is based off the golang implementation of testmatch.rs + fn split_regexp(pattern: &str) -> Vec<&str> { + let mut pattern = pattern; + let mut parts = Vec::with_capacity(pattern.matches('/').count()); + let mut square_bracket_counter = 0; + let mut parenthesis_counter = 0; + let mut index = 0; + while index < pattern.len() { + match pattern.chars().nth(index).unwrap() { + '[' => square_bracket_counter += 1, + ']' => { + if square_bracket_counter > 0 { + square_bracket_counter -= 1; + } + } + '(' => { + if square_bracket_counter == 0 { + parenthesis_counter += 1; + } + } + ')' => { + if square_bracket_counter == 0 { + parenthesis_counter -= 1; + } + } + '\\' => { + index += 1; + } + '/' => { + if square_bracket_counter == 0 && parenthesis_counter == 0 { + parts.push(&pattern[..index]); + pattern = &pattern[index + 1..]; + index = 0; + continue; + } + } + _ => {} + } + index += 1; + } + parts.push(pattern); + parts + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_regexp() { + let pattern = "suite/test"; + let parts = TestMatcher::split_regexp(pattern); + assert_eq!(parts, vec!["suite", "test"]); + + let pattern = "suite/test/1"; + let parts = TestMatcher::split_regexp(pattern); + assert_eq!(parts, vec!["suite", "test", "1"]); + + let pattern = "suite/test/1/2"; + let parts = TestMatcher::split_regexp(pattern); + assert_eq!(parts, vec!["suite", "test", "1", "2"]); + + let pattern = "suite/test/1/2/3"; + let parts = TestMatcher::split_regexp(pattern); + assert_eq!(parts, vec!["suite", "test", "1", "2", "3"]); + + let pattern = "suite/test/1/2/3/4"; + let parts = TestMatcher::split_regexp(pattern); + assert_eq!(parts, vec!["suite", "test", "1", "2", "3", "4"]); + + let pattern = "suite/test/1/2/3/4/5"; + let parts = TestMatcher::split_regexp(pattern); + assert_eq!(parts, vec!["suite", "test", "1", "2", "3", "4", "5"]); + + let pattern = "suite/test/1/2/3/4/5/6"; + let parts = TestMatcher::split_regexp(pattern); + assert_eq!(parts, vec!["suite", "test", "1", "2", "3", "4", "5", "6"]); + + let pattern = "suite/test/1/2/3/4/5/6/7"; + let parts = TestMatcher::split_regexp(pattern); + assert_eq!( + parts, + vec!["suite", "test", "1", "2", "3", "4", "5", "6", "7"] + ); + } + + #[test] + fn test_match_test() { + let matcher = TestMatcher::new("sim/test"); + + assert!(matcher.match_test("sim", "test")); + assert!(matcher.match_test("Sim", "Test")); + assert!(matcher.match_test("Sim", "TestTest")); + assert!(!matcher.match_test("Sim", "Tst"), false); + + let matcher = TestMatcher::new("/test"); + + assert!(matcher.match_test("sim", "test")); + assert!(matcher.match_test("", "Test")); + assert!(matcher.match_test("", "aTesta")); + assert!(matcher.match_test("bob", "test")); + + let matcher = TestMatcher::new("/GetEnr"); + assert!(matcher.match_test("history-rpc-compat", "portal_historyGetEnr Local Enr"),); + } + + #[test] + fn test_match_suite() { + let matcher = TestMatcher::new("sim"); + + assert!(matcher.match_test("sim", "")); + assert!(matcher.match_test("Sim", "")); + assert!(matcher.match_test("Sim", "Test")); + assert!(matcher.match_test("Sim", "Tst")); + assert!(matcher.match_test("Sim", "Tst")); + assert!(matcher.match_test("Sim", "Tst")); + } +} diff --git a/hivesim-rs/src/types.rs b/hivesim-rs/src/types.rs new file mode 100644 index 0000000000..66cf36e74b --- /dev/null +++ b/hivesim-rs/src/types.rs @@ -0,0 +1,77 @@ +use serde::{Deserialize, Serialize}; + +pub type SuiteID = u32; +pub type TestID = u32; + +/// StartNodeReponse is returned by the client startup endpoint. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct StartNodeResponse { + pub id: String, // Container ID. + pub ip: String, // IP address in bridge network +} + +// ClientMetadata is part of the ClientDefinition and lists metadata +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ClientMetadata { + pub roles: Vec, +} + +// ClientDefinition is served by the /clients API endpoint to list the available clients +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ClientDefinition { + pub name: String, + pub version: String, + pub meta: ClientMetadata, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TestRequest { + pub name: String, + pub description: String, +} + +/// Describes the outcome of a test. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct TestResult { + pub pass: bool, + pub details: String, +} + +#[derive(Clone, Debug)] +pub struct ContentKeyValue { + pub key: String, + pub value: String, +} + +#[derive(Clone, Debug)] +pub struct ContentKeyOfferLookupValues { + pub key: String, + pub offer_value: String, + pub lookup_value: String, +} + +#[derive(Clone, Debug)] +pub enum TestData { + /// A list of tuples containing content key/value pairs + ContentList(Vec), + /// A list of tuples containing a content key, offer value, and lookup value + StateContentList(Vec), +} + +impl TestData { + pub fn content_list(self) -> Vec { + if let TestData::ContentList(content_list) = self { + content_list + } else { + panic!("TestData didn't contain ContentList: enum was likely filled with the wrong data {self:?}") + } + } + + pub fn state_content_list(self) -> Vec { + if let TestData::StateContentList(state_content_list) = self { + state_content_list + } else { + panic!("TestData didn't contain StateContentList: enum was likely filled with the wrong data {self:?}") + } + } +} diff --git a/hivesim-rs/src/utils.rs b/hivesim-rs/src/utils.rs new file mode 100644 index 0000000000..3e1eabc903 --- /dev/null +++ b/hivesim-rs/src/utils.rs @@ -0,0 +1,37 @@ +use crate::types::TestResult; +use tokio::task::JoinError; + +/// Ensures that 'name' contains the client type. +pub fn client_test_name(name: String, client_type: String) -> String { + if name.is_empty() { + return client_type; + } + if name.contains("CLIENT") { + return name.replace("CLIENT", &client_type); + } + format!("{} ({})", name, client_type) +} + +pub fn extract_test_results(join_handle: Result<(), JoinError>) -> TestResult { + match join_handle { + Ok(()) => TestResult { + pass: true, + details: "".to_string(), + }, + Err(err) => { + let err = err.into_panic(); + let err = if let Some(err) = err.downcast_ref::<&'static str>() { + err.to_string() + } else if let Some(err) = err.downcast_ref::() { + err.clone() + } else { + format!("?{:?}", err) + }; + + TestResult { + pass: false, + details: err, + } + } + } +}