wasi-nn: use libtest-mimic (#8825)

* wasi-nn: remove some unncecessary panics from test programs

* Make `libtest-mimic` a workspace dependency

* wasi-nn: use \`libtest-mimic\` for testing

wasi-nn's testing story is complicated by different levels of support on
different platforms (some backends work on certain architectures, others
only work on certain OSes, etc.). This change migrates the `testing`
module, which was included in `src`, to exist solely under `tests`. It
also dynamically checks whether each test is runnable and then chooses
whether to ignore it with a `libtest-mimic` flag. This ensures we can
see all the tests all the time and whether they are running or not,
which is helpful during development.

* Refactor for more subtle `ignore` behavior

On any development machine, with no prior setup, we should be able to
compile and move past the ignored tests without issue:

```console
$ cargo test -- --quiet
running 4 tests
iiii
```

With the proper setup and enabling the right features, tests that are
able to run should do so (eliding a bunch of test output):

```console
$ cargo test --all-features -- --quiet
running 4 tests
iii.
```

On CI, tests that _should_ pass will fail if they can't run:

```console
$ CI=1 cargo test --all-features -- --quiet
iFF.
```

prtest:full

* Add missing `use`

* fix: share download lock between checks

* fix: typo, winml usedx preloaded model

* fix: revert to previous winml behavior

This test was reusing the ONNX test for some reason.

* fix: fully qualify bail!

Cargo.toml

@@ -112,7 +112,7 @@ cranelift-codegen = { workspace = true }
 cranelift-reader = { workspace = true }
 toml = { workspace = true }
 similar = { workspace = true }
-libtest-mimic = "0.7.0"
+libtest-mimic = { workspace = true }
 capstone = { workspace = true }
 object = { workspace = true, features = ['std'] }
 wasmtime-test-macros = { path = "crates/test-macros" }
@@ -319,6 +319,7 @@ humantime = "2.0.0"
 postcard = { version = "1.0.8", default-features = false, features = ['alloc'] }
 criterion = { version = "0.5.0", default-features = false, features = ["html_reports", "rayon"] }
 rustc-hash = "1.1.0"
+libtest-mimic = "0.7.0"
 
 # =============================================================================
 #

crates/test-programs/src/bin/nn_image_classification.rs

@@ -1,17 +1,17 @@
-use anyhow::Result;
+use anyhow::{Context, Result};
 use std::fs;
 use test_programs::nn::{classify, sort_results};
 use wasi_nn::{ExecutionTarget, GraphBuilder, GraphEncoding};
 
 pub fn main() -> Result<()> {
     let xml = fs::read("fixture/model.xml")
-        .expect("the model file to be mapped to the fixture directory");
+        .context("the model file to be mapped to the fixture directory")?;
     let weights = fs::read("fixture/model.bin")
-        .expect("the weights file to be mapped to the fixture directory");
+        .context("the weights file to be mapped to the fixture directory")?;
     let graph = GraphBuilder::new(GraphEncoding::Openvino, ExecutionTarget::CPU)
         .build_from_bytes([&xml, &weights])?;
     let tensor = fs::read("fixture/tensor.bgr")
-        .expect("the tensor file to be mapped to the fixture directory");
+        .context("the tensor file to be mapped to the fixture directory")?;
     let results = classify(graph, tensor)?;
     let top_five = &sort_results(&results)[..5];
     println!("found results, sorted top 5: {:?}", top_five);

crates/test-programs/src/bin/nn_image_classification_named.rs

@@ -1,4 +1,4 @@
-use anyhow::Result;
+use anyhow::{Context, Result};
 use std::fs;
 use test_programs::nn::{classify, sort_results};
 use wasi_nn::{ExecutionTarget, GraphBuilder, GraphEncoding};
@@ -7,7 +7,7 @@ pub fn main() -> Result<()> {
     let graph = GraphBuilder::new(GraphEncoding::Openvino, ExecutionTarget::CPU)
         .build_from_cache("fixtures")?;
     let tensor = fs::read("fixture/tensor.bgr")
-        .expect("the tensor file to be mapped to the fixture directory");
+        .context("the tensor file to be mapped to the fixture directory")?;
     let results = classify(graph, tensor)?;
     let top_five = &sort_results(&results)[..5];
     println!("found results, sorted top 5: {:?}", top_five);

crates/test-programs/src/bin/nn_image_classification_onnx.rs

@@ -1,15 +1,15 @@
-use anyhow::Result;
+use anyhow::{Context, Result};
 use std::fs;
 use test_programs::nn::{classify, sort_results};
 use wasi_nn::{ExecutionTarget, GraphBuilder, GraphEncoding};
 
 pub fn main() -> Result<()> {
     let model = fs::read("fixture/model.onnx")
-        .expect("the model file to be mapped to the fixture directory");
+        .context("the model file to be mapped to the fixture directory")?;
     let graph =
         GraphBuilder::new(GraphEncoding::Onnx, ExecutionTarget::CPU).build_from_bytes([&model])?;
     let tensor = fs::read("fixture/000000062808.rgb")
-        .expect("the tensor file to be mapped to the fixture directory");
+        .context("the tensor file to be mapped to the fixture directory")?;
     let results = classify(graph, tensor)?;
     let top_five = &sort_results(&results)[..5];
     // 963 is meat loaf, meatloaf.

crates/test-programs/src/bin/nn_image_classification_winml.rs

@@ -1,14 +1,13 @@
-use anyhow::Result;
+use anyhow::{Context, Result};
 use std::fs;
 use test_programs::nn::{classify, sort_results};
-
 use wasi_nn::{ExecutionTarget, GraphBuilder, GraphEncoding};
 
 pub fn main() -> Result<()> {
     let graph = GraphBuilder::new(GraphEncoding::Onnx, ExecutionTarget::CPU)
         .build_from_cache("mobilenet")?;
     let tensor = fs::read("fixture/kitten.rgb")
-        .expect("the tensor file to be mapped to the fixture directory");
+        .context("the tensor file to be mapped to the fixture directory")?;
     let results = classify(graph, tensor)?;
     let top_five = &sort_results(&results)[..5];
     println!("found results, sorted top 5: {:?}", top_five);

crates/wasi-nn/Cargo.toml

@@ -29,22 +29,22 @@ openvino = { version = "0.6.0", features = [
     "runtime-linking",
 ], optional = true }
 
-ort = { version = "2.0.0-rc.0", default-features = false, features = ["copy-dylibs", "download-binaries"], optional = true }
+ort = { version = "2.0.0-rc.0", default-features = false, features = [
+    "copy-dylibs",
+    "download-binaries",
+], optional = true }
 
 [target.'cfg(windows)'.dependencies.windows]
 version = "0.52"
-features = [
-    "AI_MachineLearning",
-    "Storage_Streams",
-    "Foundation_Collections",
-]
+features = ["AI_MachineLearning", "Storage_Streams", "Foundation_Collections"]
 optional = true
 
 [build-dependencies]
 walkdir = { workspace = true }
 
 [dev-dependencies]
 cap-std = { workspace = true }
+libtest-mimic = { workspace = true }
 test-programs-artifacts = { workspace = true }
 wasi-common = { workspace = true, features = ["sync"] }
 wasmtime = { workspace = true, features = ["cranelift"] }
@@ -57,3 +57,7 @@ openvino = ["dep:openvino"]
 onnx = ["dep:ort"]
 # winml is only available on Windows 10 1809 and later.
 winml = ["dep:windows"]
+
+[[test]]
+name = "test-programs"
+harness = false

crates/wasi-nn/src/lib.rs

@@ -4,7 +4,6 @@ mod registry;
 pub mod backend;
 pub use ctx::{preload, WasiNnCtx};
 pub use registry::{GraphRegistry, InMemoryRegistry};
-pub mod testing;
 pub mod wit;
 pub mod witx;