diff --git a/Cargo.lock b/Cargo.lock index 50174de7e27..5799a5285cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4614,11 +4614,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -4955,14 +4956,15 @@ checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if 1.0.0", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6642,6 +6644,7 @@ dependencies = [ "wasmer-config 0.8.0", "wasmer-emscripten", "wasmer-object", + "wasmer-package", "wasmer-registry", "wasmer-types", "wasmer-vm", @@ -6939,6 +6942,17 @@ dependencies = [ "wasmer-types", ] +[[package]] +name = "wasmer-package" +version = "0.1.0" +dependencies = [ + "pretty_assertions", + "tempfile", + "toml 0.8.15", + "wasmer-config 0.8.0", + "webc", +] + [[package]] name = "wasmer-registry" version = "5.19.0" diff --git a/Cargo.toml b/Cargo.toml index f6843e8d12a..3139a4ada41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ members = [ "lib/derive", "lib/emscripten", "lib/object", + "lib/package", "lib/registry", "lib/sys-utils", "lib/types", @@ -116,6 +117,7 @@ mio = "1" # MIO 1.0 starts at tokio version 1.39, hence the minimum requirement. tokio = { version = "1.39.0", default-features = false} socket2 = "0.5.7" +pretty_assertions = "1.4.0" base64 = "0.22.0" [build-dependencies] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 5f9a4e3d64b..4e75d442cc8 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -117,6 +117,8 @@ wasmer-compiler-cranelift = { version = "=4.3.7", path = "../compiler-cranelift" wasmer-compiler-singlepass = { version = "=4.3.7", path = "../compiler-singlepass", optional = true } wasmer-compiler-llvm = { version = "=4.3.7", path = "../compiler-llvm", optional = true } wasmer-emscripten = { version = "=4.3.7", path = "../emscripten" } +wasmer-package = { version = "=0.1.0", path = "../package" } + wasmer-vm = { version = "=4.3.7", path = "../vm", optional = true } wasmer-wasix = { path = "../wasix", version = "=0.27.0", features = [ "logging", @@ -276,7 +278,7 @@ unix_mode = "0.1.3" [dev-dependencies] assert_cmd = "2.0.11" predicates = "3.0.3" -pretty_assertions = "1.3.0" +pretty_assertions.workspace = true [target.'cfg(target_os = "windows")'.dependencies] colored = "2.0.0" diff --git a/lib/cli/src/commands/package/unpack.rs b/lib/cli/src/commands/package/unpack.rs index 0f4784886e8..23d53b90164 100644 --- a/lib/cli/src/commands/package/unpack.rs +++ b/lib/cli/src/commands/package/unpack.rs @@ -5,27 +5,53 @@ use dialoguer::console::{style, Emoji}; use indicatif::ProgressBar; /// Extract contents of a webc image to a directory. +/// +/// See --format flag for available output formats. #[derive(clap::Parser, Debug)] pub struct PackageUnpack { /// The output directory. #[clap(short = 'o', long)] - out_dir: PathBuf, + pub out_dir: PathBuf, /// Overwrite existing directories/files. #[clap(long)] - overwrite: bool, + pub overwrite: bool, /// Run the unpack command without any output #[clap(long)] pub quiet: bool, /// Path to the package. - package_path: PathBuf, + pub package_path: PathBuf, + + /// Output format. + /// + /// * package + /// Restore a package directory with a wasmer.toml + /// NOTE: this conversion is lossy, because webcs don't store the original + /// wasmer.toml and the full contents can not be restored. + /// + /// * webc + /// Directly unpack the webc contents. + /// - Volumes will be placed in subdirectories. + /// - atoms will be placed in the root directory + /// - the full webc manifest will be placed in a manifest.json file + #[clap(short, long, default_value = "package")] + pub format: Format, } static PACKAGE_EMOJI: Emoji<'_, '_> = Emoji("📦 ", ""); static EXTRACTED_TO_EMOJI: Emoji<'_, '_> = Emoji("📂 ", ""); +/// Webc unpack format. +#[derive(clap::ValueEnum, Clone, Debug)] +pub enum Format { + /// See [`PackageUnpack::format`] for details. + Package, + /// See [`PackageUnpack::format`] for details. + Webc, +} + impl PackageUnpack { pub(crate) fn execute(&self) -> Result<(), anyhow::Error> { // Setup the progress bar @@ -52,8 +78,16 @@ impl PackageUnpack { std::fs::create_dir_all(outdir) .with_context(|| format!("could not create output directory '{}'", outdir.display()))?; - pkg.unpack(outdir, self.overwrite) - .with_context(|| "could not extract package".to_string())?; + match self.format { + Format::Package => { + wasmer_package::convert::webc_to_package_dir(&pkg, outdir) + .with_context(|| "could not extract package")?; + } + Format::Webc => { + pkg.unpack(outdir, self.overwrite) + .with_context(|| "could not extract package".to_string())?; + } + } pb.println(format!( "{} {}Extracted package contents to '{}'", @@ -89,6 +123,7 @@ mod tests { overwrite: false, package_path, quiet: true, + format: Format::Webc, }; cmd.execute().unwrap(); diff --git a/lib/config/Cargo.toml b/lib/config/Cargo.toml index 6475abb9679..1f2d6b80044 100644 --- a/lib/config/Cargo.toml +++ b/lib/config/Cargo.toml @@ -29,6 +29,6 @@ hex = "0.4.3" ciborium = "0.2.2" [dev-dependencies] -pretty_assertions = "1.4.0" +pretty_assertions.workspace = true serde_json = "1.0.116" tempfile = "3.3.0" diff --git a/lib/config/src/package/mod.rs b/lib/config/src/package/mod.rs index 9f3ce2e743f..75aa3c054cd 100644 --- a/lib/config/src/package/mod.rs +++ b/lib/config/src/package/mod.rs @@ -185,6 +185,10 @@ pub struct Package { } impl Package { + pub fn new_empty() -> Self { + PackageBuilder::default().build().unwrap() + } + /// Create a [`PackageBuilder`] populated with all mandatory fields. pub fn builder( name: impl Into, @@ -732,6 +736,16 @@ pub struct Manifest { } impl Manifest { + pub fn new_empty() -> Self { + Self { + package: None, + dependencies: HashMap::new(), + fs: IndexMap::new(), + modules: Vec::new(), + commands: Vec::new(), + } + } + /// Create a [`ManifestBuilder`] populated with all mandatory fields. pub fn builder(package: Package) -> ManifestBuilder { ManifestBuilder::new(package) diff --git a/lib/package/Cargo.toml b/lib/package/Cargo.toml new file mode 100644 index 00000000000..b7d68262df4 --- /dev/null +++ b/lib/package/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "wasmer-package" +version = "0.1.0" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +webc.workspace = true +wasmer-config = { version = "0.8.0", path = "../config" } +toml = "0.8.0" + +[dev-dependencies] +pretty_assertions.workspace = true +tempfile = "3.12.0" diff --git a/lib/package/src/convert/error.rs b/lib/package/src/convert/error.rs new file mode 100644 index 00000000000..2f6e7566b35 --- /dev/null +++ b/lib/package/src/convert/error.rs @@ -0,0 +1,47 @@ +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct ConversionError { + message: String, + cause: Option>, +} + +impl ConversionError { + pub fn msg(msg: impl Into) -> Self { + Self { + message: msg.into(), + cause: None, + } + } + + pub fn with_cause( + msg: impl Into, + cause: impl std::error::Error + Send + Sync + 'static, + ) -> Self { + Self { + message: msg.into(), + cause: Some(Arc::new(cause)), + } + } +} + +impl std::fmt::Display for ConversionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "could not convert manifest: {}", self.message)?; + if let Some(cause) = &self.cause { + write!(f, " (cause: {})", cause)?; + } + + Ok(()) + } +} + +impl std::error::Error for ConversionError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + if let Some(e) = &self.cause { + Some(e) + } else { + None + } + } +} diff --git a/lib/package/src/convert/mod.rs b/lib/package/src/convert/mod.rs new file mode 100644 index 00000000000..ca75e1a1b6f --- /dev/null +++ b/lib/package/src/convert/mod.rs @@ -0,0 +1,4 @@ +mod error; +mod webc_to_package; + +pub use self::{error::ConversionError, webc_to_package::webc_to_package_dir}; diff --git a/lib/package/src/convert/webc_to_package.rs b/lib/package/src/convert/webc_to_package.rs new file mode 100644 index 00000000000..22edafa39c1 --- /dev/null +++ b/lib/package/src/convert/webc_to_package.rs @@ -0,0 +1,332 @@ +use std::path::Path; + +use wasmer_config::package::ModuleReference; + +use super::ConversionError; + +/// Convert a webc image into a directory with a wasmer.toml file that can +/// be used for generating a new pacakge. +pub fn webc_to_package_dir( + webc: &webc::Container, + target_dir: &Path, +) -> Result<(), ConversionError> { + let mut pkg_manifest = wasmer_config::package::Manifest::new_empty(); + + let manifest = webc.manifest(); + // Convert the package annotation. + + let pkg_annotation = manifest + .wapm() + .map_err(|err| ConversionError::with_cause("could not read package annotation", err))?; + if let Some(ann) = pkg_annotation { + let mut pkg = wasmer_config::package::Package::new_empty(); + + pkg.name = ann.name; + pkg.version = if let Some(raw) = ann.version { + let v = raw + .parse() + .map_err(|e| ConversionError::with_cause("invalid package version", e))?; + Some(v) + } else { + None + }; + + pkg.description = ann.description; + pkg.license = ann.license; + + // TODO: map license_file and README (paths!) + + pkg.homepage = ann.homepage; + pkg.repository = ann.repository; + pkg.private = ann.private; + pkg.entrypoint = manifest.entrypoint.clone(); + + pkg_manifest.package = Some(pkg); + } + + // Map dependencies. + for (_name, target) in &manifest.use_map { + match target { + webc::metadata::UrlOrManifest::Url(_url) => { + // Not supported. + } + webc::metadata::UrlOrManifest::Manifest(_) => { + // Not supported. + } + webc::metadata::UrlOrManifest::RegistryDependentUrl(raw) => { + let (name, version) = if let Some((name, version_raw)) = raw.split_once('@') { + let version = version_raw.parse().map_err(|err| { + ConversionError::with_cause( + format!("Could not parse version of dependency: '{}'", raw), + err, + ) + })?; + (name.to_string(), version) + } else { + (raw.to_string(), "*".parse().unwrap()) + }; + + pkg_manifest.dependencies.insert(name, version); + } + } + } + + // Convert filesystem mappings. + + let fs_annotation = manifest + .filesystem() + .map_err(|err| ConversionError::with_cause("could n ot read fs annotation", err))?; + if let Some(ann) = fs_annotation { + for mapping in ann.0 { + if mapping.from.is_some() { + // wasmer.toml does not allow specifying dependency mounts. + continue; + } + + // Extract the volume to "/". + let volume = webc.get_volume(&mapping.volume_name).ok_or_else(|| { + ConversionError::msg(format!( + "Package annotations specify a volume that does not exist: '{}'", + mapping.volume_name + )) + })?; + + let volume_path = target_dir.join(mapping.volume_name.trim_start_matches('/')); + + std::fs::create_dir_all(&volume_path).map_err(|err| { + ConversionError::with_cause( + format!( + "could not create volume directory '{}'", + volume_path.display() + ), + err, + ) + })?; + + volume.unpack("/", &volume_path).map_err(|err| { + ConversionError::with_cause("could not unpack volume to filesystemt", err) + })?; + + let mut source_path = mapping + .volume_name + .trim_start_matches('/') + .trim_end_matches('/') + .to_string(); + if let Some(subpath) = mapping.host_path { + if !source_path.ends_with('/') { + source_path.push('/'); + } + source_path.push_str(&subpath); + } + source_path.insert_str(0, "./"); + + pkg_manifest + .fs + .insert(mapping.mount_path, source_path.into()); + } + } + + // Convert modules. + + let module_dir_name = "modules"; + let module_dir = target_dir.join(module_dir_name); + + let atoms = webc.atoms(); + if !atoms.is_empty() { + std::fs::create_dir_all(&module_dir).map_err(|err| { + ConversionError::with_cause( + format!("Could not create directory '{}'", module_dir.display(),), + err, + ) + })?; + for (atom_name, data) in atoms { + let atom_path = module_dir.join(&atom_name); + + std::fs::write(&atom_path, &data).map_err(|err| { + ConversionError::with_cause( + format!("Could not write atom to path '{}'", atom_path.display()), + err, + ) + })?; + + let relative_path = format!("./{module_dir_name}/{atom_name}"); + + pkg_manifest.modules.push(wasmer_config::package::Module { + name: atom_name, + source: relative_path.into(), + abi: wasmer_config::package::Abi::None, + kind: None, + interfaces: None, + bindings: None, + }); + } + } + + // Convert commands. + for (name, spec) in &manifest.commands { + let mut annotations = toml::Table::new(); + for (key, value) in &spec.annotations { + if key == webc::metadata::annotations::Atom::KEY { + continue; + } + + let raw_toml = toml::to_string(&value).unwrap(); + let toml_value = toml::from_str::(&raw_toml).unwrap(); + annotations.insert(key.into(), toml_value); + } + + let atom_annotation = spec + .annotation::(webc::metadata::annotations::Atom::KEY) + .map_err(|err| { + ConversionError::with_cause( + format!("could not read atom annotation for command '{}'", name), + err, + ) + })? + .ok_or_else(|| { + ConversionError::msg(format!( + "Command '{name}' is missing the required atom annotation" + )) + })?; + + let module = if let Some(dep) = atom_annotation.dependency { + ModuleReference::Dependency { + dependency: dep, + module: atom_annotation.name, + } + } else { + ModuleReference::CurrentPackage { + module: atom_annotation.name, + } + }; + + let cmd = wasmer_config::package::Command::V2(wasmer_config::package::CommandV2 { + name: name.clone(), + module, + runner: spec.runner.clone(), + annotations: Some(wasmer_config::package::CommandAnnotations::Raw( + annotations.into(), + )), + }); + + pkg_manifest.commands.push(cmd); + } + + // Write out the manifest. + let manifest_toml = toml::to_string(&pkg_manifest) + .map_err(|err| ConversionError::with_cause("could not serialize package manifest", err))?; + std::fs::write(target_dir.join("wasmer.toml"), manifest_toml) + .map_err(|err| ConversionError::with_cause("could not write wasmer.toml", err))?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::fs::create_dir_all; + + use pretty_assertions::assert_eq; + + use super::*; + + // Build a webc from a pacakge directory, and then restore the directory + // from the webc. + #[test] + fn test_wasmer_package_webc_roundtrip() { + let tmpdir = tempfile::tempdir().unwrap(); + let dir = tmpdir.path(); + + let webc = { + let dir_input = dir.join("input"); + let dir_public = dir_input.join("public"); + + create_dir_all(&dir_public).unwrap(); + + std::fs::write(dir_public.join("index.html"), "INDEX").unwrap(); + + std::fs::write(dir_input.join("mywasm.wasm"), "()").unwrap(); + + std::fs::write( + dir_input.join("wasmer.toml"), + r#" +[package] +name = "testns/testpkg" +version = "0.0.1" +description = "descr1" +license = "MIT" + +[dependencies] +"wasmer/python" = "8.12.0" + +[fs] +public = "./public" + +[[module]] +name = "mywasm" +source = "./mywasm.wasm" + +[[command]] +name = "run" +module = "mywasm" +runner = "wasi" + +[command.annotations.wasi] +env = ["A=B"] +main-args = ["/mounted/script.py"] +"#, + ) + .unwrap(); + + let pkg = webc::wasmer_package::Package::from_manifest(dir_input.join("wasmer.toml")) + .unwrap(); + let raw = pkg.serialize().unwrap(); + webc::Container::from_bytes(raw).unwrap() + }; + + let dir_output = dir.join("output"); + webc_to_package_dir(&webc, &dir_output).unwrap(); + + assert_eq!( + std::fs::read_to_string(dir_output.join("public/index.html")).unwrap(), + "INDEX", + ); + + assert_eq!( + std::fs::read_to_string(dir_output.join("modules/mywasm")).unwrap(), + "()", + ); + + assert_eq!( + std::fs::read_to_string(dir_output.join("wasmer.toml")) + .unwrap() + .trim(), + r#" + +[package] +license = "MIT" +entrypoint = "run" + +[dependencies] +"wasmer/python" = "^8.12.0" + +[fs] +"/public" = "./public" + +[[module]] +name = "mywasm" +source = "./modules/mywasm" + +[[command]] +name = "run" +module = "mywasm" +runner = "https://webc.org/runner/wasi" + +[command.annotations.wasi] +atom = "mywasm" +env = ["A=B"] +main-args = ["/mounted/script.py"] + "# + .trim(), + ); + } +} diff --git a/lib/package/src/lib.rs b/lib/package/src/lib.rs new file mode 100644 index 00000000000..b5b67213dac --- /dev/null +++ b/lib/package/src/lib.rs @@ -0,0 +1 @@ +pub mod convert; diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index b07cac2e096..418fa8062e7 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -57,7 +57,7 @@ webc.workspace = true async-tungstenite = { version = "0.25.1", features = ["tokio-runtime", "tokio-rustls-native-certs"] } [dev-dependencies] -pretty_assertions = "1.3.0" +pretty_assertions.workspace = true [package.metadata.docs.rs] features = ["build-package"] diff --git a/lib/virtual-fs/Cargo.toml b/lib/virtual-fs/Cargo.toml index 7ad563a6ad3..47afa4a371a 100644 --- a/lib/virtual-fs/Cargo.toml +++ b/lib/virtual-fs/Cargo.toml @@ -40,7 +40,7 @@ getrandom = { version = "0.2" } getrandom = { version = "0.2", features = [ "js" ] } [dev-dependencies] -pretty_assertions = "1.3.0" +pretty_assertions.workspace = true tempfile = "3.6.0" tracing-test = "0.2.4" tokio = { workspace = true, features = ["io-util", "rt"], default-features = false } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index f94edce5a9a..294b9d631a9 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -133,7 +133,7 @@ terminal_size = { version = "0.3.0" } [dev-dependencies] wasmer = { path = "../api", version = "=4.3.7", default-features = false, features = ["wat", "js-serializable-module"] } tokio = { workspace = true, features = [ "sync", "macros", "rt" ], default-features = false } -pretty_assertions = "1.3.0" +pretty_assertions.workspace = true tracing-test = "0.2.4" wasm-bindgen-test = "0.3.0" diff --git a/tests/integration/cli/Cargo.toml b/tests/integration/cli/Cargo.toml index 47e32851fa9..76fa9dcf3f3 100644 --- a/tests/integration/cli/Cargo.toml +++ b/tests/integration/cli/Cargo.toml @@ -14,7 +14,7 @@ serde = { version = "1.0.147", features = ["derive"] } insta = { version = "1.21.1", features = ["json"] } md5 = "0.7.0" hex = "0.4.3" -pretty_assertions = "1.3.0" +pretty_assertions.workspace = true object = "0.30.0" reqwest = { workspace = true, default-features = false, features = ["json", "blocking", "rustls-tls"] } tokio = { workspace = true, features = [ "rt", "rt-multi-thread", "macros" ] }