Refactor cargo-component internals and introduce proc-macro. (#101)

* Refactor `cargo-component` internals and introduce proc-macro.

This commit includes a *very* large refactoring to the internals of
`cargo-component` and how it works.

The biggest change is that `cargo-component` no longer generates an external
bindings crate that is automatically added as a dependency to a user's component
project. The motivation behind this change is that support for resources from
the component model will preclude the ability to reference types from the
user's crate because the `export!` macro is being removed.

Instead, component projects will need to reference the
`cargo-component-bindings` crate to invoke the `generate!` proc-macro. This
macro will expand to the bindings code based on the target world encoded from
dependency resolution performed by `cargo-component`.

By default, the `generate!` macro will assume that the world / all exported
interfaces are implemented by a type named `Component`, however that can be
changed with an argument to the macro.

With the removal of the need for an external bindings crate, `cargo-component`
no longer has a concrete reason to directly depend on the `cargo` crate as a
dependency. Instead, it now spawns `cargo` for all subcommands that aren't
built-in to `cargo-component`.

For this to function, `cargo-component` detects certain `cargo` options that
may alter output (`--verbose`, `--quiet`, etc.) or what gets built
(`--manifest-path`, `--target`, etc.). Before any unrecognized commands are
passed to `cargo`, `cargo-component` will update the encoding of target worlds
for bindings allowing commands that analyze or build the source code to work.

Additionally, `cargo-component` will recognize certain `cargo` subcommands that
have outputs that need to be componentized.

As a result of these changes, about 150 crate dependencies were removed, the
`cargo-component` executable shrunk about 12 MiB in size, and commands such as
`cargo component expand` now "just work" without `cargo-component` having to
provide its own implementation.

All dependencies, and the WASI adapters, are updated as part of these changes.

Note: the `test` command still isn't implemented.

Closes #97.
Closes #91.

* Update wit-bindgen deps.

* Fix windows build.

* More Windows fixes.

* Update warg dependencies.

This includes the latest API changes.

* More dep updates.

* Update crates/core/src/progress.rs

Co-authored-by: Dan Gohman <dev@sunfishcode.online>

* Update crates/core/src/progress.rs

Co-authored-by: Dan Gohman <dev@sunfishcode.online>

* Update crates/core/src/progress.rs

Co-authored-by: Dan Gohman <dev@sunfishcode.online>

* Update crates/core/src/progress.rs

Co-authored-by: Dan Gohman <dev@sunfishcode.online>

* Add `pub(crate)` to the bindings module.

Also update the example lock file.

* Add examples of commands passed through to `cargo` to README.

* Update README to explain what a "reactor" component is.

* Add more context for failure to create a target world.

This commit adds more context to when there's a problem creating the target
world.

It gives users a way of determining which cargo package is at fault for the
error when building a multi-package workspace.

---------

Co-authored-by: Dan Gohman <dev@sunfishcode.online>

Cargo.toml

@@ -2,49 +2,93 @@
 name = "cargo-component"
 version = "0.1.0"
 edition = "2021"
+authors = ["Peter Huene <peter@huene.dev>"]
+description = "A cargo extension for authoring WebAssembly components"
+license = "Apache-2.0 WITH LLVM-exception"
+documentation = "https://docs.rs/cargo-component"
+categories = ["wasm"]
+keywords = ["webassembly", "wasm", "components", "component-model"]
+repository = "https://github.com/bytecodealliance/cargo-component"
 
 [dependencies]
-anyhow = "1.0.71"
-cargo = "0.71.0"
-cargo-util = "0.2.4"
-clap = { version = "4.3.11", features = ["derive"] }
-toml_edit = { version = "0.19.12", features = ["serde"] }
+cargo-component-core = { workspace = true }
+anyhow = { workspace = true }
+clap = { workspace = true }
+toml_edit = { workspace = true }
+pretty_env_logger = { workspace = true }
+log = { workspace = true }
+tokio = { workspace = true }
+tokio-util = { workspace = true }
+heck = { workspace = true }
+semver = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+indexmap = { workspace = true }
+url = { workspace = true }
+wit-bindgen-rust-lib = { workspace = true }
+wit-parser = { workspace = true }
+wit-component = { workspace = true }
+wasm-metadata = { workspace = true }
+parse_arg = { workspace = true }
+cargo_metadata = { workspace = true }
+libc = { workspace = true }
+warg-protocol = { workspace = true }
+warg-crypto = { workspace = true }
+warg-client = { workspace = true }
+p256 = { workspace = true }
+rand_core = { workspace = true }
+rpassword = { workspace = true }
+futures = { workspace = true }
+bytes = { workspace = true }
+
+[dev-dependencies]
+assert_cmd = "2.0.12"
+predicates = "3.0.3"
+wasmparser = "0.110.0"
+wat = "1.0.69"
+warg-server = { git = "https://github.com/bytecodealliance/registry" }
+
+[workspace]
+members = ["crates/bindings", "crates/macro", "crates/core"]
+exclude = ["target/tests"]
+
+[workspace.dependencies]
+cargo-component-core = { path = "crates/core" }
+cargo-component-macro = { path = "crates/macro" }
 warg-protocol = { git = "https://github.com/bytecodealliance/registry" }
 warg-crypto = { git = "https://github.com/bytecodealliance/registry" }
 warg-client = { git = "https://github.com/bytecodealliance/registry" }
-wit-bindgen-core = "0.8.0"
-wit-bindgen-rust = "0.8.0"
-wit-bindgen-rust-lib = "0.8.0"
-wit-parser = "0.8.0"
-wit-component = "0.11.0"
-pretty_env_logger = { version = "0.5.0", optional = true }
+anyhow = "1.0.72"
+clap = { version = "4.3.19", features = ["derive"] }
+toml_edit = { version = "0.19.14", features = ["serde"] }
+pretty_env_logger = "0.5.0"
 log = "0.4.19"
+tokio = { version = "1.29.1", default-features = false, features = ["macros", "rt-multi-thread"] }
+tokio-util = "0.7.8"
 heck = "0.4.1"
-semver = "1.0.17"
-serde = { version = "1.0.166", features = ["derive"] }
+semver = "1.0.18"
+serde = { version = "1.0.176", features = ["derive"] }
+serde_json = "1.0.104"
+indexmap = "2.0.0"
 url = { version = "2.4.0", features = ["serde"] }
-tokio = { version = "1.29.1", default-features = false, features = ["macros", "rt-multi-thread"] }
-home = "0.5.5"
-p256 = "0.13.2"
+wit-bindgen-rust-lib = "0.9.0"
+wit-parser = "0.9.2"
+wit-component = "0.13.1"
+wasm-metadata = "0.10.1"
+parse_arg = "0.1.4"
+cargo_metadata = "0.17.0"
+keyring = "2.0.5"
+libc = "0.2.147"
+owo-colors = "3.5.0"
+unicode-width = "0.1.10"
+p256 = "0.13.2" 
 rand_core = "0.6.4"
-serde_json = "1.0.100"
-async-trait = "0.1.71"
-wat = "1.0.66"
-indexmap = "1.9.3"
-hex = "0.4.3"
-termcolor = "1.2.0"
-wasm-metadata = "0.8.0"
+rpassword = "7.2.0"
 futures = "0.3.28"
 bytes = "1.4.0"
-tokio-util = "0.7.8"
-rpassword = "7.2.0"
-keyring = "2.0.4"
-
-[features]
-default = ["pretty_env_logger"]
-
-[dev-dependencies]
-assert_cmd = "2.0.11"
-predicates = "3.0.3"
-wasmparser = "0.107.0"
-warg-server = { git = "https://github.com/bytecodealliance/registry" }
+proc-macro2 = "1.0.66"
+quote = "1.0.32"
+syn = "2.0.27"
+wit-bindgen-core = "0.9.0"
+wit-bindgen-rust = "0.9.0"
+wit-bindgen = "0.9.0"

README.md

@@ -25,7 +25,7 @@ using Rust as the component's implementation language.
 `cargo component` is considered to be experimental and is _not_ currently
 stable in terms of the code it supports building.
 
-Until the component model stabilizes, upgrading to a newer `cargo component` 
+Until the component model stabilizes, upgrading to a newer `cargo component`
 may cause build errors for existing component projects.
 
 ## Requirements
@@ -87,7 +87,7 @@ This enables WebAssembly runtimes to know specifically how they must
 facilitate the exchange of data between the discrete linear memories
 of components, eliminating the need for developers to do so by hand.
 
-Additionally, components can describe their dependencies in a way 
+Additionally, components can describe their dependencies in a way
 that modules simply cannot today; they can even control how their
 dependencies are _instantiated_, enabling a component to
 _virtualize_ functionality needed by a dependency. And because
@@ -105,7 +105,7 @@ That means being able to reference WebAssembly components via
 same way as Rust crate dependencies:
 
 * add a dependency on a WebAssembly component to `Cargo.toml`
-* reference it like you would an external crate (via `<name>::...`) in
+* reference it like you would an external crate (via `bindings::<name>::...`) in
   your source code
 * build using `cargo component build` and out pops your component!
 
@@ -121,16 +121,10 @@ including Rust.
 `wit-bindgen` even provides procedural macros to generate the
 bindings "inline" with the component's source code.
 
-Unlike `wit-bindgen`, `cargo component` doesn't use procedural macros
-or a `build.rs` file to generate bindings. Instead, it generates them
-into external crates that are automatically provided to the Rust
-compiler when building your component's project.
-
-This approach does come with some downsides, however. Commands like
-`cargo metadata` and `cargo check` used by many tools (e.g.
-`rust-analyzer`) simply don't work because they aren't aware of the
-generated bindings. That is why replacement commands such as
-`cargo component metadata` and `cargo component check` exist.
+Like `wit-bindgen`, `cargo component` uses a procedural macro to generate
+bindings. However, bindings are generated based on the resolved dependencies
+from `Cargo.toml` rather than parsing a local definition of the component's
+interface.
 
 The hope is that one day (in the not too distant future...) that
 WebAssembly components might become an important part of the Rust
@@ -159,27 +153,32 @@ and tooling developers can use to test their implementations.
 
 Currently `cargo component` targets `wasm32-wasi` by default.
 
-As this target is for a _preview1_ release of WASI, the WebAssembly module 
-produced by the Rust compiler must be adapted to the _preview2_ version of WASI 
+As this target is for a _preview1_ release of WASI, the WebAssembly module
+produced by the Rust compiler must be adapted to the _preview2_ version of WASI
 supported by the component model.
 
 The adaptation is automatically performed when `wasm32-wasi` is targeted.
 
-To prevent this, override the target to `wasm32-unknown-unknown` using the 
+To prevent this, override the target to `wasm32-unknown-unknown` using the
 `--target` option when building. This, however, will disable WASI support.
 
-Use the _preview2_ version of [`wasi-common`][2] in your host to run components 
+Use the _preview2_ version of [`wasi-common`][2] in your host to run components
 produced by `cargo component`.
 
-When the Rust compiler supports a [_preview2_ version of the WASI target][1], 
+When the Rust compiler supports a [_preview2_ version of the WASI target][1],
 support in `cargo component` for adapting a _preview1_ module will be removed.
 
 [1]: https://github.com/rust-lang/compiler-team/issues/594
 [2]: https://github.com/bytecodealliance/preview2-prototyping/tree/main/wasi-common
 
 ## Getting Started
 
-Use `cargo component new --lib <name>` to create a new reactor component.
+Use `cargo component new --reactor <name>` to create a new reactor component.
+
+A reactor component doesn't have a `run` (i.e. `main` in Rust) function
+exported and is meant to be used as a library rather than a command that runs
+and exits. Without the `--reactor` flag, `cargo component` defaults to creating
+a command component.
 
 This will create a `wit/world.wit` file describing the world that the
 component will target:
@@ -190,95 +189,89 @@ package my-org:my-component
 /// An example world for the component to target.
 world example {
     export hello-world: func() -> string
-}                
+}
 ```
 
 The component will export a `hello-world` function returning a string.
 
 The implementation of the component will be in `src/lib.rs`:
 
 ```rust
+cargo_component_bindings::generate!();
+
+use bindings::Example;
+
 struct Component;
 
-impl bindings::Example for Component {
+impl Example for Component {
     /// Say hello!
     fn hello_world() -> String {
         "Hello, World!".to_string()
     }
 }
-
-bindings::export!(Component);
 ```
 
-Here `bindings` is the bindings crate that `cargo component` generated for you.
+The `generate!` macro is responsible for generating the bindings to allow the
+Rust code to export what is expected of the component.
 
-The `export!` macro informs the bindings that the `Component` type exports
-all interfaces listed in `Cargo.toml`.
+It generates a Rust module named `bindings` containing the types and traits the
+correspond to the world definition.
 
 ## Usage
 
 The `cargo component` subcommand has some analogous commands to cargo itself:
 
 * `cargo component new` — creates a new WebAssembly component Rust project.
-* `cargo component add` — adds a component interface dependency to a cargo 
+* `cargo component add` — adds a component interface dependency to a cargo
   manifest file.
-* `cargo component build` — builds a WebAssembly component from a Rust project
-  using the `wasm32-wasi` target by default.
-* `cargo component doc` — generates API documentation for a WebAssembly 
-  component from a Rust project.
-* `cargo component metadata` — prints package metadata as `cargo metadata` 
-  would, except it also includes the metadata of generated bindings.
-* `cargo component check` — checks the local package and all of its dependencies
-  (including generated bindings) for errors.
-* `cargo component clippy` — same as `cargo clippy` but also checks generated 
-  bindings.
-* `cargo component update` — same as `cargo update` but also updates the 
+* `cargo component update` — same as `cargo update` but also updates the
   dependencies in the component lock file.
 * `cargo component publish` - publishes a WebAssembly component to a [warg](https://warg.io/)
   component registry.
-* `cargo component key` - manages signing keys for publishing WebAssembly 
+* `cargo component key` - manages signing keys for publishing WebAssembly
   components.
-* `cargo component wit` - manages the target WIT package of the component.
 
-More commands will be added over time.
+Unrecognized commands are passed through to `cargo` itself, but only after the
+bindings information for component packages has been updated.
+
+Some examples of commands that are passed directly to `cargo` are: `build`,
+`check`, `doc`, `clippy` and extension commands such as `expand` from
+`cargo-expand`.
+
+Certain command line options, like `--target` and `--release`, are detected by
+`cargo component` to determine what output files of a `build` command should be
+componentized.
 
 ## Using `rust-analyzer`
 
 [rust-analyzer](https://github.com/rust-analyzer/rust-analyzer) is an extremely
-useful tool for analyzing Rust code and is used in many different editors to 
+useful tool for analyzing Rust code and is used in many different editors to
 provide code completion and other features.
 
-rust-analyzer depends on `cargo metadata` and `cargo check` to discover 
+rust-analyzer depends on `cargo metadata` and `cargo check` to discover
 workspace information and to check for errors.
 
-Because `cargo component` generates code for dependencies that `cargo` itself is
-unaware of, rust-analyzer will not detect or parse the generated bindings; 
-additionally, diagnostics will highlight any use of the generated bindings as 
-errors.
-
-To solve this problem, rust-analyzer must be configured to use the 
-`cargo-component` executable as the `cargo` command. By doing so, the `cargo 
-component metadata` and `cargo component check` subcommands will inform 
-rust-analyzer of the generated bindings as if they were normal crate 
-dependencies.
+To ensure that rust-analyzer is able to discover the latest bindings
+information, rust-analyzer must be configured to use `cargo component check` as
+the check command.
 
 To configure rust-analyzer to use the `cargo-component` executable, set the
 `rust-analyzer.server.extraEnv` setting to the following:
 
 ```json
-"rust-analyzer.server.extraEnv": { "CARGO": "cargo-component" }
+"rust-analyzer.check.overrideCommand": ["cargo", "component", "check", "--message-format=json"]
 ```
 
-By default, `cargo component new` will configure Visual Studio Code to use 
-`cargo component` by creating a `.vscode/settings.json` file for you. To 
+By default, `cargo component new` will configure Visual Studio Code to use
+`cargo component check` by creating a `.vscode/settings.json` file for you. To
 prevent this, pass `--editor none` to `cargo component new`.
 
-Please check the documentation for rust-analyzer regarding how to set settings 
+Please check the documentation for rust-analyzer regarding how to set settings
 for other IDEs.
 
 ## Contributing to `cargo component`
 
-`cargo component` is a [Bytecode Alliance](https://bytecodealliance.org/) 
+`cargo component` is a [Bytecode Alliance](https://bytecodealliance.org/)
 project, and follows
 the Bytecode Alliance's [Code of Conduct](CODE_OF_CONDUCT.md) and
 [Organizational Code of Conduct](ORG_CODE_OF_CONDUCT.md).
@@ -303,7 +296,7 @@ You'll be adding tests primarily to the `tests/` directory.
 
 ### Submitting Changes
 
-Changes to `cargo component` are managed through pull requests (PRs). Everyone 
+Changes to `cargo component` are managed through pull requests (PRs). Everyone
 is welcome to submit a pull request! We'll try to get to reviewing it or
 responding to it in at most a few days.
 
@@ -317,7 +310,7 @@ command. This is checked on CI.
 The CI for the `cargo component` repository is relatively significant. It tests
 changes on Windows, macOS, and Linux.
 
-It also performs a "dry run" of the release process to ensure that release 
+It also performs a "dry run" of the release process to ensure that release
 binaries can be built and are ready to be published (_coming soon_).
 
 ### Publishing (_coming soon_)

build.rs

@@ -1,6 +1,6 @@
 use std::{path::Path, process::Command};
 
-const WASI_ADAPTER_VERSION: &str = "7f4d569";
+const WASI_ADAPTER_VERSION: &str = "7b9189b";
 
 fn main() {
     println!("cargo:rerun-if-changed=build.rs");

crates/bindings/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "cargo-component-bindings"
+version = "0.1.0"
+edition = "2021"
+authors = ["Peter Huene <peter@huene.dev>"]
+description = "A crate for generating bindings of WebAssembly components."
+license = "Apache-2.0 WITH LLVM-exception"
+documentation = "https://docs.rs/cargo-component-bindings"
+categories = ["wasm"]
+keywords = ["webassembly", "wasm", "components", "component model"]
+repository = "https://github.com/bytecodealliance/cargo-component"
+
+[dependencies]
+cargo-component-macro = { workspace = true }
+wit-bindgen = { workspace = true }

crates/bindings/src/lib.rs

@@ -0,0 +1,10 @@
+//! A crate for generating bindings with `cargo-component`.
+
+#![deny(missing_docs)]
+
+// Export the `generate` macro.
+pub use cargo_component_macro::generate;
+
+// Re-export `wit_bindgen::rt` module for the generated code to use.
+#[doc(hidden)]
+pub use wit_bindgen::rt;