From 45d937ff19604a34806c1e343cd4ea5efb0e0cba Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sat, 2 Dec 2023 14:02:55 -0800 Subject: [PATCH 01/15] Gate more functionality behind `debug-builtins` This commit extends the gating behavior of the preexisting `debug-builtins` Cargo feature to cover more GDB-related functionality associated with debugging. This can additionally slim down the set of exposed symbols from Wasmtime over the default with them included. --- crates/jit/Cargo.toml | 1 + crates/jit/src/instantiate.rs | 24 +++++++++++++++--------- crates/jit/src/lib.rs | 1 + crates/runtime/Cargo.toml | 4 ++-- crates/runtime/src/lib.rs | 1 + 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index 93ba8bde6693..ee6f30112bdd 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -46,3 +46,4 @@ ittapi = { version = "0.4.0", optional = true } [features] profiling = ['dep:wasmtime-jit-debug', 'dep:ittapi'] demangle = ['dep:rustc-demangle', 'dep:cpp_demangle'] +debug-builtins = [] diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index a991c9a4cbb8..7f73d77451df 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -4,9 +4,8 @@ //! steps. use crate::code_memory::CodeMemory; -use crate::debug::create_gdbjit_image; use crate::profiling::ProfilingAgent; -use anyhow::{bail, Context, Error, Result}; +use anyhow::{bail, Error, Result}; use object::write::{Object, SectionId, StandardSegment, WritableBuffer}; use object::SectionKind; use serde_derive::{Deserialize, Serialize}; @@ -19,9 +18,7 @@ use wasmtime_environ::{ DefinedFuncIndex, FuncIndex, FunctionLoc, MemoryInitialization, Module, ModuleTranslation, PrimaryMap, SignatureIndex, StackMapInformation, Tunables, WasmFunctionInfo, }; -use wasmtime_runtime::{ - CompiledModuleId, CompiledModuleIdAllocator, GdbJitImageRegistration, MmapVec, -}; +use wasmtime_runtime::{CompiledModuleId, CompiledModuleIdAllocator, MmapVec}; /// Secondary in-memory results of function compilation. #[derive(Serialize, Deserialize)] @@ -426,7 +423,8 @@ pub struct CompiledModule { wasm_to_native_trampolines: Vec<(SignatureIndex, FunctionLoc)>, meta: Metadata, code_memory: Arc, - dbg_jit_registration: Option, + #[cfg(feature = "debug-builtins")] + dbg_jit_registration: Option, /// A unique ID used to register this module with the engine. unique_id: CompiledModuleId, func_names: Vec, @@ -459,6 +457,7 @@ impl CompiledModule { module: Arc::new(info.module), funcs: info.funcs, wasm_to_native_trampolines: info.wasm_to_native_trampolines, + #[cfg(feature = "debug-builtins")] dbg_jit_registration: None, code_memory, meta: info.meta, @@ -471,11 +470,17 @@ impl CompiledModule { } fn register_debug_and_profiling(&mut self, profiler: &dyn ProfilingAgent) -> Result<()> { + #[cfg(feature = "debug-builtins")] if self.meta.native_debug_info_present { + use anyhow::Context; + let text = self.text(); - let bytes = create_gdbjit_image(self.mmap().to_vec(), (text.as_ptr(), text.len())) - .context("failed to create jit image for gdb")?; - let reg = GdbJitImageRegistration::register(bytes); + let bytes = crate::debug::create_gdbjit_image( + self.mmap().to_vec(), + (text.as_ptr(), text.len()), + ) + .context("failed to create jit image for gdb")?; + let reg = wasmtime_runtime::GdbJitImageRegistration::register(bytes); self.dbg_jit_registration = Some(reg); } profiler.register_module(&self.code_memory, &|addr| { @@ -669,6 +674,7 @@ impl CompiledModule { /// what filename and line number a wasm pc comes from. #[cfg(feature = "addr2line")] pub fn symbolize_context(&self) -> Result>> { + use anyhow::Context; use gimli::EndianSlice; if !self.meta.has_wasm_debuginfo { return Ok(None); diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 438e44aacc24..142fd00da060 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -3,6 +3,7 @@ #![deny(missing_docs)] mod code_memory; +#[cfg(feature = "debug-builtins")] mod debug; mod demangling; mod instantiate; diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index fa0074e74886..c2ad2363fe7f 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -18,7 +18,7 @@ wasmtime-wmemcheck = { workspace = true } wasmtime-asm-macros = { workspace = true } wasmtime-environ = { workspace = true } wasmtime-fiber = { workspace = true, optional = true } -wasmtime-jit-debug = { workspace = true, features = ["gdb_jit_int"] } +wasmtime-jit-debug = { workspace = true, features = ["gdb_jit_int"], optional = true } wasmtime-versioned-export-macros = { workspace = true } libc = { version = "0.2.112", default-features = false } log = { workspace = true } @@ -64,4 +64,4 @@ async = ["wasmtime-fiber"] pooling-allocator = [] component-model = ["wasmtime-environ/component-model", "dep:encoding_rs"] wmemcheck = [] -debug-builtins = [] +debug-builtins = ['wasmtime-jit-debug'] diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index d3cf7565abbf..2bb277622ab7 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -34,6 +34,7 @@ pub mod debug_builtins; pub mod libcalls; pub mod mpk; +#[cfg(feature = "debug-builtins")] pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration; pub use crate::export::*; From eebe1d77cd3dbf94e500eedb703902232e044e22 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sat, 2 Dec 2023 14:00:40 -0800 Subject: [PATCH 02/15] Move timing in Cranelift behind a Cargo feature This commit adds a `timing` feature to the `cranelift-codegen` crate which is enabled by default. This feature gates the timing functionality in Cranelift to enable turning it off if desired. The goal of this commit is to remove a system dependency on `Instant` for possibly esoteric environments. --- cranelift/Cargo.toml | 2 +- cranelift/codegen/Cargo.toml | 7 +- cranelift/codegen/src/timing.rs | 122 +++++++++++++++++++------------- crates/cranelift/Cargo.toml | 2 +- 4 files changed, 82 insertions(+), 51 deletions(-) diff --git a/cranelift/Cargo.toml b/cranelift/Cargo.toml index 06dcf4ef8ce6..31dd54578b1b 100644 --- a/cranelift/Cargo.toml +++ b/cranelift/Cargo.toml @@ -23,7 +23,7 @@ harness = false [dependencies] cfg-if = { workspace = true } -cranelift-codegen = { workspace = true, features = ["disas", "trace-log"] } +cranelift-codegen = { workspace = true, features = ["disas", "trace-log", "timing"] } cranelift-entity = { workspace = true } cranelift-interpreter = { workspace = true } cranelift-reader = { workspace = true } diff --git a/cranelift/codegen/Cargo.toml b/cranelift/codegen/Cargo.toml index bde448f5a5a1..25ae26af144f 100644 --- a/cranelift/codegen/Cargo.toml +++ b/cranelift/codegen/Cargo.toml @@ -48,7 +48,7 @@ cranelift-codegen-meta = { path = "meta", version = "0.103.0" } cranelift-isle = { path = "../isle/isle", version = "=0.103.0" } [features] -default = ["std", "unwind", "host-arch"] +default = ["std", "unwind", "host-arch", "timing"] # The "std" feature enables use of libstd. The "core" feature enables use # of some minimal std-like replacement libraries. At least one of these two @@ -114,6 +114,11 @@ isle-errors = ["cranelift-isle/fancy-errors"] # inspection, rather than inside of target/. isle-in-source-tree = [] +# Enable tracking how long passes take in Cranelift. +# +# Enabled by default. +timing = [] + [[bench]] name = "x64-evex-encoding" harness = false diff --git a/cranelift/codegen/src/timing.rs b/cranelift/codegen/src/timing.rs index 573b82c0cd96..47f0eaec1cec 100644 --- a/cranelift/codegen/src/timing.rs +++ b/cranelift/codegen/src/timing.rs @@ -5,9 +5,9 @@ use core::fmt; use std::any::Any; use std::boxed::Box; -use std::cell::{Cell, RefCell}; +use std::cell::RefCell; use std::mem; -use std::time::{Duration, Instant}; +use std::time::Duration; // Each pass that can be timed is predefined with the `define_passes!` macro. Each pass has a // snake_case name and a plain text description used when printing out the timing report. @@ -130,22 +130,6 @@ fn start_pass(pass: Pass) -> Box { PROFILER.with(|profiler| profiler.borrow().start_pass(pass)) } -/// A timing token is responsible for timing the currently running pass. Timing starts when it -/// is created and ends when it is dropped. -/// -/// Multiple passes can be active at the same time, but they must be started and stopped in a -/// LIFO fashion. -struct DefaultTimingToken { - /// Start time for this pass. - start: Instant, - - // Pass being timed by this token. - pass: Pass, - - // The previously active pass which will be restored when this token is dropped. - prev: Pass, -} - /// Accumulated timing information for a single pass. #[derive(Default, Copy, Clone)] struct PassTime { @@ -215,42 +199,12 @@ impl fmt::Display for PassTimes { // Information about passes in a single thread. thread_local! { - static CURRENT_PASS: Cell = const { Cell::new(Pass::None) }; static PASS_TIME: RefCell = RefCell::new(Default::default()); } /// The default profiler. You can get the results using [`take_current`]. pub struct DefaultProfiler; -impl Profiler for DefaultProfiler { - fn start_pass(&self, pass: Pass) -> Box { - let prev = CURRENT_PASS.with(|p| p.replace(pass)); - log::debug!("timing: Starting {}, (during {})", pass, prev); - Box::new(DefaultTimingToken { - start: Instant::now(), - pass, - prev, - }) - } -} - -/// Dropping a timing token indicated the end of the pass. -impl Drop for DefaultTimingToken { - fn drop(&mut self) { - let duration = self.start.elapsed(); - log::debug!("timing: Ending {}: {}ms", self.pass, duration.as_millis()); - let old_cur = CURRENT_PASS.with(|p| p.replace(self.prev)); - debug_assert_eq!(self.pass, old_cur, "Timing tokens dropped out of order"); - PASS_TIME.with(|rc| { - let mut table = rc.borrow_mut(); - table.pass[self.pass.idx()].total += duration; - if let Some(parent) = table.pass.get_mut(self.prev.idx()) { - parent.child += duration; - } - }) - } -} - /// Take the current accumulated pass timings and reset the timings for the current thread. /// /// Only applies when [`DefaultProfiler`] is used. @@ -258,6 +212,78 @@ pub fn take_current() -> PassTimes { PASS_TIME.with(|rc| mem::take(&mut *rc.borrow_mut())) } +#[cfg(feature = "timing")] +mod enabled { + use super::{DefaultProfiler, Pass, Profiler, PASS_TIME}; + use std::any::Any; + use std::boxed::Box; + use std::cell::Cell; + use std::time::Instant; + + // Information about passes in a single thread. + thread_local! { + static CURRENT_PASS: Cell = const { Cell::new(Pass::None) }; + } + + impl Profiler for DefaultProfiler { + fn start_pass(&self, pass: Pass) -> Box { + let prev = CURRENT_PASS.with(|p| p.replace(pass)); + log::debug!("timing: Starting {}, (during {})", pass, prev); + Box::new(DefaultTimingToken { + start: Instant::now(), + pass, + prev, + }) + } + } + + /// A timing token is responsible for timing the currently running pass. Timing starts when it + /// is created and ends when it is dropped. + /// + /// Multiple passes can be active at the same time, but they must be started and stopped in a + /// LIFO fashion. + struct DefaultTimingToken { + /// Start time for this pass. + start: Instant, + + // Pass being timed by this token. + pass: Pass, + + // The previously active pass which will be restored when this token is dropped. + prev: Pass, + } + + /// Dropping a timing token indicated the end of the pass. + impl Drop for DefaultTimingToken { + fn drop(&mut self) { + let duration = self.start.elapsed(); + log::debug!("timing: Ending {}: {}ms", self.pass, duration.as_millis()); + let old_cur = CURRENT_PASS.with(|p| p.replace(self.prev)); + debug_assert_eq!(self.pass, old_cur, "Timing tokens dropped out of order"); + PASS_TIME.with(|rc| { + let mut table = rc.borrow_mut(); + table.pass[self.pass.idx()].total += duration; + if let Some(parent) = table.pass.get_mut(self.prev.idx()) { + parent.child += duration; + } + }) + } + } +} + +#[cfg(not(feature = "timing"))] +mod disabled { + use super::{DefaultProfiler, Pass, Profiler}; + use std::any::Any; + use std::boxed::Box; + + impl Profiler for DefaultProfiler { + fn start_pass(&self, _pass: Pass) -> Box { + Box::new(()) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index 733c9a377810..354e29e163bd 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -18,7 +18,7 @@ anyhow = { workspace = true } log = { workspace = true } wasmtime-environ = { workspace = true } cranelift-wasm = { workspace = true } -cranelift-codegen = { workspace = true, features = ["default"] } +cranelift-codegen = { workspace = true, features = ["host-arch"] } cranelift-frontend = { workspace = true } cranelift-entity = { workspace = true } cranelift-native = { workspace = true } From 554efbd6d1119130316f5c26e870382bc05f3cc5 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sat, 2 Dec 2023 14:03:11 -0800 Subject: [PATCH 03/15] Consolidate platform-specific definitions in Wasmtime Prior to this commit Wasmtime did not not have a style or system for containing platform-specific logic in files. The goal of this commit is to consolidate all platform-specific functionality into one location to make it easier to port Wasmtime to new systems. This commit creates a `sys` module within the `wasmtime-runtime` crate which conditionally defines all of the platform support that Wasmtime requires, namely things related to virtual memory management and trap handling. Many of the previous `unix.rs` files interspersed throughout the tree are now all located in a single `unix` directory. This additionally helps split out `miri`-specific functionality by pretending `miri` is its own platform. This change additionally goes through `#[cfg]` directives throughout `wasmtime-runtime`, `wasmtime-jit`, and `wasmtime` itself to place all of this target-specific functionality within this `sys` directory. The end state is that there are two new top-level modules in the `wasmtime-runtime` crate: * `arch` - this conditionally defines architecture-specific logic such as the state used by backtraces, libcalls, etc. * `sys` - this conditionally defines OS or platform-specific logic such as virtual memory management. One secondary goal of this commit is to enable the ability to easily add new platforms to Wasmtime, even if it's in a fork of Wasmtime. Previously patches might have to touch a good number of locations where now they'd ideally only have to touch `sys/mod.rs` to declare a new platform and `sys/$platform/*.rs` to define all the functionality. --- Cargo.lock | 2 +- crates/jit/src/code_memory.rs | 4 +- crates/jit/src/lib.rs | 1 - crates/jit/src/unwind.rs | 14 - crates/runtime/Cargo.toml | 9 +- crates/runtime/build.rs | 11 +- .../backtrace => arch}/aarch64.rs | 65 +++++ crates/runtime/src/arch/mod.rs | 32 +++ .../src/{trampolines => arch}/riscv64.rs | 43 +++ .../runtime/src/{trampolines => arch}/s390x.S | 0 crates/runtime/src/arch/s390x.rs | 61 +++++ .../src/{trampolines => arch}/x86_64.rs | 58 +++- crates/runtime/src/cow.rs | 252 +++--------------- .../runtime/src/instance/allocator/pooling.rs | 10 - .../instance/allocator/pooling/memory_pool.rs | 2 +- .../instance/allocator/pooling/stack_pool.rs | 8 +- .../instance/allocator/pooling/table_pool.rs | 12 +- .../src/instance/allocator/pooling/unix.rs | 56 ---- .../src/instance/allocator/pooling/windows.rs | 38 --- crates/runtime/src/lib.rs | 26 +- crates/runtime/src/libcalls.rs | 1 + crates/runtime/src/mmap.rs | 24 +- .../src/{mmap/miri.rs => sys/miri/mmap.rs} | 0 crates/runtime/src/sys/miri/mod.rs | 10 + crates/runtime/src/sys/miri/traphandlers.rs | 42 +++ .../src/sys/miri/unwind.rs} | 2 + crates/runtime/src/sys/miri/vm.rs | 63 +++++ crates/runtime/src/sys/mod.rs | 30 +++ .../macos.rs => sys/unix/machports.rs} | 13 +- .../src/sys/unix/macos_traphandlers.rs | 28 ++ .../src/{mmap/unix.rs => sys/unix/mmap.rs} | 0 crates/runtime/src/sys/unix/mod.rs | 21 ++ .../unix.rs => sys/unix/signals.rs} | 33 ++- .../src/sys/unix/unwind.rs} | 3 +- crates/runtime/src/sys/unix/vm.rs | 208 +++++++++++++++ .../{mmap/windows.rs => sys/windows/mmap.rs} | 0 crates/runtime/src/sys/windows/mod.rs | 6 + .../windows/traphandlers.rs} | 18 +- .../src/sys/windows/unwind.rs} | 2 + crates/runtime/src/sys/windows/vm.rs | 73 +++++ crates/runtime/src/trampolines.rs | 19 -- crates/runtime/src/trampolines/aarch64.rs | 42 --- crates/runtime/src/trampolines/s390x.rs | 25 -- crates/runtime/src/traphandlers.rs | 122 ++------- crates/runtime/src/traphandlers/backtrace.rs | 28 +- .../src/traphandlers/backtrace/riscv64.rs | 18 -- .../src/traphandlers/backtrace/s390x.rs | 22 -- .../src/traphandlers/backtrace/x86_64.rs | 20 -- crates/wasmtime/Cargo.toml | 3 - crates/wasmtime/src/func.rs | 39 +-- crates/wasmtime/src/v128.rs | 37 +-- 51 files changed, 890 insertions(+), 766 deletions(-) delete mode 100644 crates/jit/src/unwind.rs rename crates/runtime/src/{traphandlers/backtrace => arch}/aarch64.rs (53%) create mode 100644 crates/runtime/src/arch/mod.rs rename crates/runtime/src/{trampolines => arch}/riscv64.rs (53%) rename crates/runtime/src/{trampolines => arch}/s390x.S (100%) create mode 100644 crates/runtime/src/arch/s390x.rs rename crates/runtime/src/{trampolines => arch}/x86_64.rs (51%) delete mode 100644 crates/runtime/src/instance/allocator/pooling/unix.rs delete mode 100644 crates/runtime/src/instance/allocator/pooling/windows.rs rename crates/runtime/src/{mmap/miri.rs => sys/miri/mmap.rs} (100%) create mode 100644 crates/runtime/src/sys/miri/mod.rs create mode 100644 crates/runtime/src/sys/miri/traphandlers.rs rename crates/{jit/src/unwind/miri.rs => runtime/src/sys/miri/unwind.rs} (93%) create mode 100644 crates/runtime/src/sys/miri/vm.rs create mode 100644 crates/runtime/src/sys/mod.rs rename crates/runtime/src/{traphandlers/macos.rs => sys/unix/machports.rs} (98%) create mode 100644 crates/runtime/src/sys/unix/macos_traphandlers.rs rename crates/runtime/src/{mmap/unix.rs => sys/unix/mmap.rs} (100%) create mode 100644 crates/runtime/src/sys/unix/mod.rs rename crates/runtime/src/{traphandlers/unix.rs => sys/unix/signals.rs} (95%) rename crates/{jit/src/unwind/systemv.rs => runtime/src/sys/unix/unwind.rs} (97%) create mode 100644 crates/runtime/src/sys/unix/vm.rs rename crates/runtime/src/{mmap/windows.rs => sys/windows/mmap.rs} (100%) create mode 100644 crates/runtime/src/sys/windows/mod.rs rename crates/runtime/src/{traphandlers/windows.rs => sys/windows/traphandlers.rs} (88%) rename crates/{jit/src/unwind/winx64.rs => runtime/src/sys/windows/unwind.rs} (95%) create mode 100644 crates/runtime/src/sys/windows/vm.rs delete mode 100644 crates/runtime/src/trampolines.rs delete mode 100644 crates/runtime/src/trampolines/aarch64.rs delete mode 100644 crates/runtime/src/trampolines/s390x.rs delete mode 100644 crates/runtime/src/traphandlers/backtrace/riscv64.rs delete mode 100644 crates/runtime/src/traphandlers/backtrace/s390x.rs delete mode 100644 crates/runtime/src/traphandlers/backtrace/x86_64.rs diff --git a/Cargo.lock b/Cargo.lock index ae5e9bbb3e21..498d1ca0b4d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3207,7 +3207,6 @@ dependencies = [ "object", "once_cell", "paste", - "psm", "rayon", "serde", "serde_derive", @@ -3618,6 +3617,7 @@ dependencies = [ "once_cell", "paste", "proptest", + "psm", "rand", "rustix", "sptr", diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index 517b5d1b9768..2a08b5af2c20 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -1,7 +1,6 @@ //! Memory management for executable code. use crate::subslice_range; -use crate::unwind::UnwindRegistration; use anyhow::{anyhow, bail, Context, Result}; use object::read::{File, Object, ObjectSection}; use object::ObjectSymbol; @@ -9,8 +8,7 @@ use std::mem::ManuallyDrop; use std::ops::Range; use wasmtime_environ::obj; use wasmtime_jit_icache_coherence as icache_coherence; -use wasmtime_runtime::libcalls; -use wasmtime_runtime::MmapVec; +use wasmtime_runtime::{libcalls, MmapVec, UnwindRegistration}; /// Management of executable memory within a `MmapVec` /// diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 142fd00da060..7aac110b8416 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -8,7 +8,6 @@ mod debug; mod demangling; mod instantiate; pub mod profiling; -mod unwind; pub use crate::code_memory::CodeMemory; #[cfg(feature = "addr2line")] diff --git a/crates/jit/src/unwind.rs b/crates/jit/src/unwind.rs deleted file mode 100644 index 9ce8660f36c9..000000000000 --- a/crates/jit/src/unwind.rs +++ /dev/null @@ -1,14 +0,0 @@ -cfg_if::cfg_if! { - if #[cfg(all(windows, any(target_arch = "x86_64", target_arch = "aarch64")))] { - mod winx64; - pub use self::winx64::*; - } else if #[cfg(miri)] { - mod miri; - pub use self::miri::*; - } else if #[cfg(unix)] { - mod systemv; - pub use self::systemv::*; - } else { - compile_error!("unsupported target platform for unwind"); - } -} diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index c2ad2363fe7f..3738335b4035 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -25,14 +25,15 @@ log = { workspace = true } memoffset = "0.9.0" indexmap = { workspace = true } cfg-if = { workspace = true } -rand = { version = "0.8.3", features = ['small_rng'] } anyhow = { workspace = true } -memfd = "0.6.2" paste = "1.0.3" encoding_rs = { version = "0.8.31", optional = true } sptr = "0.3.2" wasm-encoder = { workspace = true } +[target.'cfg(target_os = "linux")'.dependencies] +memfd = "0.6.2" + [target.'cfg(target_os = "macos")'.dependencies] mach = "0.3.2" @@ -51,9 +52,13 @@ features = [ "Win32_Security", ] +[target.'cfg(target_arch = "s390x")'.dependencies] +psm = "0.1.11" + [dev-dependencies] once_cell = { workspace = true } proptest = "1.0.0" +rand = { version = "0.8.3", features = ['small_rng'] } [build-dependencies] cc = "1.0" diff --git a/crates/runtime/build.rs b/crates/runtime/build.rs index 571f0c4b2267..d8ba14f6ccbd 100644 --- a/crates/runtime/build.rs +++ b/crates/runtime/build.rs @@ -2,6 +2,15 @@ use std::env; use wasmtime_versioned_export_macros::versioned_suffix; fn main() { + println!("cargo:rerun-if-changed=build.rs"); + + // If this platform is neither unix nor windows then there's no default need + // for a C helper library since `helpers.c` is tailored for just these + // platforms currently. + if env::var("CARGO_CFG_UNIX").is_err() && env::var("CARGO_CFG_WINDOWS").is_err() { + return; + } + let mut build = cc::Build::new(); build.warnings(true); let arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); @@ -11,7 +20,7 @@ fn main() { build.define("VERSIONED_SUFFIX", Some(versioned_suffix!())); if arch == "s390x" { println!("cargo:rerun-if-changed=src/trampolines/s390x.S"); - build.file("src/trampolines/s390x.S"); + build.file("src/arch/s390x.S"); } println!("cargo:rerun-if-changed=src/helpers.c"); build.file("src/helpers.c"); diff --git a/crates/runtime/src/traphandlers/backtrace/aarch64.rs b/crates/runtime/src/arch/aarch64.rs similarity index 53% rename from crates/runtime/src/traphandlers/backtrace/aarch64.rs rename to crates/runtime/src/arch/aarch64.rs index 8c36291e37af..e212b412d2e4 100644 --- a/crates/runtime/src/traphandlers/backtrace/aarch64.rs +++ b/crates/runtime/src/arch/aarch64.rs @@ -1,3 +1,24 @@ +//! Arm64-specific definitions of architecture-specific functions in Wasmtime. + +/// AArch64 uses vector registered which here is used with a vector type. +/// Note that the specific type shouldn't matter too much but the choice +/// of using a vector is the significant part. +pub type V128Abi = std::arch::aarch64::uint8x16_t; + +#[inline] +#[allow(missing_docs)] +pub fn get_stack_pointer() -> usize { + let stack_pointer: usize; + unsafe { + std::arch::asm!( + "mov {}, sp", + out(reg) stack_pointer, + options(nostack,nomem), + ); + } + stack_pointer +} + // The aarch64 calling conventions save the return PC one i64 above the FP and // the previous FP is pointed to by the current FP: // @@ -53,3 +74,47 @@ pub fn assert_fp_is_aligned(_fp: usize) { // // [0]: https://github.com/ARM-software/abi-aa/blob/2022Q1/aapcs64/aapcs64.rst#the-frame-pointer } + +#[rustfmt::skip] +macro_rules! wasm_to_libcall_trampoline { + ($libcall:ident ; $libcall_impl:ident) => { + wasmtime_asm_macros::asm_func!( + wasmtime_versioned_export_macros::versioned_stringify_ident!($libcall), + " + .cfi_startproc + bti c + + // Load the pointer to `VMRuntimeLimits` in `x9`. + ldur x9, [x0, #8] + + // Store the last Wasm FP into the `last_wasm_exit_fp` in the limits. + stur fp, [x9, #24] + + // Store the last Wasm PC into the `last_wasm_exit_pc` in the limits. + stur lr, [x9, #32] + + // Tail call to the actual implementation of this libcall. + b {} + + .cfi_endproc + ", + sym $libcall_impl + ); + }; +} +pub(crate) use wasm_to_libcall_trampoline; + +#[cfg(test)] +mod wasm_to_libcall_trampoline_offsets_tests { + use wasmtime_environ::{Module, PtrSize, VMOffsets}; + + #[test] + fn test() { + let module = Module::new(); + let offsets = VMOffsets::new(std::mem::size_of::<*mut u8>() as u8, &module); + + assert_eq!(8, offsets.vmctx_runtime_limits()); + assert_eq!(24, offsets.ptr.vmruntime_limits_last_wasm_exit_fp()); + assert_eq!(32, offsets.ptr.vmruntime_limits_last_wasm_exit_pc()); + } +} diff --git a/crates/runtime/src/arch/mod.rs b/crates/runtime/src/arch/mod.rs new file mode 100644 index 000000000000..a46f152dbd33 --- /dev/null +++ b/crates/runtime/src/arch/mod.rs @@ -0,0 +1,32 @@ +//! Architecture-specific support required by Wasmtime. +//! +//! This crate houses any architecture-specific tidbits required when running +//! Wasmtime. Each architecture has its own file in the `arch` folder which is +//! referenced here. +//! +//! All architectures have the same interface when exposed to the rest of the +//! crate. + +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub use x86_64::*; + } else if #[cfg(target_arch = "aarch64")] { + mod aarch64; + pub use aarch64::*; + } else if #[cfg(target_arch = "s390x")] { + mod s390x; + pub use s390x::*; + } else if #[cfg(target_arch = "riscv64")] { + mod riscv64; + pub use riscv64::*; + } else { + compile_error!( + "Wasmtime is being compiled for an architecture \ + that it does not support. If this architecture is \ + one you would like to see supported you may file an \ + issue on Wasmtime's issue tracker: \ + https://github.com/bytecodealliance/wasmtime/issues/new\ + "); + } +} diff --git a/crates/runtime/src/trampolines/riscv64.rs b/crates/runtime/src/arch/riscv64.rs similarity index 53% rename from crates/runtime/src/trampolines/riscv64.rs rename to crates/runtime/src/arch/riscv64.rs index b43afcbf4518..7f600b43dfe2 100644 --- a/crates/runtime/src/trampolines/riscv64.rs +++ b/crates/runtime/src/arch/riscv64.rs @@ -1,3 +1,45 @@ +//! Riscv64-specific definitions of architecture-specific functions in Wasmtime. + +/// RISC-V currently always passes all vector arguments indirectly in the +/// ABI. Currently Rust has no stable means of representing this meaning +/// that a 128-bit representation is chosen here but it can't be passed +/// directly to WebAssembly, for example, and must instead be passed +/// through an array-call trampoline. +pub type V128Abi = u128; + +#[inline] +#[allow(missing_docs)] +pub fn get_stack_pointer() -> usize { + let stack_pointer: usize; + unsafe { + std::arch::asm!( + "mv {}, sp", + out(reg) stack_pointer, + options(nostack,nomem), + ); + } + stack_pointer +} + +pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { + *(fp as *mut usize).offset(1) +} + +// And the current frame pointer points to the next older frame pointer. +pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; + +pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { + fp >= entry_sp +} + +pub fn assert_entry_sp_is_aligned(sp: usize) { + assert_eq!(sp % 16, 0, "stack should always be aligned to 16"); +} + +pub fn assert_fp_is_aligned(fp: usize) { + assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); +} + #[rustfmt::skip] macro_rules! wasm_to_libcall_trampoline { ($libcall:ident ; $libcall_impl:ident) => { @@ -28,6 +70,7 @@ macro_rules! wasm_to_libcall_trampoline { ); }; } +pub(crate) use wasm_to_libcall_trampoline; #[cfg(test)] mod wasm_to_libcall_trampoline_offsets_tests { diff --git a/crates/runtime/src/trampolines/s390x.S b/crates/runtime/src/arch/s390x.S similarity index 100% rename from crates/runtime/src/trampolines/s390x.S rename to crates/runtime/src/arch/s390x.S diff --git a/crates/runtime/src/arch/s390x.rs b/crates/runtime/src/arch/s390x.rs new file mode 100644 index 000000000000..3ad6889b1efb --- /dev/null +++ b/crates/runtime/src/arch/s390x.rs @@ -0,0 +1,61 @@ +//! s390x-specific definitions of architecture-specific functions in Wasmtime. + +/// Currently Rust has no stable means of representing vector registers +/// so like RISC-V at this time this uses a bland 128-bit representation. +pub type V128Abi = u128; + +#[inline] +#[allow(missing_docs)] +pub fn get_stack_pointer() -> usize { + psm::stack_pointer() as usize +} + +pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { + // The next older PC can be found in register %r14 at function entry, which + // was saved into slot 14 of the register save area pointed to by "FP" (the + // backchain pointer). + *(fp as *mut usize).offset(14) +} + +// The next older "FP" (backchain pointer) was saved in the slot pointed to +// by the current "FP". +pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; + +pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { + fp > entry_sp +} + +pub fn assert_entry_sp_is_aligned(sp: usize) { + assert_eq!(sp % 8, 0, "stack should always be aligned to 8"); +} + +pub fn assert_fp_is_aligned(fp: usize) { + assert_eq!(fp % 8, 0, "stack should always be aligned to 8"); +} + +// The implementation for libcall trampolines is in the s390x.S +// file. We provide this dummy definition of wasm_to_libcall_trampoline +// here to make libcalls.rs compile on s390x. Note that this means we +// have to duplicate the list of libcalls used in the assembler file. + +macro_rules! wasm_to_libcall_trampoline { + ($libcall:ident ; $libcall_impl:ident) => {}; +} +pub(crate) use wasm_to_libcall_trampoline; + +// The wasm_to_host_trampoline implementation is in the s390x.S +// file, but we still want to have this unit test here. +#[cfg(test)] +mod wasm_to_libcall_trampoline_offsets_tests { + use wasmtime_environ::{Module, PtrSize, VMOffsets}; + + #[test] + fn test() { + let module = Module::new(); + let offsets = VMOffsets::new(std::mem::size_of::<*mut u8>() as u8, &module); + + assert_eq!(8, offsets.vmctx_runtime_limits()); + assert_eq!(24, offsets.ptr.vmruntime_limits_last_wasm_exit_fp()); + assert_eq!(32, offsets.ptr.vmruntime_limits_last_wasm_exit_pc()); + } +} diff --git a/crates/runtime/src/trampolines/x86_64.rs b/crates/runtime/src/arch/x86_64.rs similarity index 51% rename from crates/runtime/src/trampolines/x86_64.rs rename to crates/runtime/src/arch/x86_64.rs index d22bcaf1a0e3..a179831f955a 100644 --- a/crates/runtime/src/trampolines/x86_64.rs +++ b/crates/runtime/src/arch/x86_64.rs @@ -1,21 +1,62 @@ +//! x86_64-specific definitions of architecture-specific functions in Wasmtime. + +/// x86 vectors are represented with XMM registers which are represented +/// with the `__m128i` type. This type is considered a vector type for +/// ABI purposes which is implemented by Cranelift. +pub type V128Abi = std::arch::x86_64::__m128i; + +#[inline] +#[allow(missing_docs)] +pub fn get_stack_pointer() -> usize { + let stack_pointer: usize; + unsafe { + std::arch::asm!( + "mov {}, rsp", + out(reg) stack_pointer, + options(nostack,nomem), + ); + } + stack_pointer +} + +pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { + // The calling convention always pushes the return pointer (aka the PC of + // the next older frame) just before this frame. + *(fp as *mut usize).offset(1) +} + +// And the current frame pointer points to the next older frame pointer. +pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; + +pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { + fp >= entry_sp +} + +pub fn assert_entry_sp_is_aligned(sp: usize) { + assert_eq!(sp % 16, 0, "stack should always be aligned to 16"); +} + +pub fn assert_fp_is_aligned(fp: usize) { + assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); +} + // Helper macros for getting the first and second arguments according to the // system calling convention, as well as some callee-saved scratch registers we // can safely use in the trampolines. cfg_if::cfg_if! { if #[cfg(windows)] { macro_rules! callee_vmctx { () => ("rcx") } - #[allow(unused)] macro_rules! caller_vmctx { () => ("rdx") } macro_rules! scratch0 { () => ("r10") } macro_rules! scratch1 { () => ("r11") } } else if #[cfg(unix)] { macro_rules! callee_vmctx { () => ("rdi") } - #[allow(unused)] macro_rules! caller_vmctx { () => ("rsi") } macro_rules! scratch0 { () => ("r10") } macro_rules! scratch1 { () => ("r11") } } else { - compile_error!("platform not supported"); + compile_error!("default calling convention for this platform is not known"); } } +pub(crate) use {callee_vmctx, scratch0, scratch1}; #[rustfmt::skip] macro_rules! wasm_to_libcall_trampoline { @@ -27,15 +68,15 @@ macro_rules! wasm_to_libcall_trampoline { .cfi_startproc simple .cfi_def_cfa_offset 0 - // Load the pointer to `VMRuntimeLimits` in `", scratch0!(), "`. - mov ", scratch0!(), ", 8[", callee_vmctx!(), "] + // Load the pointer to `VMRuntimeLimits` in `scratch0!()`. + mov ", crate::arch::scratch0!(), ", 8[", crate::arch::callee_vmctx!(), "] // Store the last Wasm FP into the `last_wasm_exit_fp` in the limits. - mov 24[", scratch0!(), "], rbp + mov 24[", crate::arch::scratch0!(), "], rbp // Store the last Wasm PC into the `last_wasm_exit_pc` in the limits. - mov ", scratch1!(), ", [rsp] - mov 32[", scratch0!(), "], ", scratch1!(), " + mov ", crate::arch::scratch1!(), ", [rsp] + mov 32[", crate::arch::scratch0!(), "], ", crate::arch::scratch1!(), " // Tail call to the actual implementation of this libcall. jmp {} @@ -47,6 +88,7 @@ macro_rules! wasm_to_libcall_trampoline { ); }; } +pub(crate) use wasm_to_libcall_trampoline; #[cfg(test)] mod wasm_to_libcall_trampoline_offsets_tests { diff --git a/crates/runtime/src/cow.rs b/crates/runtime/src/cow.rs index ccd34b828910..0fe6a8416c75 100644 --- a/crates/runtime/src/cow.rs +++ b/crates/runtime/src/cow.rs @@ -1,12 +1,10 @@ //! Copy-on-write initialization support: creation of backing images for //! modules, and logic to support mapping these backing images into memory. -#![cfg_attr(any(not(unix), miri), allow(unused_imports, unused_variables))] - +use crate::sys::vm::{self, MemoryImageSource}; use crate::{MmapVec, SendSyncPtr}; use anyhow::Result; -use libc::c_void; -use std::fs::File; +use std::ffi::c_void; use std::ptr::NonNull; use std::sync::Arc; use std::{convert::TryFrom, ops::Range}; @@ -37,7 +35,7 @@ pub struct MemoryImage { /// This might be an mmaped `*.cwasm` file or on Linux it could also be a /// `Memfd` as an anonymous file in memory. In either case this is used as /// the backing-source for the CoW image. - fd: FdSource, + fd: MemoryImageSource, /// Length of image, in bytes. /// @@ -61,39 +59,6 @@ pub struct MemoryImage { linear_memory_offset: usize, } -#[derive(Debug)] -enum FdSource { - #[cfg(all(unix, not(miri)))] - Mmap(Arc), - #[cfg(all(target_os = "linux", not(miri)))] - Memfd(memfd::Memfd), -} - -impl FdSource { - #[cfg(all(unix, not(miri)))] - fn as_file(&self) -> &File { - match self { - FdSource::Mmap(ref file) => file, - #[cfg(target_os = "linux")] - FdSource::Memfd(ref memfd) => memfd.as_file(), - } - } -} - -impl PartialEq for FdSource { - fn eq(&self, other: &FdSource) -> bool { - cfg_if::cfg_if! { - if #[cfg(all(unix, not(miri)))] { - use rustix::fd::AsRawFd; - self.as_file().as_raw_fd() == other.as_file().as_raw_fd() - } else { - let _ = other; - match *self {} - } - } - } -} - impl MemoryImage { fn new( page_size: u32, @@ -124,7 +89,6 @@ impl MemoryImage { // files, but for now this is still a Linux-specific region of Wasmtime. // Some work will be needed to get this file compiling for macOS and // Windows. - #[cfg(not(any(windows, miri)))] if let Some(mmap) = mmap { let start = mmap.as_ptr() as usize; let end = start + mmap.len(); @@ -137,126 +101,45 @@ impl MemoryImage { assert_eq!((mmap.original_offset() as u32) % page_size, 0); if let Some(file) = mmap.original_file() { - return Ok(Some(MemoryImage { - fd: FdSource::Mmap(file.clone()), - fd_offset: u64::try_from(mmap.original_offset() + (data_start - start)) - .unwrap(), - linear_memory_offset, - len, - })); + if let Some(fd) = MemoryImageSource::from_file(file) { + return Ok(Some(MemoryImage { + fd, + fd_offset: u64::try_from(mmap.original_offset() + (data_start - start)) + .unwrap(), + linear_memory_offset, + len, + })); + } } } // If `mmap` doesn't come from a file then platform-specific mechanisms // may be used to place the data in a form that's amenable to an mmap. - - cfg_if::cfg_if! { - if #[cfg(all(target_os = "linux", not(miri)))] { - // On Linux `memfd_create` is used to create an anonymous - // in-memory file to represent the heap image. This anonymous - // file is then used as the basis for further mmaps. - - use std::io::Write; - - let memfd = match create_memfd()? { - Some(memfd) => memfd, - None => return Ok(None), - }; - memfd.as_file().write_all(data)?; - - // Seal the memfd's data and length. - // - // This is a defense-in-depth security mitigation. The - // memfd will serve as the starting point for the heap of - // every instance of this module. If anything were to - // write to this, it could affect every execution. The - // memfd object itself is owned by the machinery here and - // not exposed elsewhere, but it is still an ambient open - // file descriptor at the syscall level, so some other - // vulnerability that allowed writes to arbitrary fds - // could modify it. Or we could have some issue with the - // way that we map it into each instance. To be - // extra-super-sure that it never changes, and because - // this costs very little, we use the kernel's "seal" API - // to make the memfd image permanently read-only. - memfd.add_seals(&[ - memfd::FileSeal::SealGrow, - memfd::FileSeal::SealShrink, - memfd::FileSeal::SealWrite, - memfd::FileSeal::SealSeal, - ])?; - - Ok(Some(MemoryImage { - fd: FdSource::Memfd(memfd), - fd_offset: 0, - linear_memory_offset, - len, - })) - } else { - // Other platforms don't have an easily available way of - // representing the heap image as an mmap-source right now. We - // could theoretically create a file and immediately unlink it - // but that means that data may likely be preserved to disk - // which isn't what we want here. - Ok(None) - } + if let Some(fd) = MemoryImageSource::from_data(data)? { + return Ok(Some(MemoryImage { + fd, + fd_offset: 0, + linear_memory_offset, + len, + })); } + + Ok(None) } unsafe fn map_at(&self, base: *mut u8) -> Result<()> { - cfg_if::cfg_if! { - if #[cfg(all(unix, not(miri)))] { - let ptr = rustix::mm::mmap( - base.add(self.linear_memory_offset).cast(), - self.len, - rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE, - rustix::mm::MapFlags::PRIVATE | rustix::mm::MapFlags::FIXED, - self.fd.as_file(), - self.fd_offset, - )?; - assert_eq!(ptr, base.add(self.linear_memory_offset).cast()); - Ok(()) - } else { - match self.fd {} - } - } + self.fd.map_at( + base.add(self.linear_memory_offset), + self.len, + self.fd_offset, + )?; + Ok(()) } unsafe fn remap_as_zeros_at(&self, base: *mut u8) -> Result<()> { - cfg_if::cfg_if! { - if #[cfg(unix)] { - let ptr = rustix::mm::mmap_anonymous( - base.add(self.linear_memory_offset).cast(), - self.len, - rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE, - rustix::mm::MapFlags::PRIVATE | rustix::mm::MapFlags::FIXED, - )?; - assert_eq!(ptr.cast(), base.add(self.linear_memory_offset)); - Ok(()) - } else { - match self.fd {} - } - } - } -} - -#[cfg(all(target_os = "linux", not(miri)))] -fn create_memfd() -> Result> { - use std::io::ErrorKind; - - // Create the memfd. It needs a name, but the - // documentation for `memfd_create()` says that names can - // be duplicated with no issues. - match memfd::MemfdOptions::new() - .allow_sealing(true) - .create("wasm-memory-image") - { - Ok(memfd) => Ok(Some(memfd)), - // If this kernel is old enough to not support memfd then attempt to - // gracefully handle that and fall back to skipping the memfd - // optimization. - Err(memfd::Error::Create(err)) if err.kind() == ErrorKind::Unsupported => Ok(None), - Err(e) => Err(e.into()), + self.fd + .remap_as_zeros_at(base.add(self.linear_memory_offset), self.len)?; + Ok(()) } } @@ -593,7 +476,7 @@ impl MemoryImageSlot { #[allow(dead_code)] // ignore warnings as this is only used in some cfgs unsafe fn reset_all_memory_contents(&mut self, keep_resident: usize) -> Result<()> { - if !cfg!(target_os = "linux") || cfg!(miri) { + if !vm::supports_madvise_dontneed() { // If we're not on Linux then there's no generic platform way to // reset memory back to its original state, so instead reset memory // back to entirely zeros with an anonymous backing. @@ -703,18 +586,8 @@ impl MemoryImageSlot { if len == 0 { return Ok(()); } - cfg_if::cfg_if! { - if #[cfg(target_os = "linux")] { - rustix::mm::madvise( - self.base.as_ptr().add(base).cast(), - len, - rustix::mm::Advice::LinuxDontNeed, - )?; - Ok(()) - } else { - unreachable!(); - } - } + vm::madvise_dontneed(self.base.as_ptr().add(base), len)?; + Ok(()) } fn set_protection(&self, range: Range, readwrite: bool) -> Result<()> { @@ -726,30 +599,10 @@ impl MemoryImageSlot { unsafe { let start = self.base.as_ptr().add(range.start); - cfg_if::cfg_if! { - if #[cfg(miri)] { - if readwrite { - std::ptr::write_bytes(start, 0u8, range.len()); - } - } else if #[cfg(unix)] { - let flags = if readwrite { - rustix::mm::MprotectFlags::READ | rustix::mm::MprotectFlags::WRITE - } else { - rustix::mm::MprotectFlags::empty() - }; - rustix::mm::mprotect(start.cast(), range.len(), flags)?; - } else { - use windows_sys::Win32::System::Memory::*; - - let failure = if readwrite { - VirtualAlloc(start.cast(), range.len(), MEM_COMMIT, PAGE_READWRITE).is_null() - } else { - VirtualFree(start.cast(), range.len(), MEM_DECOMMIT) == 0 - }; - if failure { - return Err(std::io::Error::last_os_error().into()); - } - } + if readwrite { + vm::expose_exisiting_mapping(start, range.len())?; + } else { + vm::hide_existing_mapping(start, range.len())?; } } @@ -775,24 +628,7 @@ impl MemoryImageSlot { } unsafe { - cfg_if::cfg_if! { - if #[cfg(miri)] { - std::ptr::write_bytes(self.base.as_ptr(), 0, self.static_size); - } else if #[cfg(unix)] { - let ptr = rustix::mm::mmap_anonymous( - self.base.as_ptr().cast(), - self.static_size, - rustix::mm::ProtFlags::empty(), - rustix::mm::MapFlags::PRIVATE | rustix::mm::MapFlags::FIXED, - )?; - assert_eq!(ptr, self.base.as_ptr().cast()); - } else { - use windows_sys::Win32::System::Memory::*; - if VirtualFree(self.base.as_ptr().cast(), self.static_size, MEM_DECOMMIT) == 0 { - return Err(std::io::Error::last_os_error().into()); - } - } - } + vm::erase_existing_mapping(self.base.as_ptr(), self.static_size)?; } self.image = None; @@ -845,29 +681,21 @@ impl Drop for MemoryImageSlot { mod test { use std::sync::Arc; - use super::{FdSource, MemoryImage, MemoryImageSlot, MemoryPlan, MemoryStyle}; + use super::{MemoryImage, MemoryImageSlot, MemoryImageSource, MemoryPlan, MemoryStyle}; use crate::mmap::Mmap; use anyhow::Result; - use std::io::Write; use wasmtime_environ::Memory; - fn create_memfd() -> Result { - Ok(super::create_memfd()?.expect("kernel doesn't support memfd")) - } - fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result { // Offset must be page-aligned. let page_size = crate::page_size(); assert_eq!(offset & (page_size - 1), 0); - let memfd = create_memfd()?; - memfd.as_file().write_all(data)?; // The image length is rounded up to the nearest page size let image_len = (data.len() + page_size - 1) & !(page_size - 1); - memfd.as_file().set_len(image_len as u64)?; Ok(MemoryImage { - fd: FdSource::Memfd(memfd), + fd: MemoryImageSource::from_data(data)?.unwrap(), len: image_len, fd_offset: 0, linear_memory_offset: offset, diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index e0584726fa67..9fa5d83b301f 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -65,16 +65,6 @@ mod table_pool; #[cfg(all(feature = "async", unix, not(miri)))] mod stack_pool; -cfg_if::cfg_if! { - if #[cfg(windows)] { - mod windows; - use windows as imp; - } else { - mod unix; - use unix as imp; - } -} - use super::{ InstanceAllocationRequest, InstanceAllocatorImpl, MemoryAllocationIndex, TableAllocationIndex, }; diff --git a/crates/runtime/src/instance/allocator/pooling/memory_pool.rs b/crates/runtime/src/instance/allocator/pooling/memory_pool.rs index 3bf096927974..67aefe106980 100644 --- a/crates/runtime/src/instance/allocator/pooling/memory_pool.rs +++ b/crates/runtime/src/instance/allocator/pooling/memory_pool.rs @@ -68,7 +68,7 @@ use crate::{ MpkEnabled, PoolingInstanceAllocatorConfig, }; use anyhow::{anyhow, bail, Context, Result}; -use libc::c_void; +use std::ffi::c_void; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Mutex; use wasmtime_environ::{ diff --git a/crates/runtime/src/instance/allocator/pooling/stack_pool.rs b/crates/runtime/src/instance/allocator/pooling/stack_pool.rs index edb169f348da..eb1581d5f669 100644 --- a/crates/runtime/src/instance/allocator/pooling/stack_pool.rs +++ b/crates/runtime/src/instance/allocator/pooling/stack_pool.rs @@ -1,8 +1,8 @@ use super::{ - imp::{commit_stack_pages, reset_stack_pages_to_zero}, index_allocator::{SimpleIndexAllocator, SlotId}, round_up_to_pow2, }; +use crate::sys::vm::{commit_stack_pages, reset_stack_pages_to_zero}; use crate::{Mmap, PoolingInstanceAllocatorConfig}; use anyhow::{anyhow, bail, Context, Result}; @@ -166,10 +166,10 @@ impl StackPool { 0, size_to_memset, ); - } - // Use the system to reset remaining stack pages to zero. - reset_stack_pages_to_zero(bottom as _, size - size_to_memset).unwrap(); + // Use the system to reset remaining stack pages to zero. + reset_stack_pages_to_zero(bottom as _, size - size_to_memset).unwrap(); + } } } diff --git a/crates/runtime/src/instance/allocator/pooling/table_pool.rs b/crates/runtime/src/instance/allocator/pooling/table_pool.rs index 558cb548e966..45ce3c3b965f 100644 --- a/crates/runtime/src/instance/allocator/pooling/table_pool.rs +++ b/crates/runtime/src/instance/allocator/pooling/table_pool.rs @@ -1,8 +1,8 @@ use super::{ - imp::{commit_table_pages, decommit_table_pages}, index_allocator::{SimpleIndexAllocator, SlotId}, round_up_to_pow2, TableAllocationIndex, }; +use crate::sys::vm::{commit_table_pages, decommit_table_pages}; use crate::{InstanceAllocationRequest, Mmap, PoolingInstanceAllocatorConfig, SendSyncPtr, Table}; use anyhow::{anyhow, bail, Context, Result}; use std::mem; @@ -129,10 +129,12 @@ impl TablePool { match (|| { let base = self.get(allocation_index); - commit_table_pages( - base as *mut u8, - self.table_elements * mem::size_of::<*mut u8>(), - )?; + unsafe { + commit_table_pages( + base as *mut u8, + self.table_elements * mem::size_of::<*mut u8>(), + )?; + } let ptr = NonNull::new(std::ptr::slice_from_raw_parts_mut( base.cast(), diff --git a/crates/runtime/src/instance/allocator/pooling/unix.rs b/crates/runtime/src/instance/allocator/pooling/unix.rs deleted file mode 100644 index 00259f4b0afd..000000000000 --- a/crates/runtime/src/instance/allocator/pooling/unix.rs +++ /dev/null @@ -1,56 +0,0 @@ -use anyhow::Result; - -fn decommit(addr: *mut u8, len: usize) -> Result<()> { - if len == 0 { - return Ok(()); - } - - unsafe { - cfg_if::cfg_if! { - if #[cfg(miri)] { - std::ptr::write_bytes(addr, 0, len); - } else if #[cfg(target_os = "linux")] { - use rustix::mm::{madvise, Advice}; - - // On Linux, this is enough to cause the kernel to initialize - // the pages to 0 on next access - madvise(addr as _, len, Advice::LinuxDontNeed)?; - } else { - use rustix::mm::{mmap_anonymous, ProtFlags, MapFlags}; - - // By creating a new mapping at the same location, this will - // discard the mapping for the pages in the given range. - // The new mapping will be to the CoW zero page, so this - // effectively zeroes the pages. - mmap_anonymous( - addr as _, - len, - ProtFlags::READ | ProtFlags::WRITE, - MapFlags::PRIVATE | MapFlags::FIXED, - )?; - } - } - } - - Ok(()) -} - -pub fn commit_table_pages(_addr: *mut u8, _len: usize) -> Result<()> { - // A no-op as table pages remain READ|WRITE - Ok(()) -} - -pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { - decommit(addr, len) -} - -#[cfg(all(feature = "async", not(miri)))] -pub fn commit_stack_pages(_addr: *mut u8, _len: usize) -> Result<()> { - // A no-op as stack pages remain READ|WRITE - Ok(()) -} - -#[cfg(all(feature = "async", not(miri)))] -pub fn reset_stack_pages_to_zero(addr: *mut u8, len: usize) -> Result<()> { - decommit(addr, len) -} diff --git a/crates/runtime/src/instance/allocator/pooling/windows.rs b/crates/runtime/src/instance/allocator/pooling/windows.rs deleted file mode 100644 index 5e9d0c51e414..000000000000 --- a/crates/runtime/src/instance/allocator/pooling/windows.rs +++ /dev/null @@ -1,38 +0,0 @@ -use anyhow::{bail, Result}; -use windows_sys::Win32::System::Memory::*; - -pub fn commit(addr: *mut u8, len: usize) -> Result<()> { - if len == 0 { - return Ok(()); - } - - // Memory needs to be committed, so don't use the `region` crate - if unsafe { VirtualAlloc(addr as _, len, MEM_COMMIT, PAGE_READWRITE).is_null() } { - bail!("failed to commit memory as read/write"); - } - - Ok(()) -} - -pub fn decommit(addr: *mut u8, len: usize) -> Result<()> { - if len == 0 { - return Ok(()); - } - - if unsafe { VirtualFree(addr as _, len, MEM_DECOMMIT) } == 0 { - bail!( - "failed to decommit memory pages: {}", - std::io::Error::last_os_error() - ); - } - - Ok(()) -} - -pub fn commit_table_pages(addr: *mut u8, len: usize) -> Result<()> { - commit(addr, len) -} - -pub fn decommit_table_pages(addr: *mut u8, len: usize) -> Result<()> { - decommit(addr, len) -} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 2bb277622ab7..de6574a68304 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -10,9 +10,7 @@ use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; use wasmtime_environ::{DefinedFuncIndex, DefinedMemoryIndex, HostPtr, VMOffsets}; -#[macro_use] -mod trampolines; - +mod arch; #[cfg(feature = "component-model")] pub mod component; mod export; @@ -25,6 +23,7 @@ mod mmap_vec; mod parking_spot; mod send_sync_ptr; mod store_box; +mod sys; mod table; mod traphandlers; mod vmcontext; @@ -37,6 +36,7 @@ pub mod mpk; #[cfg(feature = "debug-builtins")] pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration; +pub use crate::arch::{get_stack_pointer, V128Abi}; pub use crate::export::*; pub use crate::externref::*; pub use crate::imports::Imports; @@ -55,6 +55,7 @@ pub use crate::mmap::Mmap; pub use crate::mmap_vec::MmapVec; pub use crate::mpk::MpkEnabled; pub use crate::store_box::*; +pub use crate::sys::unwind::UnwindRegistration; pub use crate::table::{Table, TableElement}; pub use crate::traphandlers::*; pub use crate::vmcontext::{ @@ -220,30 +221,13 @@ pub fn page_size() -> usize { return match PAGE_SIZE.load(Ordering::Relaxed) { 0 => { - let size = get_page_size(); + let size = sys::vm::get_page_size(); assert!(size != 0); PAGE_SIZE.store(size, Ordering::Relaxed); size } n => n, }; - - #[cfg(windows)] - fn get_page_size() -> usize { - use std::mem::MaybeUninit; - use windows_sys::Win32::System::SystemInformation::*; - - unsafe { - let mut info = MaybeUninit::uninit(); - GetSystemInfo(info.as_mut_ptr()); - info.assume_init_ref().dwPageSize as usize - } - } - - #[cfg(unix)] - fn get_page_size() -> usize { - unsafe { libc::sysconf(libc::_SC_PAGESIZE).try_into().unwrap() } - } } /// Result of [`Memory::atomic_wait32`] and [`Memory::atomic_wait64`] diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index b43c11b33cee..539a1b3f5f31 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -80,6 +80,7 @@ use wasmtime_wmemcheck::AccessError::{ /// now to ensure that the fp/sp on exit are recorded for backtraces to work /// properly. pub mod trampolines { + use crate::arch::wasm_to_libcall_trampoline; use crate::{Instance, TrapReason, VMContext}; macro_rules! libcall { diff --git a/crates/runtime/src/mmap.rs b/crates/runtime/src/mmap.rs index 341cd4deea36..4275cd68f833 100644 --- a/crates/runtime/src/mmap.rs +++ b/crates/runtime/src/mmap.rs @@ -1,30 +1,18 @@ //! Low-level abstraction for allocating and managing zero-filled pages //! of memory. +use crate::sys::mmap; use anyhow::{Context, Result}; use std::fs::File; use std::ops::Range; use std::path::Path; use std::sync::Arc; -cfg_if::cfg_if! { - if #[cfg(windows)] { - mod windows; - use windows as sys; - } else if #[cfg(miri)] { - mod miri; - use miri as sys; - } else { - mod unix; - use unix as sys; - } -} - /// A simple struct consisting of a page-aligned pointer to page-aligned /// and initially-zeroed memory and a length. #[derive(Debug)] pub struct Mmap { - sys: sys::Mmap, + sys: mmap::Mmap, file: Option>, } @@ -47,7 +35,7 @@ impl Mmap { /// The memory mapping and the length of the file within the mapping are /// returned. pub fn from_file(path: &Path) -> Result { - let (sys, file) = sys::Mmap::from_file(path)?; + let (sys, file) = mmap::Mmap::from_file(path)?; Ok(Mmap { sys, file: Some(Arc::new(file)), @@ -70,18 +58,18 @@ impl Mmap { if mapping_size == 0 { Ok(Mmap { - sys: sys::Mmap::new_empty(), + sys: mmap::Mmap::new_empty(), file: None, }) } else if accessible_size == mapping_size { Ok(Mmap { - sys: sys::Mmap::new(mapping_size) + sys: mmap::Mmap::new(mapping_size) .context(format!("mmap failed to allocate {mapping_size:#x} bytes"))?, file: None, }) } else { let mut result = Mmap { - sys: sys::Mmap::reserve(mapping_size) + sys: mmap::Mmap::reserve(mapping_size) .context(format!("mmap failed to reserve {mapping_size:#x} bytes"))?, file: None, }; diff --git a/crates/runtime/src/mmap/miri.rs b/crates/runtime/src/sys/miri/mmap.rs similarity index 100% rename from crates/runtime/src/mmap/miri.rs rename to crates/runtime/src/sys/miri/mmap.rs diff --git a/crates/runtime/src/sys/miri/mod.rs b/crates/runtime/src/sys/miri/mod.rs new file mode 100644 index 000000000000..76cf11e686ae --- /dev/null +++ b/crates/runtime/src/sys/miri/mod.rs @@ -0,0 +1,10 @@ +//! "Dummy" implementations of some system primitives for MIRI emulation. +//! +//! Note that at this time this is just enough to run some tests in MIRI but +//! notably WebAssembly tests are not executed at this time (MIRI can't execute +//! Cranelift-generated code). + +pub mod mmap; +pub mod traphandlers; +pub mod unwind; +pub mod vm; diff --git a/crates/runtime/src/sys/miri/traphandlers.rs b/crates/runtime/src/sys/miri/traphandlers.rs new file mode 100644 index 000000000000..9bb6358c48cd --- /dev/null +++ b/crates/runtime/src/sys/miri/traphandlers.rs @@ -0,0 +1,42 @@ +// With MIRI set up just enough of a setjmp/longjmp with catching panics +// to get a few tests working that use this. +// +// Note that no actual JIT code runs in MIRI so this is purely here for +// host-to-host calls. + +use crate::VMContext; + +struct WasmtimeLongjmp; + +pub fn wasmtime_setjmp( + _jmp_buf: *mut *const u8, + callback: extern "C" fn(*mut u8, *mut VMContext), + payload: *mut u8, + callee: *mut VMContext, +) -> i32 { + use std::panic::{self, AssertUnwindSafe}; + let result = panic::catch_unwind(AssertUnwindSafe(|| { + callback(payload, callee); + })); + match result { + Ok(()) => 1, + Err(e) => { + if e.is::() { + 0 + } else { + panic::resume_unwind(e) + } + } + } +} + +pub fn wasmtime_longjmp(_jmp_buf: *const u8) -> ! { + std::panic::panic_any(WasmtimeLongjmp) +} + +#[allow(missing_docs)] +pub type SignalHandler<'a> = dyn Fn() + Send + Sync + 'a; + +pub fn platform_init() {} + +pub fn lazy_per_thread_init() {} diff --git a/crates/jit/src/unwind/miri.rs b/crates/runtime/src/sys/miri/unwind.rs similarity index 93% rename from crates/jit/src/unwind/miri.rs rename to crates/runtime/src/sys/miri/unwind.rs index 4b4527e66834..d6181940cd0f 100644 --- a/crates/jit/src/unwind/miri.rs +++ b/crates/runtime/src/sys/miri/unwind.rs @@ -1,3 +1,5 @@ +#![allow(missing_docs)] + use anyhow::Result; pub struct UnwindRegistration {} diff --git a/crates/runtime/src/sys/miri/vm.rs b/crates/runtime/src/sys/miri/vm.rs new file mode 100644 index 000000000000..7b65a74b0976 --- /dev/null +++ b/crates/runtime/src/sys/miri/vm.rs @@ -0,0 +1,63 @@ +use std::fs::File; +use std::io; +use std::sync::Arc; + +pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + std::ptr::write_bytes(ptr, 0u8, len); + Ok(()) +} + +pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + std::ptr::write_bytes(ptr, 0, len); + Ok(()) +} + +pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + std::ptr::write_bytes(ptr, 0, len); + Ok(()) +} + +#[cfg(feature = "pooling-allocator")] +pub unsafe fn commit_table_pages(ptr: *mut u8, len: usize) -> io::Result<()> { + std::ptr::write_bytes(ptr, 0, len); + Ok(()) +} + +#[cfg(feature = "pooling-allocator")] +pub unsafe fn decommit_table_pages(ptr: *mut u8, len: usize) -> io::Result<()> { + std::ptr::write_bytes(ptr, 0, len); + Ok(()) +} + +pub fn get_page_size() -> usize { + 4096 +} + +pub fn supports_madvise_dontneed() -> bool { + false +} + +pub unsafe fn madvise_dontneed(_ptr: *mut u8, _len: usize) -> io::Result<()> { + unreachable!() +} + +#[derive(PartialEq, Debug)] +pub enum MemoryImageSource {} + +impl MemoryImageSource { + pub fn from_file(_file: &Arc) -> Option { + None + } + + pub fn from_data(_data: &[u8]) -> io::Result> { + Ok(None) + } + + pub unsafe fn map_at(&self, _base: *mut u8, _len: usize, _offset: u64) -> io::Result<()> { + match *self {} + } + + pub unsafe fn remap_as_zeros_at(&self, _base: *mut u8, _len: usize) -> io::Result<()> { + match *self {} + } +} diff --git a/crates/runtime/src/sys/mod.rs b/crates/runtime/src/sys/mod.rs new file mode 100644 index 000000000000..23815f444a65 --- /dev/null +++ b/crates/runtime/src/sys/mod.rs @@ -0,0 +1,30 @@ +//! OS-related abstractions required by Wasmtime. +//! +//! This module is intended to house all logic that's specific to either Unix +//! for Windows, for example. The goal of this module is to be the "single +//! module" to edit if Wasmtime is ported to a new platform. Ideally all that's +//! needed is an extra block below and a new platform should be good to go after +//! filling out the implementation. + +#![allow(clippy::cast_sign_loss)] // platforms too fiddly to worry about this + +cfg_if::cfg_if! { + if #[cfg(miri)] { + mod miri; + pub use miri::*; + } else if #[cfg(windows)] { + mod windows; + pub use windows::*; + } else if #[cfg(unix)] { + mod unix; + pub use unix::*; + } else { + compile_error!( + "Wasmtime is being compiled for a platform \ + that it does not support. If this platform is \ + one you would like to see supported you may file an \ + issue on Wasmtime's issue tracker: \ + https://github.com/bytecodealliance/wasmtime/issues/new\ + "); + } +} diff --git a/crates/runtime/src/traphandlers/macos.rs b/crates/runtime/src/sys/unix/machports.rs similarity index 98% rename from crates/runtime/src/traphandlers/macos.rs rename to crates/runtime/src/sys/unix/machports.rs index 34d707498e7b..677579676d96 100644 --- a/crates/runtime/src/traphandlers/macos.rs +++ b/crates/runtime/src/sys/unix/machports.rs @@ -33,7 +33,8 @@ #![allow(non_snake_case, clippy::cast_sign_loss)] -use crate::traphandlers::{tls, wasmtime_longjmp}; +use crate::sys::traphandlers::wasmtime_longjmp; +use crate::traphandlers::tls; use mach::exception_types::*; use mach::kern_return::*; use mach::mach_init::*; @@ -167,9 +168,6 @@ static mut WASMTIME_PORT: mach_port_name_t = MACH_PORT_NULL; static mut CHILD_OF_FORKED_PROCESS: bool = false; pub unsafe fn platform_init() { - if cfg!(miri) { - return; - } // Mach ports do not currently work across forks, so configure Wasmtime to // panic in `lazy_per_thread_init` if the child attempts to invoke // WebAssembly. @@ -404,7 +402,7 @@ unsafe fn handle_exception(request: &mut ExceptionRequest) -> bool { // pointer value and if `MAP` changes happen after we read our entry that's // ok since they won't invalidate our entry. let (pc, fp) = get_pc_and_fp(&thread_state); - if !super::IS_WASM_PC(pc as usize) { + if !crate::traphandlers::IS_WASM_PC(pc as usize) { return false; } @@ -443,7 +441,7 @@ unsafe extern "C" fn unwind( None }; state.set_jit_trap(wasm_pc, wasm_fp, faulting_addr); - state.jmp_buf.get() + state.take_jmp_buf() }); debug_assert!(!jmp_buf.is_null()); wasmtime_longjmp(jmp_buf); @@ -468,9 +466,6 @@ unsafe extern "C" fn unwind( /// exception handlers to get registered. #[cold] pub fn lazy_per_thread_init() { - if cfg!(miri) { - return; - } unsafe { assert!( !CHILD_OF_FORKED_PROCESS, diff --git a/crates/runtime/src/sys/unix/macos_traphandlers.rs b/crates/runtime/src/sys/unix/macos_traphandlers.rs new file mode 100644 index 000000000000..fd0d607530fc --- /dev/null +++ b/crates/runtime/src/sys/unix/macos_traphandlers.rs @@ -0,0 +1,28 @@ +/// Whether or not macOS is using mach ports. +#[cfg(target_os = "macos")] +static mut USE_MACH_PORTS: bool = false; + +pub use super::signals::{wasmtime_longjmp, wasmtime_setjmp, SignalHandler}; + +pub unsafe fn platform_init(macos_use_mach_ports: bool) { + USE_MACH_PORTS = macos_use_mach_ports; + if macos_use_mach_ports { + super::machports::platform_init(); + } else { + super::signals::platform_init(false); + } +} + +pub fn using_mach_ports() -> bool { + unsafe { USE_MACH_PORTS } +} + +pub fn lazy_per_thread_init() { + unsafe { + if USE_MACH_PORTS { + super::machports::lazy_per_thread_init(); + } else { + super::signals::lazy_per_thread_init(); + } + } +} diff --git a/crates/runtime/src/mmap/unix.rs b/crates/runtime/src/sys/unix/mmap.rs similarity index 100% rename from crates/runtime/src/mmap/unix.rs rename to crates/runtime/src/sys/unix/mmap.rs diff --git a/crates/runtime/src/sys/unix/mod.rs b/crates/runtime/src/sys/unix/mod.rs new file mode 100644 index 000000000000..f124598acf7d --- /dev/null +++ b/crates/runtime/src/sys/unix/mod.rs @@ -0,0 +1,21 @@ +//! Implementation of Wasmtime's system primitives for Unix-like operating +//! systems. +//! +//! This module handles Linux and macOS for example. + +pub mod mmap; +pub mod unwind; +pub mod vm; + +pub mod signals; + +cfg_if::cfg_if! { + if #[cfg(target_os = "macos")] { + pub mod machports; + + pub mod macos_traphandlers; + pub use macos_traphandlers as traphandlers; + } else { + pub use signals as traphandlers; + } +} diff --git a/crates/runtime/src/traphandlers/unix.rs b/crates/runtime/src/sys/unix/signals.rs similarity index 95% rename from crates/runtime/src/traphandlers/unix.rs rename to crates/runtime/src/sys/unix/signals.rs index c4a993f15068..59b7982d2f57 100644 --- a/crates/runtime/src/traphandlers/unix.rs +++ b/crates/runtime/src/sys/unix/signals.rs @@ -1,11 +1,27 @@ -#![allow(clippy::cast_sign_loss)] // platforms too fiddly to worry about this +//! Trap handling on Unix based on POSIX signals. -use crate::traphandlers::{tls, wasmtime_longjmp}; +use crate::traphandlers::tls; +use crate::VMContext; use std::cell::RefCell; use std::io; use std::mem::{self, MaybeUninit}; use std::ptr::{self, null_mut}; +#[link(name = "wasmtime-helpers")] +extern "C" { + #[wasmtime_versioned_export_macros::versioned_link] + #[allow(improper_ctypes)] + pub fn wasmtime_setjmp( + jmp_buf: *mut *const u8, + callback: extern "C" fn(*mut u8, *mut VMContext), + payload: *mut u8, + callee: *mut VMContext, + ) -> i32; + + #[wasmtime_versioned_export_macros::versioned_link] + pub fn wasmtime_longjmp(jmp_buf: *const u8) -> !; +} + /// Function which may handle custom signals while processing traps. pub type SignalHandler<'a> = dyn Fn(libc::c_int, *const libc::siginfo_t, *const libc::c_void) -> bool + Send + Sync + 'a; @@ -15,10 +31,11 @@ static mut PREV_SIGBUS: MaybeUninit = MaybeUninit::uninit(); static mut PREV_SIGILL: MaybeUninit = MaybeUninit::uninit(); static mut PREV_SIGFPE: MaybeUninit = MaybeUninit::uninit(); -pub unsafe fn platform_init() { - if cfg!(miri) { - return; - } +pub unsafe fn platform_init(macos_use_mach_ports: bool) { + // Either mach ports shouldn't be in use or we shouldn't be on macOS, + // otherwise the `machports.rs` module should be used instead. + assert!(!macos_use_mach_ports || !cfg!(target_os = "macos")); + let register = |slot: &mut MaybeUninit, signal: i32| { let mut handler: libc::sigaction = mem::zeroed(); // The flags here are relatively careful, and they are... @@ -317,10 +334,6 @@ pub fn lazy_per_thread_init() { }); unsafe fn allocate_sigaltstack() -> Option { - if cfg!(miri) { - return None; - } - // Check to see if the existing sigaltstack, if it exists, is big // enough. If so we don't need to allocate our own. let mut old_stack = mem::zeroed(); diff --git a/crates/jit/src/unwind/systemv.rs b/crates/runtime/src/sys/unix/unwind.rs similarity index 97% rename from crates/jit/src/unwind/systemv.rs rename to crates/runtime/src/sys/unix/unwind.rs index 2233e4dcb350..87d0a28d421a 100644 --- a/crates/jit/src/unwind/systemv.rs +++ b/crates/runtime/src/sys/unix/unwind.rs @@ -14,6 +14,7 @@ extern "C" { } impl UnwindRegistration { + #[allow(missing_docs)] pub const SECTION_NAME: &'static str = ".eh_frame"; /// Registers precompiled unwinding information with the system. @@ -28,7 +29,7 @@ impl UnwindRegistration { unwind_len: usize, ) -> Result { debug_assert_eq!( - unwind_info as usize % wasmtime_runtime::page_size(), + unwind_info as usize % crate::page_size(), 0, "The unwind info must always be aligned to a page" ); diff --git a/crates/runtime/src/sys/unix/vm.rs b/crates/runtime/src/sys/unix/vm.rs new file mode 100644 index 000000000000..d5109a75735b --- /dev/null +++ b/crates/runtime/src/sys/unix/vm.rs @@ -0,0 +1,208 @@ +use rustix::fd::AsRawFd; +use rustix::mm::{mmap, mmap_anonymous, mprotect, MapFlags, MprotectFlags, ProtFlags}; +use std::fs::File; +use std::io; +use std::sync::Arc; + +pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + mprotect(ptr.cast(), len, MprotectFlags::READ | MprotectFlags::WRITE)?; + Ok(()) +} + +pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + mprotect(ptr.cast(), len, MprotectFlags::empty())?; + Ok(()) +} + +pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + let ret = mmap_anonymous( + ptr.cast(), + len, + ProtFlags::empty(), + MapFlags::PRIVATE | MapFlags::FIXED, + )?; + assert_eq!(ptr, ret.cast()); + Ok(()) +} + +#[cfg(any(feature = "pooling-allocator", feature = "async"))] +unsafe fn decommit(addr: *mut u8, len: usize) -> io::Result<()> { + if len == 0 { + return Ok(()); + } + + unsafe { + cfg_if::cfg_if! { + if #[cfg(target_os = "linux")] { + use rustix::mm::{madvise, Advice}; + + // On Linux, this is enough to cause the kernel to initialize + // the pages to 0 on next access + madvise(addr as _, len, Advice::LinuxDontNeed)?; + } else { + // By creating a new mapping at the same location, this will + // discard the mapping for the pages in the given range. + // The new mapping will be to the CoW zero page, so this + // effectively zeroes the pages. + mmap_anonymous( + addr as _, + len, + ProtFlags::READ | ProtFlags::WRITE, + MapFlags::PRIVATE | MapFlags::FIXED, + )?; + } + } + } + + Ok(()) +} + +#[cfg(feature = "pooling-allocator")] +pub unsafe fn commit_table_pages(_addr: *mut u8, _len: usize) -> io::Result<()> { + // Table pages are always READ | WRITE so there's nothing that needs to be + // done here. + Ok(()) +} + +#[cfg(feature = "pooling-allocator")] +pub unsafe fn decommit_table_pages(addr: *mut u8, len: usize) -> io::Result<()> { + decommit(addr, len) +} + +#[cfg(feature = "async")] +pub unsafe fn commit_stack_pages(_addr: *mut u8, _len: usize) -> io::Result<()> { + // Like table pages stack pages are always READ | WRITE so nothing extra + // needs to be done to ensure they can be committed. + Ok(()) +} + +#[cfg(feature = "async")] +pub unsafe fn reset_stack_pages_to_zero(addr: *mut u8, len: usize) -> io::Result<()> { + decommit(addr, len) +} + +pub fn get_page_size() -> usize { + unsafe { libc::sysconf(libc::_SC_PAGESIZE).try_into().unwrap() } +} + +pub fn supports_madvise_dontneed() -> bool { + cfg!(target_os = "linux") +} + +pub unsafe fn madvise_dontneed(ptr: *mut u8, len: usize) -> io::Result<()> { + cfg_if::cfg_if! { + if #[cfg(target_os = "linux")] { + rustix::mm::madvise(ptr.cast(), len, rustix::mm::Advice::LinuxDontNeed)?; + Ok(()) + } else { + let _ = (ptr, len); + unreachable!(); + } + } +} + +#[derive(Debug)] +pub enum MemoryImageSource { + Mmap(Arc), + #[cfg(target_os = "linux")] + Memfd(memfd::Memfd), +} + +impl MemoryImageSource { + pub fn from_file(file: &Arc) -> Option { + Some(MemoryImageSource::Mmap(file.clone())) + } + + #[cfg(not(target_os = "linux"))] + pub fn from_data(_data: &[u8]) -> io::Result> { + Ok(None) + } + + #[cfg(target_os = "linux")] + pub fn from_data(data: &[u8]) -> anyhow::Result> { + // On Linux `memfd_create` is used to create an anonymous + // in-memory file to represent the heap image. This anonymous + // file is then used as the basis for further mmaps. + + use std::io::{ErrorKind, Write}; + + // Create the memfd. It needs a name, but the documentation for + // `memfd_create()` says that names can be duplicated with no issues. + let memfd = match memfd::MemfdOptions::new() + .allow_sealing(true) + .create("wasm-memory-image") + { + Ok(memfd) => memfd, + // If this kernel is old enough to not support memfd then attempt to + // gracefully handle that and fall back to skipping the memfd + // optimization. + Err(memfd::Error::Create(err)) if err.kind() == ErrorKind::Unsupported => { + return Ok(None) + } + Err(e) => return Err(e.into()), + }; + memfd.as_file().write_all(data)?; + + // Seal the memfd's data and length. + // + // This is a defense-in-depth security mitigation. The + // memfd will serve as the starting point for the heap of + // every instance of this module. If anything were to + // write to this, it could affect every execution. The + // memfd object itself is owned by the machinery here and + // not exposed elsewhere, but it is still an ambient open + // file descriptor at the syscall level, so some other + // vulnerability that allowed writes to arbitrary fds + // could modify it. Or we could have some issue with the + // way that we map it into each instance. To be + // extra-super-sure that it never changes, and because + // this costs very little, we use the kernel's "seal" API + // to make the memfd image permanently read-only. + memfd.add_seals(&[ + memfd::FileSeal::SealGrow, + memfd::FileSeal::SealShrink, + memfd::FileSeal::SealWrite, + memfd::FileSeal::SealSeal, + ])?; + + Ok(Some(MemoryImageSource::Memfd(memfd))) + } + + fn as_file(&self) -> &File { + match self { + MemoryImageSource::Mmap(file) => file, + #[cfg(target_os = "linux")] + MemoryImageSource::Memfd(memfd) => memfd.as_file(), + } + } + + pub unsafe fn map_at(&self, base: *mut u8, len: usize, offset: u64) -> io::Result<()> { + let ptr = mmap( + base.cast(), + len, + ProtFlags::READ | ProtFlags::WRITE, + MapFlags::PRIVATE | MapFlags::FIXED, + self.as_file(), + offset, + )?; + assert_eq!(base, ptr.cast()); + Ok(()) + } + + pub unsafe fn remap_as_zeros_at(&self, base: *mut u8, len: usize) -> io::Result<()> { + let ptr = mmap_anonymous( + base.cast(), + len, + ProtFlags::READ | ProtFlags::WRITE, + MapFlags::PRIVATE | MapFlags::FIXED, + )?; + assert_eq!(base, ptr.cast()); + Ok(()) + } +} + +impl PartialEq for MemoryImageSource { + fn eq(&self, other: &MemoryImageSource) -> bool { + self.as_file().as_raw_fd() == other.as_file().as_raw_fd() + } +} diff --git a/crates/runtime/src/mmap/windows.rs b/crates/runtime/src/sys/windows/mmap.rs similarity index 100% rename from crates/runtime/src/mmap/windows.rs rename to crates/runtime/src/sys/windows/mmap.rs diff --git a/crates/runtime/src/sys/windows/mod.rs b/crates/runtime/src/sys/windows/mod.rs new file mode 100644 index 000000000000..d055ef6926aa --- /dev/null +++ b/crates/runtime/src/sys/windows/mod.rs @@ -0,0 +1,6 @@ +//! Implementation of Wasmtime's system primitives for Windows. + +pub mod mmap; +pub mod traphandlers; +pub mod unwind; +pub mod vm; diff --git a/crates/runtime/src/traphandlers/windows.rs b/crates/runtime/src/sys/windows/traphandlers.rs similarity index 88% rename from crates/runtime/src/traphandlers/windows.rs rename to crates/runtime/src/sys/windows/traphandlers.rs index 5ad7295f5e3c..3e87c0e7d380 100644 --- a/crates/runtime/src/traphandlers/windows.rs +++ b/crates/runtime/src/sys/windows/traphandlers.rs @@ -1,9 +1,25 @@ -use crate::traphandlers::{tls, wasmtime_longjmp}; +use crate::traphandlers::tls; +use crate::VMContext; use std::io; use windows_sys::Win32::Foundation::*; use windows_sys::Win32::System::Diagnostics::Debug::*; use windows_sys::Win32::System::Kernel::*; +#[link(name = "wasmtime-helpers")] +extern "C" { + #[wasmtime_versioned_export_macros::versioned_link] + #[allow(improper_ctypes)] + pub fn wasmtime_setjmp( + jmp_buf: *mut *const u8, + callback: extern "C" fn(*mut u8, *mut VMContext), + payload: *mut u8, + callee: *mut VMContext, + ) -> i32; + + #[wasmtime_versioned_export_macros::versioned_link] + pub fn wasmtime_longjmp(jmp_buf: *const u8) -> !; +} + /// Function which may handle custom signals while processing traps. pub type SignalHandler<'a> = dyn Fn(*mut EXCEPTION_POINTERS) -> bool + Send + Sync + 'a; diff --git a/crates/jit/src/unwind/winx64.rs b/crates/runtime/src/sys/windows/unwind.rs similarity index 95% rename from crates/jit/src/unwind/winx64.rs rename to crates/runtime/src/sys/windows/unwind.rs index 6468b87c3ec4..30f136180c15 100644 --- a/crates/jit/src/unwind/winx64.rs +++ b/crates/runtime/src/sys/windows/unwind.rs @@ -10,8 +10,10 @@ pub struct UnwindRegistration { } impl UnwindRegistration { + #[allow(missing_docs)] pub const SECTION_NAME: &'static str = ".pdata"; + #[allow(missing_docs)] pub unsafe fn new( base_address: *const u8, unwind_info: *const u8, diff --git a/crates/runtime/src/sys/windows/vm.rs b/crates/runtime/src/sys/windows/vm.rs new file mode 100644 index 000000000000..a8ff8411be19 --- /dev/null +++ b/crates/runtime/src/sys/windows/vm.rs @@ -0,0 +1,73 @@ +use std::fs::File; +use std::io; +use std::mem::MaybeUninit; +use std::sync::Arc; +use windows_sys::Win32::System::Memory::*; +use windows_sys::Win32::System::SystemInformation::*; + +pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + if VirtualAlloc(ptr.cast(), len, MEM_COMMIT, PAGE_READWRITE).is_null() { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + erase_existing_mapping(ptr, len) +} + +pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + if VirtualFree(ptr.cast(), len, MEM_DECOMMIT) == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(feature = "pooling-allocator")] +pub unsafe fn commit_table_pages(addr: *mut u8, len: usize) -> io::Result<()> { + expose_exisiting_mapping(addr, len) +} + +#[cfg(feature = "pooling-allocator")] +pub unsafe fn decommit_table_pages(addr: *mut u8, len: usize) -> io::Result<()> { + erase_existing_mapping(addr, len) +} + +pub fn get_page_size() -> usize { + unsafe { + let mut info = MaybeUninit::uninit(); + GetSystemInfo(info.as_mut_ptr()); + info.assume_init_ref().dwPageSize as usize + } +} + +pub fn supports_madvise_dontneed() -> bool { + false +} + +pub unsafe fn madvise_dontneed(_ptr: *mut u8, _len: usize) -> io::Result<()> { + unreachable!() +} + +#[derive(PartialEq, Debug)] +pub enum MemoryImageSource {} + +impl MemoryImageSource { + pub fn from_file(_file: &Arc) -> Option { + None + } + + pub fn from_data(_data: &[u8]) -> io::Result> { + Ok(None) + } + + pub unsafe fn map_at(&self, _base: *mut u8, _len: usize, _offset: u64) -> io::Result<()> { + match *self {} + } + + pub unsafe fn remap_as_zeros_at(&self, _base: *mut u8, _len: usize) -> io::Result<()> { + match *self {} + } +} diff --git a/crates/runtime/src/trampolines.rs b/crates/runtime/src/trampolines.rs deleted file mode 100644 index 16741075cef5..000000000000 --- a/crates/runtime/src/trampolines.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Wasm-to-libcall trampolines. - -cfg_if::cfg_if! { - if #[cfg(target_arch = "x86_64")] { - #[macro_use] - mod x86_64; - } else if #[cfg(target_arch = "aarch64")] { - #[macro_use] - mod aarch64; - } else if #[cfg(target_arch = "s390x")] { - #[macro_use] - mod s390x; - }else if #[cfg(target_arch = "riscv64")] { - #[macro_use] - mod riscv64; - } else { - compile_error!("unsupported architecture"); - } -} diff --git a/crates/runtime/src/trampolines/aarch64.rs b/crates/runtime/src/trampolines/aarch64.rs deleted file mode 100644 index 29852bad1ae1..000000000000 --- a/crates/runtime/src/trampolines/aarch64.rs +++ /dev/null @@ -1,42 +0,0 @@ -#[rustfmt::skip] -macro_rules! wasm_to_libcall_trampoline { - ($libcall:ident ; $libcall_impl:ident) => { - wasmtime_asm_macros::asm_func!( - wasmtime_versioned_export_macros::versioned_stringify_ident!($libcall), - " - .cfi_startproc - bti c - - // Load the pointer to `VMRuntimeLimits` in `x9`. - ldur x9, [x0, #8] - - // Store the last Wasm FP into the `last_wasm_exit_fp` in the limits. - stur fp, [x9, #24] - - // Store the last Wasm PC into the `last_wasm_exit_pc` in the limits. - stur lr, [x9, #32] - - // Tail call to the actual implementation of this libcall. - b {} - - .cfi_endproc - ", - sym $libcall_impl - ); - }; -} - -#[cfg(test)] -mod wasm_to_libcall_trampoline_offsets_tests { - use wasmtime_environ::{Module, PtrSize, VMOffsets}; - - #[test] - fn test() { - let module = Module::new(); - let offsets = VMOffsets::new(std::mem::size_of::<*mut u8>() as u8, &module); - - assert_eq!(8, offsets.vmctx_runtime_limits()); - assert_eq!(24, offsets.ptr.vmruntime_limits_last_wasm_exit_fp()); - assert_eq!(32, offsets.ptr.vmruntime_limits_last_wasm_exit_pc()); - } -} diff --git a/crates/runtime/src/trampolines/s390x.rs b/crates/runtime/src/trampolines/s390x.rs deleted file mode 100644 index f8ca65ca4fc9..000000000000 --- a/crates/runtime/src/trampolines/s390x.rs +++ /dev/null @@ -1,25 +0,0 @@ -// The implementation for libcall trampolines is in the s390x.S -// file. We provide this dummy definition of wasm_to_libcall_trampoline -// here to make libcalls.rs compile on s390x. Note that this means we -// have to duplicate the list of libcalls used in the assembler file. - -macro_rules! wasm_to_libcall_trampoline { - ($libcall:ident ; $libcall_impl:ident) => {}; -} - -// The wasm_to_host_trampoline implementation is in the s390x.S -// file, but we still want to have this unit test here. -#[cfg(test)] -mod wasm_to_libcall_trampoline_offsets_tests { - use wasmtime_environ::{Module, PtrSize, VMOffsets}; - - #[test] - fn test() { - let module = Module::new(); - let offsets = VMOffsets::new(std::mem::size_of::<*mut u8>() as u8, &module); - - assert_eq!(8, offsets.vmctx_runtime_limits()); - assert_eq!(24, offsets.ptr.vmruntime_limits_last_wasm_exit_fp()); - assert_eq!(32, offsets.ptr.vmruntime_limits_last_wasm_exit_pc()); - } -} diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index 21a7a519a1d1..5238199241c7 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -4,6 +4,7 @@ mod backtrace; mod coredump; +use crate::sys::traphandlers; use crate::{Instance, VMContext, VMRuntimeLimits}; use anyhow::Error; use std::any::Any; @@ -16,85 +17,14 @@ pub use self::backtrace::{Backtrace, Frame}; pub use self::coredump::CoreDumpStack; pub use self::tls::{tls_eager_initialize, AsyncWasmCallState, PreviousAsyncWasmCallState}; -cfg_if::cfg_if! { - if #[cfg(miri)] { - // With MIRI set up just enough of a setjmp/longjmp with catching panics - // to get a few tests working that use this. - // - // Note that no actual JIT code runs in MIRI so this is purely here for - // host-to-host calls. - - struct WasmtimeLongjmp; - - #[wasmtime_versioned_export_macros::versioned_export] - unsafe extern "C" fn wasmtime_setjmp( - _jmp_buf: *mut *const u8, - callback: extern "C" fn(*mut u8, *mut VMContext), - payload: *mut u8, - callee: *mut VMContext, - ) -> i32 { - use std::panic::{self, AssertUnwindSafe}; - let result = panic::catch_unwind(AssertUnwindSafe(|| { - callback(payload, callee); - })); - match result { - Ok(()) => 1, - Err(e) => { - if e.is::() { - 0 - } else { - panic::resume_unwind(e) - } - } - } - } - - #[wasmtime_versioned_export_macros::versioned_export] - unsafe extern "C" fn wasmtime_longjmp(_jmp_buf: *const u8) -> ! { - std::panic::panic_any(WasmtimeLongjmp) - } - } else { - #[link(name = "wasmtime-helpers")] - extern "C" { - #[wasmtime_versioned_export_macros::versioned_link] - #[allow(improper_ctypes)] - fn wasmtime_setjmp( - jmp_buf: *mut *const u8, - callback: extern "C" fn(*mut u8, *mut VMContext), - payload: *mut u8, - callee: *mut VMContext, - ) -> i32; - #[wasmtime_versioned_export_macros::versioned_link] - fn wasmtime_longjmp(jmp_buf: *const u8) -> !; - } - } -} - -cfg_if::cfg_if! { - if #[cfg(unix)] { - mod unix; - use unix as sys; - } else if #[cfg(target_os = "windows")] { - mod windows; - use windows as sys; - } -} - -#[cfg(target_os = "macos")] -mod macos; - -pub use sys::SignalHandler; +pub use traphandlers::SignalHandler; /// Globally-set callback to determine whether a program counter is actually a /// wasm trap. /// /// This is initialized during `init_traps` below. The definition lives within /// `wasmtime` currently. -static mut IS_WASM_PC: fn(usize) -> bool = |_| false; - -/// Whether or not macOS is using mach ports. -#[cfg(target_os = "macos")] -static mut MACOS_USE_MACH_PORTS: bool = false; +pub(crate) static mut IS_WASM_PC: fn(usize) -> bool = |_| false; /// This function is required to be called before any WebAssembly is entered. /// This will configure global state such as signal handlers to prepare the @@ -112,38 +42,21 @@ static mut MACOS_USE_MACH_PORTS: bool = false; pub fn init_traps(is_wasm_pc: fn(usize) -> bool, macos_use_mach_ports: bool) { static INIT: Once = Once::new(); - // only used on macos, so squelch warnings about this not being used on - // other platform. - let _ = macos_use_mach_ports; - INIT.call_once(|| unsafe { IS_WASM_PC = is_wasm_pc; - #[cfg(target_os = "macos")] - if macos_use_mach_ports { - MACOS_USE_MACH_PORTS = macos_use_mach_ports; - return macos::platform_init(); - } - sys::platform_init(); + traphandlers::platform_init(macos_use_mach_ports); }); #[cfg(target_os = "macos")] - unsafe { - assert_eq!( - MACOS_USE_MACH_PORTS, macos_use_mach_ports, - "cannot configure two different methods of signal handling in the same process" - ); - } + assert_eq!( + traphandlers::using_mach_ports(), + macos_use_mach_ports, + "cannot configure two different methods of signal handling in the same process" + ); } fn lazy_per_thread_init() { - #[cfg(target_os = "macos")] - unsafe { - if MACOS_USE_MACH_PORTS { - return macos::lazy_per_thread_init(); - } - } - - sys::lazy_per_thread_init(); + traphandlers::lazy_per_thread_init(); } /// Raises a trap immediately. @@ -303,7 +216,7 @@ where let result = CallThreadState::new(signal_handler, capture_backtrace, capture_coredump, *limits) .with(|cx| { - wasmtime_setjmp( + traphandlers::wasmtime_setjmp( cx.jmp_buf.as_ptr(), call_closure::, &mut closure as *mut F as *mut u8, @@ -470,7 +383,7 @@ impl CallThreadState { (*self.unwind.get()) .as_mut_ptr() .write((reason, backtrace, coredump)); - wasmtime_longjmp(self.jmp_buf.get()); + traphandlers::wasmtime_longjmp(self.jmp_buf.get()); } } @@ -489,8 +402,7 @@ impl CallThreadState { /// instance, and the trap handler should quickly return. /// * a different pointer - a jmp_buf buffer to longjmp to, meaning that /// the wasm trap was succesfully handled. - #[cfg_attr(target_os = "macos", allow(dead_code))] // macOS is more raw and doesn't use this - fn take_jmp_buf_if_trap( + pub(crate) fn take_jmp_buf_if_trap( &self, pc: *const u8, call_handler: impl Fn(&SignalHandler) -> bool, @@ -516,10 +428,14 @@ impl CallThreadState { // If all that passed then this is indeed a wasm trap, so return the // `jmp_buf` passed to `wasmtime_longjmp` to resume. + self.take_jmp_buf() + } + + pub(crate) fn take_jmp_buf(&self) -> *const u8 { self.jmp_buf.replace(ptr::null()) } - fn set_jit_trap(&self, pc: *const u8, fp: usize, faulting_addr: Option) { + pub(crate) fn set_jit_trap(&self, pc: *const u8, fp: usize, faulting_addr: Option) { let backtrace = self.capture_backtrace(self.limits, Some((pc as usize, fp))); let coredump = self.capture_coredump(self.limits, Some((pc as usize, fp))); unsafe { @@ -581,7 +497,7 @@ impl Drop for ResetCell<'_, T> { // happen which requires us to read some contextual state to figure out what to // do with the trap. This `tls` module is used to persist that information from // the caller to the trap site. -mod tls { +pub(crate) mod tls { use super::CallThreadState; use std::mem; use std::ops::Range; diff --git a/crates/runtime/src/traphandlers/backtrace.rs b/crates/runtime/src/traphandlers/backtrace.rs index 0b61fb425f5d..4cab189dcfc4 100644 --- a/crates/runtime/src/traphandlers/backtrace.rs +++ b/crates/runtime/src/traphandlers/backtrace.rs @@ -20,39 +20,13 @@ //! exit FP and stopping once we reach the entry SP (meaning that the next older //! frame is a host frame). +use crate::arch; use crate::{ traphandlers::{tls, CallThreadState}, VMRuntimeLimits, }; -use cfg_if::cfg_if; use std::ops::ControlFlow; -// Architecture-specific bits for stack walking. Each of these modules should -// define and export the following functions: -// -// * `unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize` -// * `unsafe fn get_next_older_fp_from_fp(fp: usize) -> usize` -// * `fn reached_entry_sp(fp: usize, first_wasm_sp: usize) -> bool` -// * `fn assert_entry_sp_is_aligned(sp: usize)` -// * `fn assert_fp_is_aligned(fp: usize)` -cfg_if! { - if #[cfg(target_arch = "x86_64")] { - mod x86_64; - use x86_64 as arch; - } else if #[cfg(target_arch = "aarch64")] { - mod aarch64; - use aarch64 as arch; - } else if #[cfg(target_arch = "s390x")] { - mod s390x; - use s390x as arch; - } else if #[cfg(target_arch = "riscv64")] { - mod riscv64; - use riscv64 as arch; - } else { - compile_error!("unsupported architecture"); - } -} - /// A WebAssembly stack trace. #[derive(Debug)] pub struct Backtrace(Vec); diff --git a/crates/runtime/src/traphandlers/backtrace/riscv64.rs b/crates/runtime/src/traphandlers/backtrace/riscv64.rs deleted file mode 100644 index bc3f5b0840d3..000000000000 --- a/crates/runtime/src/traphandlers/backtrace/riscv64.rs +++ /dev/null @@ -1,18 +0,0 @@ -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - *(fp as *mut usize).offset(1) -} - -// And the current frame pointer points to the next older frame pointer. -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { - fp >= entry_sp -} - -pub fn assert_entry_sp_is_aligned(sp: usize) { - assert_eq!(sp % 16, 0, "stack should always be aligned to 16"); -} - -pub fn assert_fp_is_aligned(fp: usize) { - assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); -} diff --git a/crates/runtime/src/traphandlers/backtrace/s390x.rs b/crates/runtime/src/traphandlers/backtrace/s390x.rs deleted file mode 100644 index 7290c4850c9a..000000000000 --- a/crates/runtime/src/traphandlers/backtrace/s390x.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - // The next older PC can be found in register %r14 at function entry, which - // was saved into slot 14 of the register save area pointed to by "FP" (the - // backchain pointer). - *(fp as *mut usize).offset(14) -} - -// The next older "FP" (backchain pointer) was saved in the slot pointed to -// by the current "FP". -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { - fp > entry_sp -} - -pub fn assert_entry_sp_is_aligned(sp: usize) { - assert_eq!(sp % 8, 0, "stack should always be aligned to 8"); -} - -pub fn assert_fp_is_aligned(fp: usize) { - assert_eq!(fp % 8, 0, "stack should always be aligned to 8"); -} diff --git a/crates/runtime/src/traphandlers/backtrace/x86_64.rs b/crates/runtime/src/traphandlers/backtrace/x86_64.rs deleted file mode 100644 index ed1a3eb4b14b..000000000000 --- a/crates/runtime/src/traphandlers/backtrace/x86_64.rs +++ /dev/null @@ -1,20 +0,0 @@ -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - // The calling convention always pushes the return pointer (aka the PC of - // the next older frame) just before this frame. - *(fp as *mut usize).offset(1) -} - -// And the current frame pointer points to the next older frame pointer. -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn reached_entry_sp(fp: usize, entry_sp: usize) -> bool { - fp >= entry_sp -} - -pub fn assert_entry_sp_is_aligned(sp: usize) { - assert_eq!(sp % 16, 0, "stack should always be aligned to 16"); -} - -pub fn assert_fp_is_aligned(fp: usize) { - assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); -} diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index a91a1c096ced..ee24eb9a4dbd 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -58,9 +58,6 @@ features = [ "Win32_System_Diagnostics_Debug", ] -[target.'cfg(target_arch = "s390x")'.dependencies] -psm = "0.1.11" - [dev-dependencies] tempfile = "3.0" wasmtime-wasi = { path = "../wasi", default-features = true } diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 3a23d51d4da7..8bb6cb2889b4 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -1416,7 +1416,7 @@ fn enter_wasm(store: &mut StoreContextMut<'_, T>) -> Option { return None; } - let stack_pointer = stack_pointer(); + let stack_pointer = wasmtime_runtime::get_stack_pointer(); // Determine the stack pointer where, after which, any wasm code will // immediately trap. This is checked on the entry to all wasm functions. @@ -1443,43 +1443,6 @@ fn enter_wasm(store: &mut StoreContextMut<'_, T>) -> Option { Some(prev_stack) } -#[inline] -fn stack_pointer() -> usize { - let stack_pointer: usize; - cfg_if::cfg_if! { - if #[cfg(target_arch = "x86_64")] { - unsafe { - std::arch::asm!( - "mov {}, rsp", - out(reg) stack_pointer, - options(nostack,nomem), - ); - } - } else if #[cfg(target_arch = "aarch64")] { - unsafe { - std::arch::asm!( - "mov {}, sp", - out(reg) stack_pointer, - options(nostack,nomem), - ); - } - } else if #[cfg(target_arch = "riscv64")] { - unsafe { - std::arch::asm!( - "mv {}, sp", - out(reg) stack_pointer, - options(nostack,nomem), - ); - } - } else if #[cfg(target_arch = "s390x")] { - stack_pointer = psm::stack_pointer() as usize; - } else { - compile_error!("unsupported platform"); - } - } - stack_pointer -} - fn exit_wasm(store: &mut StoreContextMut<'_, T>, prev_stack: Option) { // If we don't have a previous stack pointer to restore, then there's no // cleanup we need to perform here. diff --git a/crates/wasmtime/src/v128.rs b/crates/wasmtime/src/v128.rs index 25b48b1e368c..595fe8061aa2 100644 --- a/crates/wasmtime/src/v128.rs +++ b/crates/wasmtime/src/v128.rs @@ -7,6 +7,7 @@ use crate::store::StoreOpaque; use crate::{ValRaw, ValType, WasmTy}; use std::cmp::Ordering; use std::fmt; +use wasmtime_runtime::V128Abi; /// Representation of a 128-bit vector type, `v128`, for WebAssembly. /// @@ -25,40 +26,10 @@ use std::fmt; /// [`TypedFunc`]: crate::TypedFunc #[derive(Copy, Clone)] #[repr(transparent)] -pub struct V128(Abi); - -// NB: this is why this type is only suitable with `TypedFunc` on some platforms -// and no other. See the documentation for each platform for why each ABI is -// chosen. -cfg_if::cfg_if! { - if #[cfg(target_arch = "x86_64")] { - // x86 vectors are represented with XMM registers which are represented - // with the `__m128i` type. This type is considered a vector type for - // ABI purposes which is implemented by Cranelift. - type Abi = std::arch::x86_64::__m128i; - } else if #[cfg(target_arch = "aarch64")] { - // AArch64 uses vector registered which here is used with a vector type. - // Note that the specific type shouldn't matter too much but the choice - // of using a vector is the significant part. - type Abi = std::arch::aarch64::uint8x16_t; - } else if #[cfg(target_arch = "riscv64")] { - // RISC-V currently always passes all vector arguments indirectly in the - // ABI. Currently Rust has no stable means of representing this meaning - // that a 128-bit representation is chosen here but it can't be passed - // directly to WebAssembly, for example, and must instead be passed - // through an array-call trampoline. - type Abi = u128; - } else if #[cfg(target_arch = "s390x")] { - // Currently Rust has no stable means of representing vector registers - // so like RISC-V at this time this uses a bland 128-bit representation. - type Abi = u128; - } else { - compile_error!("unsupported platform"); - } -} +pub struct V128(V128Abi); union Reinterpret { - abi: Abi, + abi: V128Abi, u128: u128, } @@ -112,7 +83,7 @@ impl Ord for V128 { // the documentation above in the `cfg_if!` for why this is conditional. #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] unsafe impl WasmTy for V128 { - type Abi = Abi; + type Abi = V128Abi; #[inline] fn valtype() -> ValType { From 3e4cdde716aff0e002d126ab67ec77fd22140613 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sat, 2 Dec 2023 22:37:03 -0800 Subject: [PATCH 04/15] Fix build on Windows prtest:full --- crates/runtime/src/sys/windows/traphandlers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/runtime/src/sys/windows/traphandlers.rs b/crates/runtime/src/sys/windows/traphandlers.rs index 3e87c0e7d380..f383477ff59c 100644 --- a/crates/runtime/src/sys/windows/traphandlers.rs +++ b/crates/runtime/src/sys/windows/traphandlers.rs @@ -23,7 +23,7 @@ extern "C" { /// Function which may handle custom signals while processing traps. pub type SignalHandler<'a> = dyn Fn(*mut EXCEPTION_POINTERS) -> bool + Send + Sync + 'a; -pub unsafe fn platform_init() { +pub unsafe fn platform_init(_macos_use_mach_ports: bool) { // our trap handler needs to go first, so that we can recover from // wasm faults and continue execution, so pass `1` as a true value // here. From 16033012328baacdda097ad0b801f8ce88a0fd75 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sat, 2 Dec 2023 22:43:16 -0800 Subject: [PATCH 05/15] Fix some build warnings --- crates/runtime/src/sys/unix/vm.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/runtime/src/sys/unix/vm.rs b/crates/runtime/src/sys/unix/vm.rs index d5109a75735b..ec84b4694e8c 100644 --- a/crates/runtime/src/sys/unix/vm.rs +++ b/crates/runtime/src/sys/unix/vm.rs @@ -25,7 +25,7 @@ pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> Ok(()) } -#[cfg(any(feature = "pooling-allocator", feature = "async"))] +#[cfg(feature = "pooling-allocator")] unsafe fn decommit(addr: *mut u8, len: usize) -> io::Result<()> { if len == 0 { return Ok(()); @@ -69,14 +69,14 @@ pub unsafe fn decommit_table_pages(addr: *mut u8, len: usize) -> io::Result<()> decommit(addr, len) } -#[cfg(feature = "async")] +#[cfg(all(feature = "pooling-allocator", feature = "async"))] pub unsafe fn commit_stack_pages(_addr: *mut u8, _len: usize) -> io::Result<()> { // Like table pages stack pages are always READ | WRITE so nothing extra // needs to be done to ensure they can be committed. Ok(()) } -#[cfg(feature = "async")] +#[cfg(all(feature = "pooling-allocator", feature = "async"))] pub unsafe fn reset_stack_pages_to_zero(addr: *mut u8, len: usize) -> io::Result<()> { decommit(addr, len) } From c6e1a5e8f1cd3e652d01599953c2e9fdd4bf6ae4 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sat, 2 Dec 2023 22:43:48 -0800 Subject: [PATCH 06/15] Fix miri build --- crates/runtime/src/sys/miri/traphandlers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/runtime/src/sys/miri/traphandlers.rs b/crates/runtime/src/sys/miri/traphandlers.rs index 9bb6358c48cd..c47bb26747d1 100644 --- a/crates/runtime/src/sys/miri/traphandlers.rs +++ b/crates/runtime/src/sys/miri/traphandlers.rs @@ -37,6 +37,6 @@ pub fn wasmtime_longjmp(_jmp_buf: *const u8) -> ! { #[allow(missing_docs)] pub type SignalHandler<'a> = dyn Fn() + Send + Sync + 'a; -pub fn platform_init() {} +pub fn platform_init(_macos_use_mach_ports: bool) {} pub fn lazy_per_thread_init() {} From f791df698bf83ead11af238df7313b54c0c00a67 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sun, 3 Dec 2023 09:31:54 -0800 Subject: [PATCH 07/15] Include debug-builtins when testing --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 071809bdfd63..fd9e77dbaf37 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ rustix = { workspace = true, features = ["mm", "param"] } [dev-dependencies] # depend again on wasmtime to activate its default features for tests -wasmtime = { workspace = true, features = ['component-model', 'async', 'default', 'winch'] } +wasmtime = { workspace = true, features = ['component-model', 'async', 'default', 'winch', 'debug-builtins'] } env_logger = { workspace = true } log = { workspace = true } filecheck = { workspace = true } From f4b2695a34b838b3ff4af63c062785aa4c8a8296 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sun, 3 Dec 2023 09:38:08 -0800 Subject: [PATCH 08/15] Try to fix Windows tests --- crates/runtime/src/sys/windows/vm.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/runtime/src/sys/windows/vm.rs b/crates/runtime/src/sys/windows/vm.rs index a8ff8411be19..d063cdd95182 100644 --- a/crates/runtime/src/sys/windows/vm.rs +++ b/crates/runtime/src/sys/windows/vm.rs @@ -6,6 +6,9 @@ use windows_sys::Win32::System::Memory::*; use windows_sys::Win32::System::SystemInformation::*; pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + if len == 0 { + return Ok(()); + } if VirtualAlloc(ptr.cast(), len, MEM_COMMIT, PAGE_READWRITE).is_null() { Err(std::io::Error::last_os_error()) } else { @@ -18,6 +21,9 @@ pub unsafe fn hide_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> } pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { + if len == 0 { + return Ok(()); + } if VirtualFree(ptr.cast(), len, MEM_DECOMMIT) == 0 { Err(std::io::Error::last_os_error()) } else { From 10a88b44cdff5aa2f2a6d708cedc1037dcd59a5c Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sun, 3 Dec 2023 09:40:12 -0800 Subject: [PATCH 09/15] Fix warnings on miri --- crates/runtime/src/traphandlers.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index 5238199241c7..3a0eeed68bc5 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -402,6 +402,7 @@ impl CallThreadState { /// instance, and the trap handler should quickly return. /// * a different pointer - a jmp_buf buffer to longjmp to, meaning that /// the wasm trap was succesfully handled. + #[cfg_attr(miri, allow(dead_code))] // miri doesn't handle traps yet pub(crate) fn take_jmp_buf_if_trap( &self, pc: *const u8, @@ -435,6 +436,7 @@ impl CallThreadState { self.jmp_buf.replace(ptr::null()) } + #[cfg_attr(miri, allow(dead_code))] // miri doesn't handle traps yet pub(crate) fn set_jit_trap(&self, pc: *const u8, fp: usize, faulting_addr: Option) { let backtrace = self.capture_backtrace(self.limits, Some((pc as usize, fp))); let coredump = self.capture_coredump(self.limits, Some((pc as usize, fp))); From f34a2d316c6f5e3d010c6c1d004b7317f20759a5 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sun, 3 Dec 2023 10:31:42 -0800 Subject: [PATCH 10/15] Fix miri build --- crates/wasmtime/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs index da6a6c3a06d4..152475f5d5ab 100644 --- a/crates/wasmtime/src/lib.rs +++ b/crates/wasmtime/src/lib.rs @@ -476,7 +476,9 @@ pub use anyhow::{Error, Result}; pub mod component; cfg_if::cfg_if! { - if #[cfg(unix)] { + if #[cfg(miri)] { + // no extensions on miri + } else if #[cfg(unix)] { pub mod unix; } else if #[cfg(windows)] { pub mod windows; From 0886dec010f8b5ba8c7491f103f73f85fa4bfeea Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sun, 3 Dec 2023 10:34:22 -0800 Subject: [PATCH 11/15] Fix debug-builtins feature --- crates/jit/Cargo.toml | 8 +++----- crates/wasmtime/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index ee6f30112bdd..6e36be36a4fe 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -15,9 +15,7 @@ workspace = true [dependencies] wasmtime-environ = { workspace = true } -wasmtime-jit-debug = { workspace = true, features = [ - "perf_jitdump", -], optional = true } +wasmtime-jit-debug = { workspace = true, features = ["perf_jitdump"], optional = true } wasmtime-runtime = { workspace = true } target-lexicon = { workspace = true } anyhow = { workspace = true } @@ -44,6 +42,6 @@ features = ["Win32_System_Diagnostics_Debug"] ittapi = { version = "0.4.0", optional = true } [features] -profiling = ['dep:wasmtime-jit-debug', 'dep:ittapi'] +profiling = ['dep:ittapi'] demangle = ['dep:rustc-demangle', 'dep:cpp_demangle'] -debug-builtins = [] +debug-builtins = ['dep:wasmtime-jit-debug', 'wasmtime-runtime/debug-builtins'] diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index ee24eb9a4dbd..f2a2e1c9dd78 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -158,4 +158,4 @@ addr2line = ["wasmtime-jit/addr2line"] # Export some symbols from the final binary to assist in debugging # Cranelift-generated code with native debuggers like GDB and LLDB. -debug-builtins = ["wasmtime-runtime/debug-builtins"] +debug-builtins = ["wasmtime-runtime/debug-builtins", "wasmtime-jit/debug-builtins"] From e5138672b5c95f66128513fbaffda6cc2f357ae1 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Sun, 3 Dec 2023 10:39:18 -0800 Subject: [PATCH 12/15] More feature fixes --- crates/jit/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index 6e36be36a4fe..c8412d1817d9 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -42,6 +42,6 @@ features = ["Win32_System_Diagnostics_Debug"] ittapi = { version = "0.4.0", optional = true } [features] -profiling = ['dep:ittapi'] +profiling = ['dep:ittapi', 'dep:wasmtime-jit-debug'] demangle = ['dep:rustc-demangle', 'dep:cpp_demangle'] -debug-builtins = ['dep:wasmtime-jit-debug', 'wasmtime-runtime/debug-builtins'] +debug-builtins = ['wasmtime-runtime/debug-builtins'] From 4e6c731cc740b24a84d13c4e70066da9a861d4c6 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 4 Dec 2023 10:34:02 -0800 Subject: [PATCH 13/15] Rename `fd` field --- crates/runtime/src/cow.rs | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/runtime/src/cow.rs b/crates/runtime/src/cow.rs index 0fe6a8416c75..faf295d8c121 100644 --- a/crates/runtime/src/cow.rs +++ b/crates/runtime/src/cow.rs @@ -30,12 +30,12 @@ impl ModuleMemoryImages { /// One backing image for one memory. #[derive(Debug, PartialEq)] pub struct MemoryImage { - /// The file descriptor source of this image. + /// The platform-specific source of this image. /// - /// This might be an mmaped `*.cwasm` file or on Linux it could also be a - /// `Memfd` as an anonymous file in memory. In either case this is used as - /// the backing-source for the CoW image. - fd: MemoryImageSource, + /// This might be a mapped `*.cwasm` file or on Unix it could also be a + /// `Memfd` as an anonymous file in memory on Linux. In either case this is + /// used as the backing-source for the CoW image. + source: MemoryImageSource, /// Length of image, in bytes. /// @@ -45,13 +45,13 @@ pub struct MemoryImage { /// Must be a multiple of the system page size. len: usize, - /// Image starts this many bytes into `fd` source. + /// Image starts this many bytes into `source`. /// - /// This is 0 for anonymous-backed memfd files and is the offset of the data - /// section in a `*.cwasm` file for `*.cwasm`-backed images. + /// This is 0 for anonymous-backed memfd files and is the offset of the + /// data section in a `*.cwasm` file for `*.cwasm`-backed images. /// /// Must be a multiple of the system page size. - fd_offset: u64, + source_offset: u64, /// Image starts this many bytes into heap space. /// @@ -101,10 +101,10 @@ impl MemoryImage { assert_eq!((mmap.original_offset() as u32) % page_size, 0); if let Some(file) = mmap.original_file() { - if let Some(fd) = MemoryImageSource::from_file(file) { + if let Some(source) = MemoryImageSource::from_file(file) { return Ok(Some(MemoryImage { - fd, - fd_offset: u64::try_from(mmap.original_offset() + (data_start - start)) + source, + source_offset: u64::try_from(mmap.original_offset() + (data_start - start)) .unwrap(), linear_memory_offset, len, @@ -115,10 +115,10 @@ impl MemoryImage { // If `mmap` doesn't come from a file then platform-specific mechanisms // may be used to place the data in a form that's amenable to an mmap. - if let Some(fd) = MemoryImageSource::from_data(data)? { + if let Some(source) = MemoryImageSource::from_data(data)? { return Ok(Some(MemoryImage { - fd, - fd_offset: 0, + source, + source_offset: 0, linear_memory_offset, len, })); @@ -128,16 +128,16 @@ impl MemoryImage { } unsafe fn map_at(&self, base: *mut u8) -> Result<()> { - self.fd.map_at( + self.source.map_at( base.add(self.linear_memory_offset), self.len, - self.fd_offset, + self.source_offset, )?; Ok(()) } unsafe fn remap_as_zeros_at(&self, base: *mut u8) -> Result<()> { - self.fd + self.source .remap_as_zeros_at(base.add(self.linear_memory_offset), self.len)?; Ok(()) } @@ -695,9 +695,9 @@ mod test { let image_len = (data.len() + page_size - 1) & !(page_size - 1); Ok(MemoryImage { - fd: MemoryImageSource::from_data(data)?.unwrap(), + source: MemoryImageSource::from_data(data)?.unwrap(), len: image_len, - fd_offset: 0, + source_offset: 0, linear_memory_offset: offset, }) } From 9be4d339903f87779db50a24605f605433b98385 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 4 Dec 2023 10:41:29 -0800 Subject: [PATCH 14/15] Fix speeling --- crates/runtime/src/cow.rs | 2 +- crates/runtime/src/sys/miri/vm.rs | 2 +- crates/runtime/src/sys/unix/vm.rs | 2 +- crates/runtime/src/sys/windows/vm.rs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/runtime/src/cow.rs b/crates/runtime/src/cow.rs index faf295d8c121..75738be7f35c 100644 --- a/crates/runtime/src/cow.rs +++ b/crates/runtime/src/cow.rs @@ -600,7 +600,7 @@ impl MemoryImageSlot { unsafe { let start = self.base.as_ptr().add(range.start); if readwrite { - vm::expose_exisiting_mapping(start, range.len())?; + vm::expose_existing_mapping(start, range.len())?; } else { vm::hide_existing_mapping(start, range.len())?; } diff --git a/crates/runtime/src/sys/miri/vm.rs b/crates/runtime/src/sys/miri/vm.rs index 7b65a74b0976..a4bf0de57e4b 100644 --- a/crates/runtime/src/sys/miri/vm.rs +++ b/crates/runtime/src/sys/miri/vm.rs @@ -2,7 +2,7 @@ use std::fs::File; use std::io; use std::sync::Arc; -pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { +pub unsafe fn expose_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { std::ptr::write_bytes(ptr, 0u8, len); Ok(()) } diff --git a/crates/runtime/src/sys/unix/vm.rs b/crates/runtime/src/sys/unix/vm.rs index ec84b4694e8c..d0e76a14567f 100644 --- a/crates/runtime/src/sys/unix/vm.rs +++ b/crates/runtime/src/sys/unix/vm.rs @@ -4,7 +4,7 @@ use std::fs::File; use std::io; use std::sync::Arc; -pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { +pub unsafe fn expose_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { mprotect(ptr.cast(), len, MprotectFlags::READ | MprotectFlags::WRITE)?; Ok(()) } diff --git a/crates/runtime/src/sys/windows/vm.rs b/crates/runtime/src/sys/windows/vm.rs index d063cdd95182..52501f5cf4db 100644 --- a/crates/runtime/src/sys/windows/vm.rs +++ b/crates/runtime/src/sys/windows/vm.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use windows_sys::Win32::System::Memory::*; use windows_sys::Win32::System::SystemInformation::*; -pub unsafe fn expose_exisiting_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { +pub unsafe fn expose_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> { if len == 0 { return Ok(()); } @@ -33,7 +33,7 @@ pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()> #[cfg(feature = "pooling-allocator")] pub unsafe fn commit_table_pages(addr: *mut u8, len: usize) -> io::Result<()> { - expose_exisiting_mapping(addr, len) + expose_existing_mapping(addr, len) } #[cfg(feature = "pooling-allocator")] From 37c89ea17dbf48ef2449f6aaf2eb2395ae3d1b89 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 4 Dec 2023 10:42:16 -0800 Subject: [PATCH 15/15] Review comments --- crates/runtime/src/sys/miri/traphandlers.rs | 2 +- crates/runtime/src/sys/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/runtime/src/sys/miri/traphandlers.rs b/crates/runtime/src/sys/miri/traphandlers.rs index c47bb26747d1..ab600913ebb9 100644 --- a/crates/runtime/src/sys/miri/traphandlers.rs +++ b/crates/runtime/src/sys/miri/traphandlers.rs @@ -1,4 +1,4 @@ -// With MIRI set up just enough of a setjmp/longjmp with catching panics +// For MIRI, set up just enough of a setjmp/longjmp with catching panics // to get a few tests working that use this. // // Note that no actual JIT code runs in MIRI so this is purely here for diff --git a/crates/runtime/src/sys/mod.rs b/crates/runtime/src/sys/mod.rs index 23815f444a65..e88435682f01 100644 --- a/crates/runtime/src/sys/mod.rs +++ b/crates/runtime/src/sys/mod.rs @@ -1,7 +1,7 @@ //! OS-related abstractions required by Wasmtime. //! //! This module is intended to house all logic that's specific to either Unix -//! for Windows, for example. The goal of this module is to be the "single +//! or Windows, for example. The goal of this module is to be the "single //! module" to edit if Wasmtime is ported to a new platform. Ideally all that's //! needed is an extra block below and a new platform should be good to go after //! filling out the implementation.