From 1573c889d3dce2e637dbe2e09bc14365f9aeaeee Mon Sep 17 00:00:00 2001 From: Chris Fallin Date: Thu, 27 Jan 2022 11:23:20 -0800 Subject: [PATCH] Implement lazy VMCallerCheckedAnyfunc initialization. Currently, in the instance initialization path, we build an "anyfunc" for every imported function and every function in the module. These anyfuncs are used as function references in tables, both within a module and across modules (via function imports). Building all of these is quite wasteful if we never use most of them. Instead, it would be better to build only the ones that we use. This commit implements a technique to lazily initialize them: at the point that a pointer to an anyfunc is taken, we check whether it has actually been initialized. If it hasn't, we do so. We track whether an anyfunc has been initialized with a separate bitmap, and we only need to zero this bitmap at instantiation time. (This is important for performance too: even just zeroing a large array of anyfunc structs can be expensive, at the microsecond scale, for a module with thousands of functions.) Keeping the information around that is needed for this lazy init required a bit of refactoring in the InstanceAllocationRequest; a few fields now live in an `Arc` held by the instance while it exists. --- crates/cranelift/src/func_environ.rs | 14 ++- crates/environ/src/builtin.rs | 2 + crates/environ/src/vmoffsets.rs | 23 +++- crates/jit/src/instantiate.rs | 6 +- crates/runtime/src/instance.rs | 102 +++++++++++++++++- crates/runtime/src/instance/allocator.rs | 94 ++++++++-------- .../runtime/src/instance/allocator/pooling.rs | 3 + crates/runtime/src/lib.rs | 4 +- crates/runtime/src/libcalls.rs | 13 ++- crates/runtime/src/vmcontext.rs | 3 +- crates/wasmtime/src/func.rs | 4 +- crates/wasmtime/src/func/typed.rs | 8 +- crates/wasmtime/src/instance.rs | 9 +- crates/wasmtime/src/module.rs | 28 +++++ crates/wasmtime/src/module/registry.rs | 2 +- crates/wasmtime/src/store.rs | 17 +-- crates/wasmtime/src/trampoline.rs | 14 +-- crates/wasmtime/src/trampoline/func.rs | 11 +- crates/wasmtime/src/trampoline/global.rs | 4 +- 19 files changed, 267 insertions(+), 94 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 71bcf215de2a..277d30213984 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1248,10 +1248,16 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m mut pos: cranelift_codegen::cursor::FuncCursor<'_>, func_index: FuncIndex, ) -> WasmResult { - let vmctx = self.vmctx(&mut pos.func); - let vmctx = pos.ins().global_value(self.pointer_type(), vmctx); - let offset = self.offsets.vmctx_anyfunc(func_index); - Ok(pos.ins().iadd_imm(vmctx, i64::from(offset))) + let func_index = pos.ins().iconst(I32, func_index.as_u32() as i64); + let builtin_index = BuiltinFunctionIndex::ref_func(); + let builtin_sig = self.builtin_function_signatures.ref_func(&mut pos.func); + let (vmctx, builtin_addr) = + self.translate_load_builtin_function_address(&mut pos, builtin_index); + + let call_inst = pos + .ins() + .call_indirect(builtin_sig, builtin_addr, &[vmctx, func_index]); + Ok(pos.func.dfg.first_result(call_inst)) } fn translate_custom_global_get( diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index e660ffc2d7cd..46491a3be3b9 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -18,6 +18,8 @@ macro_rules! foreach_builtin_function { memory_fill(vmctx, i32, i64, i32, i64) -> (); /// Returns an index for wasm's `memory.init` instruction. memory_init(vmctx, i32, i32, i64, i32, i32) -> (); + /// Returns a value for wasm's `ref.func` instruction. + ref_func(vmctx, i32) -> (pointer); /// Returns an index for wasm's `data.drop` instruction. data_drop(vmctx, i32) -> (); /// Returns an index for Wasm's `table.grow` instruction for `funcref`s. diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index a2f0b9d558f4..069e5ecb3950 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -85,6 +85,8 @@ pub struct VMOffsets

{ defined_memories: u32, defined_globals: u32, defined_anyfuncs: u32, + anyfuncs_init: u32, + anyfuncs_init_u64_words: u32, builtin_functions: u32, size: u32, } @@ -187,6 +189,8 @@ impl From> for VMOffsets

{ defined_memories: 0, defined_globals: 0, defined_anyfuncs: 0, + anyfuncs_init: 0, + anyfuncs_init_u64_words: 0, builtin_functions: 0, size: 0, }; @@ -275,7 +279,7 @@ impl From> for VMOffsets

{ .unwrap(), ) .unwrap(); - ret.builtin_functions = ret + ret.anyfuncs_init = ret .defined_anyfuncs .checked_add( ret.num_imported_functions @@ -285,6 +289,11 @@ impl From> for VMOffsets

{ .unwrap(), ) .unwrap(); + ret.anyfuncs_init_u64_words = (ret.num_defined_functions + 63) / 64; + ret.builtin_functions = ret + .anyfuncs_init + .checked_add(ret.anyfuncs_init_u64_words.checked_mul(8).unwrap()) + .unwrap(); ret.size = ret .builtin_functions .checked_add( @@ -589,6 +598,18 @@ impl VMOffsets

{ self.defined_globals } + /// The offset of the `anyfuncs_init` bitset. + #[inline] + pub fn vmctx_anyfuncs_init_begin(&self) -> u32 { + self.anyfuncs_init + } + + /// The length of the `anyfuncs_init` bitset in bytes. + #[inline] + pub fn vmctx_anyfuncs_init_len(&self) -> u32 { + self.anyfuncs_init_u64_words * 8 + } + /// The offset of the `anyfuncs` array. #[inline] pub fn vmctx_anyfuncs_begin(&self) -> u32 { diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index cc6a3844d187..9119126b7692 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -242,7 +242,7 @@ pub struct CompiledModule { address_map_data: Range, trap_data: Range, module: Arc, - funcs: PrimaryMap, + funcs: Arc>, trampolines: Vec, meta: Metadata, code: Range, @@ -298,7 +298,7 @@ impl CompiledModule { let mut ret = Self { module: Arc::new(info.module), - funcs: info.funcs, + funcs: Arc::new(info.funcs), trampolines: info.trampolines, wasm_data: subslice_range(section(ELF_WASM_DATA)?, code.mmap), address_map_data: code @@ -374,7 +374,7 @@ impl CompiledModule { } /// Returns the `FunctionInfo` map for all defined functions. - pub fn functions(&self) -> &PrimaryMap { + pub fn functions(&self) -> &Arc> { &self.funcs } diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 2c9487d75ec9..8c1fc4078d88 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -54,6 +54,9 @@ pub(crate) struct Instance { /// The `Module` this `Instance` was instantiated from. module: Arc, + /// The instantiation info needed for deferred initialization. + info: Arc, + /// Offsets in the `vmctx` region, precomputed from the `module` above. offsets: VMOffsets, @@ -433,6 +436,48 @@ impl Instance { Layout::from_size_align(size, align).unwrap() } + /// Construct a new VMCallerCheckedAnyfunc for the given function + /// (imported or defined in this module). Used during lazy + /// initialization the first time the VMCallerCheckedAnyfunc in + /// the VMContext is referenced; written into place in a careful + /// atomic way by `get_caller_checked_anyfunc()` below. + fn construct_anyfunc(&self, index: FuncIndex) -> VMCallerCheckedAnyfunc { + let sig = self.module.functions[index]; + let type_index = self.info.shared_signatures.lookup(sig); + + let (func_ptr, vmctx) = if let Some(def_index) = self.module.defined_func_index(index) { + ( + (self.info.image_base + self.info.functions[def_index].start as usize) as *mut _, + self.vmctx_ptr(), + ) + } else { + let import = self.imported_function(index); + (import.body.as_ptr(), import.vmctx) + }; + + VMCallerCheckedAnyfunc { + func_ptr, + type_index, + vmctx, + } + } + + /// Get a word of the bitmap of anyfunc-initialized bits, and the + /// bitmask for the particular bit. + pub(crate) fn get_anyfunc_bitmap_word(&self, index: FuncIndex) -> (&mut u64, u64) { + let word_index = index.as_u32() / 64; + let bit_index = index.as_u32() % 64; + let word = unsafe { + self.vmctx_plus_offset::( + self.offsets.vmctx_anyfuncs_init_begin() + (word_index * 8), + ) + .as_mut() + .unwrap() + }; + let mask = 1u64 << (bit_index as u64); + (word, mask) + } + /// Get a `&VMCallerCheckedAnyfunc` for the given `FuncIndex`. /// /// Returns `None` if the index is the reserved index value. @@ -447,7 +492,61 @@ impl Instance { return None; } - unsafe { Some(&*self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index))) } + unsafe { + let anyfunc = + self.vmctx_plus_offset::(self.offsets.vmctx_anyfunc(index)); + let (bitmap_word, bitmask) = self.get_anyfunc_bitmap_word(index); + if *bitmap_word & bitmask == 0 { + // N.B.: both the anyfunc write and the bitmap-word + // write are non-atomic, yet this is inside a `&self` + // method. How does that work? In fact it is a benign + // race. In the most common case, an anyfunc is + // queried either as part of table init or as part of + // executing a function, both of which have a `&mut + // Store` so are actually exclusive. But if + // e.g. export-lookups happen in parallel, we might + // actually race. + // + // The race is benign because (i) the write is + // idempotent -- the constructed anyfunc will always + // be exactly the same, once it is constructed, so a + // second write, even partially complete, is fine + // (even if fragmented/split; any fragment written + // will just overwrite the same bytes); and (ii) + // racing to set a bit in the bitmap might miss some + // updates, and falsely clear a bit, but that just + // means that we do another init later, which as per + // (i) is safe. + // + // The important safety property is that before we + // hand out a reference to an anyfunc, we have + // initialized it *at least once*, and this property + // is always true because (i) unrelated bits are never + // *set* in the bitmap spontaneously, even by a race; + // (ii) this function always verifies set bit before + // returning; and (iii) once initialized, an anyfunc + // stays initialized (idempotent writes discussed + // above). + // + // Note that we tried it the other way first, with + // atomics rather than benign races. Using an atomic + // in the anyfunc itself to tell whether it's + // initialized is expensive because we have to zero + // all anyfuncs at instantiation, and zeroing words, + // located sparsely in the vmcontext, is expensive + // compared to zeroing a dense bitmap. Using atomic + // updates to the bitmap is expensive because + // `fetch_or` is expensive; the `lock or` instruction + // on x86-64 was the majority of the profile in an + // instantiation microbenchmark. + *anyfunc = self.construct_anyfunc(index); + let (bitmap_word, bitmask) = self.get_anyfunc_bitmap_word(index); + *bitmap_word |= bitmask; + } + + let anyfunc = anyfunc.as_ref().unwrap(); + Some(anyfunc) + } } unsafe fn anyfunc_base(&self) -> *mut VMCallerCheckedAnyfunc { @@ -513,6 +612,7 @@ impl Instance { ptr::null_mut() } else { debug_assert!(idx.as_u32() < self.offsets.num_defined_functions); + self.get_caller_checked_anyfunc(*idx); // Force lazy init base.add(usize::try_from(idx.as_u32()).unwrap()) } }), diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 82c1eec31e8e..0f015601fcfb 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -13,7 +13,7 @@ use std::alloc; use std::any::Any; use std::convert::TryFrom; use std::marker; -use std::ptr::{self, NonNull}; +use std::ptr; use std::slice; use std::sync::Arc; use thiserror::Error; @@ -31,24 +31,34 @@ pub use self::pooling::{ InstanceLimits, ModuleLimits, PoolingAllocationStrategy, PoolingInstanceAllocator, }; -/// Represents a request for a new runtime instance. -pub struct InstanceAllocationRequest<'a> { - /// The module being instantiated. - pub module: Arc, - +/// Information needed for the instance allocation request and +/// afterward as well (for lazy initialization). Will be held alive by +/// the instance. +#[derive(Default)] +pub struct InstanceAllocationInfo { /// The base address of where JIT functions are located. pub image_base: usize, /// Descriptors about each compiled function, such as the offset from /// `image_base`. - pub functions: &'a PrimaryMap, + pub functions: Arc>, + + /// Translation from `SignatureIndex` to `VMSharedSignatureIndex` + pub shared_signatures: SharedSignatures, +} + +/// Represents a request for a new runtime instance. +pub struct InstanceAllocationRequest<'a> { + /// The info needed for both the allocation request and deferred + /// initialization. + pub info: Arc, + + /// The module being instantiated. + pub module: Arc, /// The imports to use for the instantiation. pub imports: Imports<'a>, - /// Translation from `SignatureIndex` to `VMSharedSignatureIndex` - pub shared_signatures: SharedSignatures<'a>, - /// The host state to associate with the instance. pub host_state: Box, @@ -213,17 +223,18 @@ pub unsafe trait InstanceAllocator: Send + Sync { unsafe fn deallocate_fiber_stack(&self, stack: &wasmtime_fiber::FiberStack); } -pub enum SharedSignatures<'a> { +/// Shared signatures. +pub enum SharedSignatures { /// Used for instantiating user-defined modules - Table(&'a PrimaryMap), + Table(PrimaryMap), /// Used for instance creation that has only a single function Always(VMSharedSignatureIndex), /// Used for instance creation that has no functions None, } -impl SharedSignatures<'_> { - fn lookup(&self, index: SignatureIndex) -> VMSharedSignatureIndex { +impl SharedSignatures { + pub(crate) fn lookup(&self, index: SignatureIndex) -> VMSharedSignatureIndex { match self { SharedSignatures::Table(table) => table[index], SharedSignatures::Always(index) => *index, @@ -232,14 +243,20 @@ impl SharedSignatures<'_> { } } -impl<'a> From for SharedSignatures<'a> { - fn from(val: VMSharedSignatureIndex) -> SharedSignatures<'a> { +impl std::default::Default for SharedSignatures { + fn default() -> Self { + SharedSignatures::None + } +} + +impl From for SharedSignatures { + fn from(val: VMSharedSignatureIndex) -> SharedSignatures { SharedSignatures::Always(val) } } -impl<'a> From> for SharedSignatures<'a> { - fn from(val: Option) -> SharedSignatures<'a> { +impl From> for SharedSignatures { + fn from(val: Option) -> SharedSignatures { match val { Some(idx) => SharedSignatures::Always(idx), None => SharedSignatures::None, @@ -247,9 +264,9 @@ impl<'a> From> for SharedSignatures<'a> { } } -impl<'a> From<&'a PrimaryMap> for SharedSignatures<'a> { - fn from(val: &'a PrimaryMap) -> SharedSignatures<'a> { - SharedSignatures::Table(val) +impl From<&PrimaryMap> for SharedSignatures { + fn from(val: &PrimaryMap) -> SharedSignatures { + SharedSignatures::Table(val.clone()) } } @@ -474,7 +491,7 @@ unsafe fn initialize_vmcontext(instance: &mut Instance, req: InstanceAllocationR let mut ptr = instance.vmctx_plus_offset(instance.offsets.vmctx_signature_ids_begin()); for sig in module.types.values() { *ptr = match sig { - ModuleType::Function(sig) => req.shared_signatures.lookup(*sig), + ModuleType::Function(sig) => req.info.shared_signatures.lookup(*sig), _ => VMSharedSignatureIndex::new(u32::max_value()), }; ptr = ptr.add(1); @@ -512,32 +529,12 @@ unsafe fn initialize_vmcontext(instance: &mut Instance, req: InstanceAllocationR req.imports.globals.len(), ); - // Initialize the functions - let mut base = instance.anyfunc_base(); - for (index, sig) in instance.module.functions.iter() { - let type_index = req.shared_signatures.lookup(*sig); - - let (func_ptr, vmctx) = if let Some(def_index) = instance.module.defined_func_index(index) { - ( - NonNull::new((req.image_base + req.functions[def_index].start as usize) as *mut _) - .unwrap(), - instance.vmctx_ptr(), - ) - } else { - let import = instance.imported_function(index); - (import.body, import.vmctx) - }; - - ptr::write( - base, - VMCallerCheckedAnyfunc { - func_ptr, - type_index, - vmctx, - }, - ); - base = base.add(1); - } + // Zero the anyfunc-initialized bitmap -- they will be lazily initialized as requred + let base = instance.vmctx_plus_offset(instance.offsets.vmctx_anyfuncs_init_begin()); + let bitmap_words = (instance.module.functions.len() + 63) / 64; + let len = bitmap_words * 8; + let slice = std::slice::from_raw_parts_mut(base as *mut u8, len); + slice.fill(0); // Initialize the defined tables let mut ptr = instance.vmctx_plus_offset(instance.offsets.vmctx_tables_begin()); @@ -693,6 +690,7 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { let mut handle = { let instance = Instance { module: req.module.clone(), + info: req.info.clone(), offsets: VMOffsets::new(HostPtr, &req.module), memories, tables, diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 76614137d5e9..93caf2670a95 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -11,6 +11,7 @@ use super::{ initialize_instance, initialize_vmcontext, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstantiationError, }; +use crate::InstanceAllocationInfo; use crate::{instance::Instance, Memory, Mmap, Table, VMContext}; use anyhow::{anyhow, bail, Context, Result}; use rand::Rng; @@ -357,6 +358,7 @@ impl InstancePool { instance as _, Instance { module: self.empty_module.clone(), + info: Arc::new(InstanceAllocationInfo::default()), offsets: VMOffsets::new(HostPtr, &self.empty_module), memories: PrimaryMap::with_capacity(limits.memories as usize), tables: PrimaryMap::with_capacity(limits.tables as usize), @@ -380,6 +382,7 @@ impl InstancePool { let instance = self.instance(index); instance.module = req.module.clone(); + instance.info = req.info.clone(); instance.offsets = VMOffsets::new(HostPtr, instance.module.as_ref()); instance.host_state = std::mem::replace(&mut req.host_state, Box::new(())); instance.wasm_data = &*req.wasm_data; diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index f96e7d8dda27..522bcadb6fdb 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -42,8 +42,8 @@ pub use crate::export::*; pub use crate::externref::*; pub use crate::imports::Imports; pub use crate::instance::{ - InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstantiationError, LinkError, - OnDemandInstanceAllocator, StorePtr, + InstanceAllocationInfo, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, + InstantiationError, LinkError, OnDemandInstanceAllocator, SharedSignatures, StorePtr, }; #[cfg(feature = "pooling-allocator")] pub use crate::instance::{ diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index 2544e1673987..4ae82a888257 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -64,7 +64,9 @@ use crate::vmcontext::{VMCallerCheckedAnyfunc, VMContext}; use backtrace::Backtrace; use std::mem; use std::ptr::{self, NonNull}; -use wasmtime_environ::{DataIndex, ElemIndex, GlobalIndex, MemoryIndex, TableIndex, TrapCode}; +use wasmtime_environ::{ + DataIndex, ElemIndex, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TrapCode, +}; const TOINT_32: f32 = 1.0 / f32::EPSILON; const TOINT_64: f64 = 1.0 / f64::EPSILON; @@ -379,6 +381,15 @@ pub unsafe extern "C" fn wasmtime_memory_init( } } +/// Implementation of `ref.func`. +pub unsafe extern "C" fn wasmtime_ref_func(vmctx: *mut VMContext, func_index: u32) -> usize { + let instance = (*vmctx).instance(); + let anyfunc = instance + .get_caller_checked_anyfunc(FuncIndex::from_u32(func_index)) + .unwrap(); + anyfunc as *const VMCallerCheckedAnyfunc as usize +} + /// Implementation of `data.drop`. pub unsafe extern "C" fn wasmtime_data_drop(vmctx: *mut VMContext, data_index: u32) { let data_index = DataIndex::from_u32(data_index); diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index d2832eb6ceef..7823402c0982 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -541,7 +541,7 @@ impl Default for VMSharedSignatureIndex { #[repr(C)] pub struct VMCallerCheckedAnyfunc { /// Function body. - pub func_ptr: NonNull, + pub func_ptr: *mut VMFunctionBody, /// Function signature id. pub type_index: VMSharedSignatureIndex, /// Function `VMContext`. @@ -611,6 +611,7 @@ impl VMBuiltinFunctionsArray { ptrs[BuiltinFunctionIndex::memory_copy().index() as usize] = wasmtime_memory_copy as usize; ptrs[BuiltinFunctionIndex::memory_fill().index() as usize] = wasmtime_memory_fill as usize; ptrs[BuiltinFunctionIndex::memory_init().index() as usize] = wasmtime_memory_init as usize; + ptrs[BuiltinFunctionIndex::ref_func().index() as usize] = wasmtime_ref_func as usize; ptrs[BuiltinFunctionIndex::data_drop().index() as usize] = wasmtime_data_drop as usize; ptrs[BuiltinFunctionIndex::drop_externref().index() as usize] = wasmtime_drop_externref as usize; diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 3f4b2ad496d9..e4de20ff8f9b 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -754,7 +754,7 @@ impl Func { trampoline( (*anyfunc.as_ptr()).vmctx, callee, - (*anyfunc.as_ptr()).func_ptr.as_ptr(), + (*anyfunc.as_ptr()).func_ptr, params_and_returns, ) }) @@ -947,7 +947,7 @@ impl Func { unsafe { let f = self.caller_checked_anyfunc(store); VMFunctionImport { - body: f.as_ref().func_ptr, + body: NonNull::new(f.as_ref().func_ptr).unwrap(), vmctx: f.as_ref().vmctx, } } diff --git a/crates/wasmtime/src/func/typed.rs b/crates/wasmtime/src/func/typed.rs index 4ddced224d1e..620188309236 100644 --- a/crates/wasmtime/src/func/typed.rs +++ b/crates/wasmtime/src/func/typed.rs @@ -160,12 +160,8 @@ where let result = invoke_wasm_and_catch_traps(store, |callee| { let (anyfunc, ret, params, returned) = &mut captures; let anyfunc = anyfunc.as_ref(); - let result = Params::invoke::( - anyfunc.func_ptr.as_ptr(), - anyfunc.vmctx, - callee, - *params, - ); + let result = + Params::invoke::(anyfunc.func_ptr, anyfunc.vmctx, callee, *params); ptr::write(ret.as_mut_ptr(), result); *returned = true }); diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index aec6c1ba06f6..42edc49123a8 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -701,16 +701,15 @@ impl<'a> Instantiator<'a> { // this instance, so we determine what the ID is and then assert // it's the same later when we do actually insert it. let instance_to_be = store.store_data().next_id::(); + let mut instance_handle = store .engine() .allocator() .allocate(InstanceAllocationRequest { module: compiled_module.module().clone(), - image_base: compiled_module.code().as_ptr() as usize, - functions: compiled_module.functions(), + info: self.cur.module.alloc_info().clone(), imports: self.cur.build(), - shared_signatures: self.cur.module.signatures().as_module_map().into(), host_state: Box::new(Instance(instance_to_be)), store: StorePtr::new(store.traitobj()), wasm_data: compiled_module.wasm_data(), @@ -827,9 +826,7 @@ impl<'a> Instantiator<'a> { mem::transmute::< *const VMFunctionBody, unsafe extern "C" fn(*mut VMContext, *mut VMContext), - >(f.anyfunc.as_ref().func_ptr.as_ptr())( - f.anyfunc.as_ref().vmctx, vmctx - ) + >(f.anyfunc.as_ref().func_ptr)(f.anyfunc.as_ref().vmctx, vmctx) })?; } Ok(()) diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 04c695f2141e..861a5be99540 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use wasmparser::{Parser, ValidPayload, Validator}; use wasmtime_environ::{ModuleEnvironment, ModuleIndex, PrimaryMap}; use wasmtime_jit::{CompiledModule, CompiledModuleInfo, MmapVec, TypeTables}; +use wasmtime_runtime::InstanceAllocationInfo; mod registry; mod serialization; @@ -107,6 +108,8 @@ struct ModuleInner { types: Arc, /// Registered shared signature for the module. signatures: Arc, + /// InstanceAllocationInfo shared by all instantiated modules, used for lazy initialization. + alloc_info: Arc, } impl Module { @@ -488,6 +491,17 @@ impl Module { module.into_module(engine) } + fn create_alloc_info( + module: &CompiledModule, + sigs: &SignatureCollection, + ) -> Arc { + Arc::new(InstanceAllocationInfo { + image_base: module.code().as_ptr() as usize, + functions: module.functions().clone(), + shared_signatures: sigs.as_module_map().into(), + }) + } + fn from_parts( engine: &Engine, mut modules: Vec>, @@ -508,6 +522,8 @@ impl Module { let module = modules.remove(main_module); + let alloc_info = Module::create_alloc_info(&module, &signatures); + let module_upvars = module_upvars .iter() .map(|m| { @@ -531,6 +547,7 @@ impl Module { artifact_upvars: modules, module_upvars, signatures, + alloc_info, }), }); @@ -543,6 +560,7 @@ impl Module { module_upvars: &[serialization::SerializedModuleUpvar], signatures: &Arc, ) -> Result { + let alloc_info = Module::create_alloc_info(&artifacts[module_index], signatures); Ok(Module { inner: Arc::new(ModuleInner { engine: engine.clone(), @@ -567,6 +585,7 @@ impl Module { }) .collect::>>()?, signatures: signatures.clone(), + alloc_info, }), }) } @@ -667,6 +686,10 @@ impl Module { module_upvars: &[wasmtime_environ::ModuleUpvar], modules: &PrimaryMap, ) -> Module { + let alloc_info = Module::create_alloc_info( + &self.inner.artifact_upvars[artifact_index], + &self.inner.signatures, + ); Module { inner: Arc::new(ModuleInner { types: self.inner.types.clone(), @@ -686,6 +709,7 @@ impl Module { }) .collect(), signatures: self.inner.signatures.clone(), + alloc_info, }), } } @@ -706,6 +730,10 @@ impl Module { &self.inner.signatures } + pub(crate) fn alloc_info(&self) -> &Arc { + &self.inner.alloc_info + } + /// Looks up the module upvar value at the `index` specified. /// /// Note that this panics if `index` is out of bounds since this should diff --git a/crates/wasmtime/src/module/registry.rs b/crates/wasmtime/src/module/registry.rs index 60bbbfffd783..a1a97e740622 100644 --- a/crates/wasmtime/src/module/registry.rs +++ b/crates/wasmtime/src/module/registry.rs @@ -95,7 +95,7 @@ impl ModuleRegistry { /// Looks up a trampoline from an anyfunc. pub fn lookup_trampoline(&self, anyfunc: &VMCallerCheckedAnyfunc) -> Option { - let module = self.module(anyfunc.func_ptr.as_ptr() as usize)?; + let module = self.module(anyfunc.func_ptr as usize)?; module.signatures.trampoline(anyfunc.type_index) } } diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index c6d7914e4742..73be2ef4f313 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -92,9 +92,10 @@ use std::sync::atomic::AtomicU64; use std::sync::Arc; use std::task::{Context, Poll}; use wasmtime_runtime::{ - InstanceAllocationRequest, InstanceAllocator, InstanceHandle, ModuleInfo, - OnDemandInstanceAllocator, SignalHandler, StorePtr, VMCallerCheckedAnyfunc, VMContext, - VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, VMTrampoline, + InstanceAllocationInfo, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, + ModuleInfo, OnDemandInstanceAllocator, SignalHandler, StorePtr, VMCallerCheckedAnyfunc, + VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, + VMTrampoline, }; mod context; @@ -404,7 +405,6 @@ impl Store { /// tables created to 10,000. This can be overridden with the /// [`Store::limiter`] configuration method. pub fn new(engine: &Engine, data: T) -> Self { - let functions = &Default::default(); // Wasmtime uses the callee argument to host functions to learn about // the original pointer to the `Store` itself, allowing it to // reconstruct a `StoreContextMut`. When we initially call a `Func`, @@ -413,16 +413,19 @@ impl Store { // part of `Func::call` to guarantee that the `callee: *mut VMContext` // is never null. let default_callee = unsafe { + let info = Arc::new(InstanceAllocationInfo { + image_base: 0, + functions: Default::default(), + shared_signatures: None.into(), + }); OnDemandInstanceAllocator::default() .allocate(InstanceAllocationRequest { host_state: Box::new(()), - image_base: 0, - functions, - shared_signatures: None.into(), imports: Default::default(), module: Arc::new(wasmtime_environ::Module::default()), store: StorePtr::empty(), wasm_data: &[], + info, }) .expect("failed to allocate default callee") }; diff --git a/crates/wasmtime/src/trampoline.rs b/crates/wasmtime/src/trampoline.rs index c1f8038a5ace..b2835bb381da 100644 --- a/crates/wasmtime/src/trampoline.rs +++ b/crates/wasmtime/src/trampoline.rs @@ -18,8 +18,8 @@ use std::any::Any; use std::sync::Arc; use wasmtime_environ::{EntityIndex, GlobalIndex, MemoryIndex, Module, TableIndex}; use wasmtime_runtime::{ - Imports, InstanceAllocationRequest, InstanceAllocator, OnDemandInstanceAllocator, StorePtr, - VMFunctionImport, VMSharedSignatureIndex, + Imports, InstanceAllocationInfo, InstanceAllocationRequest, InstanceAllocator, + OnDemandInstanceAllocator, StorePtr, VMFunctionImport, VMSharedSignatureIndex, }; fn create_handle( @@ -31,7 +31,7 @@ fn create_handle( ) -> Result { let mut imports = Imports::default(); imports.functions = func_imports; - let functions = &Default::default(); + let functions = Default::default(); unsafe { let config = store.engine().config(); @@ -41,13 +41,15 @@ fn create_handle( let handle = OnDemandInstanceAllocator::new(config.mem_creator.clone(), 0).allocate( InstanceAllocationRequest { module: Arc::new(module), - functions, - image_base: 0, imports, - shared_signatures: shared_signature_id.into(), host_state, store: StorePtr::new(store.traitobj()), wasm_data: &[], + info: Arc::new(InstanceAllocationInfo { + functions, + image_base: 0, + shared_signatures: shared_signature_id.into(), + }), }, )?; diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 67d57fc3345d..dc8944fb7288 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use wasmtime_environ::{EntityIndex, Module, ModuleType, PrimaryMap, SignatureIndex}; use wasmtime_jit::{CodeMemory, MmapVec, ProfilingAgent}; use wasmtime_runtime::{ - Imports, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, + Imports, InstanceAllocationInfo, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, StorePtr, VMContext, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, }; @@ -158,16 +158,19 @@ pub unsafe fn create_raw_function( .exports .insert(String::new(), EntityIndex::Function(func_id)); + let info = Arc::new(InstanceAllocationInfo { + functions: Arc::new(functions), + image_base: (*func).as_ptr() as usize, + shared_signatures: sig.into(), + }); Ok( OnDemandInstanceAllocator::default().allocate(InstanceAllocationRequest { module: Arc::new(module), - functions: &functions, - image_base: (*func).as_ptr() as usize, imports: Imports::default(), - shared_signatures: sig.into(), host_state, store: StorePtr::empty(), wasm_data: &[], + info, })?, ) } diff --git a/crates/wasmtime/src/trampoline/global.rs b/crates/wasmtime/src/trampoline/global.rs index 3feb599b26ee..5c6a8c9f519a 100644 --- a/crates/wasmtime/src/trampoline/global.rs +++ b/crates/wasmtime/src/trampoline/global.rs @@ -1,3 +1,5 @@ +use std::ptr::NonNull; + use crate::store::{InstanceId, StoreOpaque}; use crate::trampoline::create_handle; use crate::{GlobalType, Mutability, Val}; @@ -51,7 +53,7 @@ pub fn create_global(store: &mut StoreOpaque, gt: &GlobalType, val: Val) -> Resu }); func_imports.push(VMFunctionImport { - body: f.func_ptr, + body: NonNull::new(f.func_ptr).unwrap(), vmctx: f.vmctx, });