diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 05da364dede1..54a6fe61b21f 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -460,6 +460,7 @@ impl Instance { /// (imported or defined in this module) and store into the given /// location. Used during lazy initialization the first time the /// VMCallerCheckedAnyfunc in the VMContext is referenced. + #[cold] fn construct_anyfunc(&mut self, index: FuncIndex, into: *mut VMCallerCheckedAnyfunc) { let sig = self.module.functions[index]; let type_index = self.runtime_info.shared_signatures().lookup(sig); diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 4e81c6b9ebee..4142c7e9faeb 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -481,12 +481,17 @@ fn initialize_instance( /// Initialize the VMContext data associated with an Instance. /// -/// Precondition: the VMContext memory must be zeroed. We omit writes -/// here to fields that should be initialized to zero. This allows the -/// caller to use an efficient means of zeroing memory (such as using -/// anonymous-mmap'd zero pages) if available, rather than falling -/// back onto a memset (or the manual equivalent) here. -unsafe fn initialize_vmcontext(instance: &mut Instance, req: InstanceAllocationRequest) { +/// Optimization: if `pre_zeroed` is set, then the VMContext memory +/// (following the `Instance`) is assumed to be zeroed. If so, we omit +/// writes here to fields that should be initialized to zero. This +/// allows the caller to use an efficient means of zeroing memory +/// (such as using anonymous-mmap'd zero pages) if available, rather +/// than falling back onto a memset (or the manual equivalent) here. +unsafe fn initialize_vmcontext( + instance: &mut Instance, + req: InstanceAllocationRequest, + pre_zeroed: bool, +) { if let Some(store) = req.store.as_raw() { *instance.interrupts() = (*store).vminterrupts(); *instance.epoch_ptr() = (*store).epoch_ptr(); @@ -536,9 +541,27 @@ unsafe fn initialize_vmcontext(instance: &mut Instance, req: InstanceAllocationR req.imports.globals.len(), ); - // N.B.: no need to initialize the anyfuncs array; it is zeroed - // already (by the precondition for this function) and the lazy - // init will fill in entries as we take pointers to them. + // N.B.: if `pre_zeroed`, then there is no need to initialize the + // anyfuncs array; it is zeroed already (by the precondition for + // this function) and the lazy init will fill in entries as we + // take pointers to them. + // + // If so, we'll debug_assert that it's all zeroes; if not, we'll + // fill it. + let anyfuncs_bytes = std::slice::from_raw_parts_mut( + instance.anyfunc_base() as *mut u8, + instance + .module + .functions + .len() + .checked_mul(std::mem::size_of::()) + .unwrap(), + ); + if pre_zeroed { + debug_assert!(anyfuncs_bytes.iter().all(|b| *b == 0)); + } else { + anyfuncs_bytes.fill(0); + } // Initialize the defined tables let mut ptr = instance.vmctx_plus_offset(instance.offsets.vmctx_tables_begin()); @@ -705,7 +728,7 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { let host_state = std::mem::replace(&mut req.host_state, Box::new(())); - let (mut handle, vmctx_data) = { + let mut handle = { let instance = Instance::create_raw( &req.module, req.runtime_info.clone(), @@ -724,25 +747,10 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { ptr::write(instance_ptr, instance); - let vmctx_data = { - let vmctx_ptr = (*instance_ptr).vmctx_ptr() as *mut u8; - let vmctx_len = (*instance_ptr).offsets.size_of_vmctx() as usize; - std::slice::from_raw_parts_mut(vmctx_ptr, vmctx_len) - }; - - (handle, vmctx_data) + handle }; - // Zero the VMContext memory first; `initialize_vmcontext()` - // requires this. - vmctx_data.fill(0); - // Drop the &mut slice over the VMContext data before writing - // to it via `initialize_vmcontext()` below; we should not let - // its lifetime extend over the function call that initializes - // the VMContext. - drop(vmctx_data); - - initialize_vmcontext(handle.instance_mut(), req); + initialize_vmcontext(handle.instance_mut(), req, /* pre_zeroed = */ false); Ok(handle) } diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 99b4d604062a..986ec97973c5 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -399,7 +399,7 @@ impl InstancePool { // will use either madvise(MADV_DONTNEED) (on Linux) or // mmap-of-fresh-anonymous-memory to efficiently zero the // instance state. - initialize_vmcontext(instance, req); + initialize_vmcontext(instance, req, /* pre_zeroed = */ true); Ok(InstanceHandle { instance: instance as _,